def set_outputs_store(self, outputs_store=None, outputs_path=None, set_env_vars=False): if not any([outputs_store, outputs_path]): raise PolyaxonClientException( 'An Store instance or and outputs path is required.') self.outputs_store = outputs_store or StoreManager(path=outputs_path) if self.outputs_store and set_env_vars: self.outputs_store.set_env_vars()
def __init__(self, project=None, client=None, track_logs=True, track_code=True, track_env=True, outputs_store=None): if settings.NO_OP: return if not settings.IN_CLUSTER and project is None: raise PolyaxonClientException('Please provide a valid project.') self.client = client or PolyaxonClient() if settings.IN_CLUSTER: self.user = None else: self.user = (self.client.auth.get_user().username if self.client.api_config.schema_response else self.client.auth.get_user().get('username')) username, project_name = get_project_info(current_user=self.user, project=project) self.track_logs = track_logs self.track_code = track_code self.track_env = track_env self.project = project self.username = username self.project_name = project_name self.outputs_store = outputs_store # Setup the outputs store if outputs_store is None and settings.IN_CLUSTER: self.set_outputs_store(outputs_path=get_outputs_path(), set_env_vars=True)
def client_backend(self): if settings.NO_OP: return None if self.job_type == 'jobs': return self.client.job elif self.job_type == 'builds': return self.client.build_job raise PolyaxonClientException('Job type {} not supported'.format(self.job_type))
def validate_config(config, config_schema): if isinstance(config, Mapping): return config_schema.from_dict(config, unknown=settings.VALIDATION_UNKNOWN_BEHAVIOUR) elif not isinstance(config, config_schema): raise PolyaxonClientException( 'Received an invalid config. ' 'Expects a Mapping or an instance of `{}`.'.format(config_schema.__name__)) return config
def validate_config(config, config_schema): if isinstance(config, Mapping): return config_schema.from_dict(config) elif not isinstance(config, config_schema): raise PolyaxonClientException( 'Received an invalid config. ' 'Expects a Mapping or an instance of `{}`.'.format( config_schema.__name__)) return config
def parse_numbers(value): if isinstance(value, (int, float, complex, type(None))): return value if isinstance(value, np.integer): return int(value) if isinstance(value, np.floating): return float(value) raise PolyaxonClientException( 'Client could not parse the value `{}`, it expects a number.'. format(value))
def create(self, name=None, backend=None, tags=None, description=None, content=None, build_id=None, base_outputs_path=None): job_config = {'run_env': get_run_env()} if self.track_env else {} if name: job_config['name'] = name if tags: job_config['tags'] = tags job_config['backend'] = OTHER_BACKEND if backend: job_config['backend'] = backend if description: job_config['description'] = description if build_id: job_config['build_job'] = str(build_id) job_config['is_managed'] = settings.IS_MANAGED job = None if self.client: if content: job_config['content'] = self.client.project.validate_content(content=content) job = self.client.project.create_job( username=self.username, project_name=self.project_name, job_config=job_config, ) if not job: raise PolyaxonClientException('Could not create job.') if not settings.IS_MANAGED and self.track_logs: setup_logging(send_logs=self.send_logs) self.job_id = self._get_entity_id(job) self.job = job self.last_status = 'created' # Setup the outputs store base_outputs_path = base_outputs_path or get_base_outputs_path() if self.outputs_store is None and base_outputs_path: outputs_path = '{}/{}/{}/jobs/{}'.format( base_outputs_path, self.username, self.project_name, self.job_id) self.set_outputs_store(outputs_path=outputs_path) if self.track_code: self.log_code_ref() if not settings.IS_MANAGED: self._start() self._set_health_url() return self
def get_project_info(current_user, project): if not project: raise PolyaxonClientException( 'You need to provide a project for tracking experiments.') parts = project.replace('.', '/').split('/') if len(parts) == 2: user, project_name = parts else: user = current_user project_name = project return user, project_name
def create(self, name=None, tags=None, description=None, config=None, base_outputs_path=None): if settings.NO_OP: return None experiment_config = {'run_env': get_run_env()} if self.track_env else {} if name: experiment_config['name'] = name if tags: experiment_config['tags'] = tags if description: experiment_config['description'] = description if config: experiment_config['config'] = config experiment = self.client.project.create_experiment( username=self.username, project_name=self.project_name, experiment_config=experiment_config, group=self.group_id, ) if not experiment: raise PolyaxonClientException('Could not create experiment.') if not settings.IN_CLUSTER and self.track_logs: setup_logging(PolyaxonHandler(send_logs=self._send_logs)) self.experiment_id = (experiment.id if self.client.api_config.schema_response else experiment.get('id')) self.experiment = experiment self.last_status = 'created' # Setup the outputs store base_outputs_path = base_outputs_path or get_base_outputs_path() if self.outputs_store is None and base_outputs_path: if self.group_id: outputs_path = '{}/{}/{}/{}/{}'.format( base_outputs_path, self.username, self.project_name, self.group_id, self.experiment_id) else: outputs_path = '{}/{}/{}/{}'.format( base_outputs_path, self.username, self.project_name, self.experiment_id) self.set_outputs_store(outputs_path=outputs_path) if self.track_code: self.log_code_ref() if not settings.IN_CLUSTER: self._start() self._set_health_url() return self
def get_periodic_http_worker(self, **kwargs): worker = self.periodic_http_worker if not worker or not worker.is_alive(): if 'request' not in kwargs: raise PolyaxonClientException( 'Periodic worker expects a request argument.') self._periodic_http_worker = PeriodicWorker( callback=self.queue_periodic_request, worker_interval=self.config.interval, worker_timeout=self.config.timeout, kwargs=kwargs) self._periodic_http_worker.start() return self.periodic_http_worker
def validate_content(content): if not content: return content if isinstance(content, Mapping): return '{}'.format(content) if isinstance(content, BaseConfig): return '{}'.format(content.to_dict()) if isinstance(content, six.string_types): return content if isinstance(content, BaseSpecification): return content.raw_data raise PolyaxonClientException( 'Received an invalid content. ' 'Expects a Str, a Mapping or a valid instance of the entity config.')
def get_entity_data(self): if self.job_type == 'jobs': self._entity_data = self.client_backend.get_job( username=self.username, project_name=self.project_name, job_id=self.job_id) return elif self.job_type == 'builds': self._entity_data = self.client_backend.get_build( username=self.username, project_name=self.project_name, job_id=self.job_id) return raise PolyaxonClientException('Job type {} not supported'.format(self.job_type))
def validate_tags(tags): if not tags: return None if isinstance(tags, six.string_types): tags = [tag.strip() for tag in tags.split(',')] for tag in to_list(tags): if not (tag and isinstance(tag, six.string_types)): raise PolyaxonClientException( 'Received an invalid tag {}. ' 'Please make sure that the tags are list of strings, ' 'or a string containing comma separated values.'.format(tag)) return tags
def create(self, name=None, backend=None, tags=None, description=None, content=None): build_config = {} if name: build_config['name'] = name if tags: build_config['tags'] = tags build_config['backend'] = OTHER_BACKEND if backend: build_config['backend'] = backend if description: build_config['description'] = description build_config['is_managed'] = settings.IS_MANAGED build = None if self.client: if content: build_config['content'] = self.client.project.validate_content( content=content) build = self.client.project.create_build( username=self.username, project_name=self.project_name, build_config=build_config, ) if not build: raise PolyaxonClientException('Could not create build.') if not settings.IS_MANAGED and self.track_logs: setup_logging(send_logs=self.send_logs) self.job_id = self._get_entity_id(build) self.job = build self.last_status = 'created' if self.track_code: self.log_code_ref() if not settings.IS_MANAGED: self._start() self._set_health_url() return self
def __init__(self, project=None, client=None, track_logs=True, track_code=True, track_env=True, outputs_store=None): if not settings.IS_MANAGED and project is None: raise PolyaxonClientException('Please provide a valid project.') elif self.is_notebook_job: job_info = self.get_notebook_job_info() project = job_info['project_name'] self.last_status = None self.client = client self.user = None if not (self.client or settings.IS_OFFLINE): self.client = PolyaxonClient() if self.client and not settings.IS_MANAGED: self.user = (self.client.auth.get_user().username if self.client.api_config.schema_response else self.client.auth.get_user().get('username')) username, project_name = get_project_info(current_user=self.user, project=project) self.track_logs = track_logs self.track_code = track_code self.track_env = track_env self.project = project self.username = username self.project_name = project_name self.outputs_store = outputs_store self._entity_data = None self._health_is_running = False # Setup the outputs store if outputs_store is None and settings.IS_MANAGED and self.REQUIRES_OUTPUTS: self.set_outputs_store(outputs_path=get_outputs_path(), set_env_vars=True)
def __init__(self, host=None, http_port=None, ws_port=None, token=None, version=None, authentication_type=None, use_https=None, verify_ssl=None, in_cluster=None, schema_response=None, reraise=False, timeout=None, interval=None): self.token = token or settings.SECRET_USER_TOKEN self.host = host or settings.API_HOST self.in_cluster = self._get_bool(in_cluster, settings.IN_CLUSTER) self.use_https = self._get_bool(use_https, settings.USE_HTTPS) self.verify_ssl = self._get_bool(verify_ssl, settings.VERIFY_SSL) if not self.host and not self.in_cluster: raise PolyaxonClientException( 'Api config requires at least a host if not running in-cluster.' ) self.http_port = http_port or settings.HTTP_PORT or ( settings.DEFAULT_HTTPS_PORT if self.use_https else settings.DEFAULT_HTTP_PORT) self.ws_port = ws_port or settings.WS_PORT or ( settings.DEFAULT_HTTPS_PORT if self.use_https else settings.DEFAULT_HTTP_PORT) self.version = version or settings.API_VERSION self.internal_header = None if self.in_cluster: if not settings.API_HTTP_HOST: print( 'Could get api host info, ' 'please make sure this is running inside a polyaxon job.') self.http_host = settings.API_HTTP_HOST self.ws_host = settings.API_WS_HOST if all( [settings.INTERNAL_HEADER, settings.INTERNAL_HEADER_SERVICE]): self.internal_header = { settings.INTERNAL_HEADER: settings.INTERNAL_HEADER_SERVICE } internal_token_cond = (self.internal_header and not self.token and settings.SECRET_INTERNAL_TOKEN) if internal_token_cond: self.token = settings.SECRET_INTERNAL_TOKEN else: http_protocol = 'https' if self.use_https else 'http' ws_protocol = 'wss' if self.use_https else 'ws' self.http_host = '{}://{}:{}'.format(http_protocol, self.host, self.http_port) self.ws_host = '{}://{}:{}'.format(ws_protocol, self.host, self.ws_port) self.base_url = self.BASE_URL.format(self.http_host, self.version) self.base_ws_url = self.BASE_WS_URL.format(self.ws_host, self.version) self.authentication_type = (authentication_type or settings.AUTHENTICATION_TYPE or AuthenticationTypes.TOKEN) self.schema_response = self._get_bool(schema_response, settings.SCHEMA_RESPONSE) self.reraise = reraise self.timeout = timeout if timeout is not None else settings.TIMEOUT self.interval = interval if timeout is not None else settings.INTERVAL
def create(self, name=None, framework=None, backend=None, tags=None, description=None, content=None, build_id=None, base_outputs_path=None): experiment_config = {'run_env': get_run_env()} if self.track_env else {} if name: experiment_config['name'] = name if tags: experiment_config['tags'] = tags if framework: experiment_config['framework'] = framework experiment_config['backend'] = OTHER_BACKEND if backend: experiment_config['backend'] = backend if description: experiment_config['description'] = description if build_id: experiment_config['build_job'] = str(build_id) experiment_config['is_managed'] = settings.IS_MANAGED experiment = None if self.client: if content: experiment_config['content'] = self.client.project.validate_content(content=content) experiment = self.client.project.create_experiment( username=self.username, project_name=self.project_name, experiment_config=experiment_config, group=self.group_id, ) if not experiment: raise PolyaxonClientException('Could not create experiment.') if not settings.IS_MANAGED and self.track_logs: setup_logging(send_logs=self.send_logs) self.experiment_id = self._get_entity_id(experiment) self.experiment = experiment self.last_status = 'created' # Setup the outputs store base_outputs_path = base_outputs_path or get_base_outputs_path() if self.outputs_store is None and base_outputs_path: if self.group_id: outputs_path = '{}/{}/{}/{}/{}'.format( base_outputs_path, self.username, self.project_name, self.group_id, self.experiment_id) else: outputs_path = '{}/{}/{}/{}'.format( base_outputs_path, self.username, self.project_name, self.experiment_id) self.set_outputs_store(outputs_path=outputs_path) if self.track_code: self.log_code_ref() if not settings.IS_MANAGED: self._start() self._set_health_url() return self
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function from polyaxon_client.exceptions import PolyaxonClientException from polyaxon_client.tracking import Experiment from polyaxon_client import settings try: from tensorflow.train import LoggingTensorHook except ImportError: raise PolyaxonClientException( 'tensorflow is required to use PolyaxonLoggingTensorHook') class PolyaxonLoggingTensorHook(LoggingTensorHook): """Hook that logs data to console and Polyaxon""" def __init__(self, tensors, experiment=None, every_n_iter=None, every_n_secs=None): super(PolyaxonLoggingTensorHook, self).__init__(tensors=tensors, every_n_iter=every_n_iter, every_n_secs=every_n_secs) self.experiment = experiment if settings.IS_MANAGED: self.experiment = self.experiment or Experiment() def _log_tensors(self, tensor_values): super(PolyaxonLoggingTensorHook, self)._log_tensors(tensor_values)
def ensure_is_managed(): if not settings.IS_MANAGED and not settings.IN_CLUSTER: raise PolyaxonClientException( 'This experiment/job is not managed by Polyaxon.')
def ensure_in_custer(): if not settings.IN_CLUSTER: raise PolyaxonClientException( 'This experiment/job is not running inside a Polyaxon job.')
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function from polyaxon_client import settings from polyaxon_client.exceptions import PolyaxonClientException from polyaxon_client.tracking import Experiment try: from keras.callbacks import Callback except ImportError: try: from tensorflow.keras.callbacks import Callback except ImportError: raise PolyaxonClientException('Keras is required to use PolyaxonKeras') class PolyaxonKeras(Callback): def __init__(self, experiment=None, metrics=None): self.experiment = experiment if settings.IS_MANAGED: self.experiment = self.experiment or Experiment() self.metrics = metrics def on_epoch_end(self, epoch, logs=None): if not logs or not self.experiment: return if self.metrics: metrics = { metric: logs[metric] for metric in self.metrics if metric in logs }
# -*- coding: utf-8 -*- from __future__ import absolute_import, division, print_function from polyaxon_client import settings from polyaxon_client.exceptions import PolyaxonClientException from polyaxon_client.tracking import Experiment try: from fastai.callbacks import TrackerCallback except ImportError: raise PolyaxonClientException('Fastai is required to use PolyaxonFastai') class PolyaxonFastai(TrackerCallback): def __init__(self, learn, experiment=None, monitor='val_loss', mode='auto'): super(PolyaxonFastai, self).__init__(learn, monitor=monitor, mode=mode) self.experiment = experiment if settings.IS_MANAGED: self.experiment = self.experiment or Experiment() def on_epoch_end(self, epoch, smooth_loss, last_metrics, **kwargs): if not self.experiment: return metrics = { name: stat for name, stat in list( zip(self.learn.recorder.names, [epoch, smooth_loss] +