def set_outputs_store(self, outputs_store=None, outputs_path=None, set_env_vars=False):
if not any([outputs_store, outputs_path]):
raise PolyaxonClientException(
'An Store instance or and outputs path is required.')
self.outputs_store = outputs_store or StoreManager(path=outputs_path)
if self.outputs_store and set_env_vars:
self.outputs_store.set_env_vars()
def __init__(self,
project=None,
client=None,
track_logs=True,
track_code=True,
track_env=True,
outputs_store=None):
if settings.NO_OP:
return
if not settings.IN_CLUSTER and project is None:
raise PolyaxonClientException('Please provide a valid project.')
self.client = client or PolyaxonClient()
if settings.IN_CLUSTER:
self.user = None
else:
self.user = (self.client.auth.get_user().username
if self.client.api_config.schema_response else
self.client.auth.get_user().get('username'))
username, project_name = get_project_info(current_user=self.user,
project=project)
self.track_logs = track_logs
self.track_code = track_code
self.track_env = track_env
self.project = project
self.username = username
self.project_name = project_name
self.outputs_store = outputs_store
# Setup the outputs store
if outputs_store is None and settings.IN_CLUSTER:
self.set_outputs_store(outputs_path=get_outputs_path(),
set_env_vars=True)
def client_backend(self):
if settings.NO_OP:
return None
if self.job_type == 'jobs':
return self.client.job
elif self.job_type == 'builds':
return self.client.build_job
raise PolyaxonClientException('Job type {} not supported'.format(self.job_type))
def validate_config(config, config_schema):
if isinstance(config, Mapping):
return config_schema.from_dict(config, unknown=settings.VALIDATION_UNKNOWN_BEHAVIOUR)
elif not isinstance(config, config_schema):
raise PolyaxonClientException(
'Received an invalid config. '
'Expects a Mapping or an instance of `{}`.'.format(config_schema.__name__))
return config
def validate_config(config, config_schema):
if isinstance(config, Mapping):
return config_schema.from_dict(config)
elif not isinstance(config, config_schema):
raise PolyaxonClientException(
'Received an invalid config. '
'Expects a Mapping or an instance of `{}`.'.format(
config_schema.__name__))
return config
def parse_numbers(value):
if isinstance(value, (int, float, complex, type(None))):
return value
if isinstance(value, np.integer):
return int(value)
if isinstance(value, np.floating):
return float(value)
raise PolyaxonClientException(
'Client could not parse the value `{}`, it expects a number.'.
format(value))
def create(self,
name=None,
backend=None,
tags=None,
description=None,
content=None,
build_id=None,
base_outputs_path=None):
job_config = {'run_env': get_run_env()} if self.track_env else {}
if name:
job_config['name'] = name
if tags:
job_config['tags'] = tags
job_config['backend'] = OTHER_BACKEND
if backend:
job_config['backend'] = backend
if description:
job_config['description'] = description
if build_id:
job_config['build_job'] = str(build_id)
job_config['is_managed'] = settings.IS_MANAGED
job = None
if self.client:
if content:
job_config['content'] = self.client.project.validate_content(content=content)
job = self.client.project.create_job(
username=self.username,
project_name=self.project_name,
job_config=job_config,
)
if not job:
raise PolyaxonClientException('Could not create job.')
if not settings.IS_MANAGED and self.track_logs:
setup_logging(send_logs=self.send_logs)
self.job_id = self._get_entity_id(job)
self.job = job
self.last_status = 'created'
# Setup the outputs store
base_outputs_path = base_outputs_path or get_base_outputs_path()
if self.outputs_store is None and base_outputs_path:
outputs_path = '{}/{}/{}/jobs/{}'.format(
base_outputs_path, self.username, self.project_name, self.job_id)
self.set_outputs_store(outputs_path=outputs_path)
if self.track_code:
self.log_code_ref()
if not settings.IS_MANAGED:
self._start()
self._set_health_url()
return self
def get_project_info(current_user, project):
if not project:
raise PolyaxonClientException(
'You need to provide a project for tracking experiments.')
parts = project.replace('.', '/').split('/')
if len(parts) == 2:
user, project_name = parts
else:
user = current_user
project_name = project
return user, project_name
def create(self, name=None, tags=None, description=None, config=None, base_outputs_path=None):
if settings.NO_OP:
return None
experiment_config = {'run_env': get_run_env()} if self.track_env else {}
if name:
experiment_config['name'] = name
if tags:
experiment_config['tags'] = tags
if description:
experiment_config['description'] = description
if config:
experiment_config['config'] = config
experiment = self.client.project.create_experiment(
username=self.username,
project_name=self.project_name,
experiment_config=experiment_config,
group=self.group_id,
)
if not experiment:
raise PolyaxonClientException('Could not create experiment.')
if not settings.IN_CLUSTER and self.track_logs:
setup_logging(PolyaxonHandler(send_logs=self._send_logs))
self.experiment_id = (experiment.id
if self.client.api_config.schema_response
else experiment.get('id'))
self.experiment = experiment
self.last_status = 'created'
# Setup the outputs store
base_outputs_path = base_outputs_path or get_base_outputs_path()
if self.outputs_store is None and base_outputs_path:
if self.group_id:
outputs_path = '{}/{}/{}/{}/{}'.format(
base_outputs_path,
self.username,
self.project_name,
self.group_id,
self.experiment_id)
else:
outputs_path = '{}/{}/{}/{}'.format(
base_outputs_path, self.username, self.project_name, self.experiment_id)
self.set_outputs_store(outputs_path=outputs_path)
if self.track_code:
self.log_code_ref()
if not settings.IN_CLUSTER:
self._start()
self._set_health_url()
return self
def get_periodic_http_worker(self, **kwargs):
worker = self.periodic_http_worker
if not worker or not worker.is_alive():
if 'request' not in kwargs:
raise PolyaxonClientException(
'Periodic worker expects a request argument.')
self._periodic_http_worker = PeriodicWorker(
callback=self.queue_periodic_request,
worker_interval=self.config.interval,
worker_timeout=self.config.timeout,
kwargs=kwargs)
self._periodic_http_worker.start()
return self.periodic_http_worker
def validate_content(content):
if not content:
return content
if isinstance(content, Mapping):
return '{}'.format(content)
if isinstance(content, BaseConfig):
return '{}'.format(content.to_dict())
if isinstance(content, six.string_types):
return content
if isinstance(content, BaseSpecification):
return content.raw_data
raise PolyaxonClientException(
'Received an invalid content. '
'Expects a Str, a Mapping or a valid instance of the entity config.')
def get_entity_data(self):
if self.job_type == 'jobs':
self._entity_data = self.client_backend.get_job(
username=self.username,
project_name=self.project_name,
job_id=self.job_id)
return
elif self.job_type == 'builds':
self._entity_data = self.client_backend.get_build(
username=self.username,
project_name=self.project_name,
job_id=self.job_id)
return
raise PolyaxonClientException('Job type {} not supported'.format(self.job_type))
def validate_tags(tags):
if not tags:
return None
if isinstance(tags, six.string_types):
tags = [tag.strip() for tag in tags.split(',')]
for tag in to_list(tags):
if not (tag and isinstance(tag, six.string_types)):
raise PolyaxonClientException(
'Received an invalid tag {}. '
'Please make sure that the tags are list of strings, '
'or a string containing comma separated values.'.format(tag))
return tags
def create(self,
name=None,
backend=None,
tags=None,
description=None,
content=None):
build_config = {}
if name:
build_config['name'] = name
if tags:
build_config['tags'] = tags
build_config['backend'] = OTHER_BACKEND
if backend:
build_config['backend'] = backend
if description:
build_config['description'] = description
build_config['is_managed'] = settings.IS_MANAGED
build = None
if self.client:
if content:
build_config['content'] = self.client.project.validate_content(
content=content)
build = self.client.project.create_build(
username=self.username,
project_name=self.project_name,
build_config=build_config,
)
if not build:
raise PolyaxonClientException('Could not create build.')
if not settings.IS_MANAGED and self.track_logs:
setup_logging(send_logs=self.send_logs)
self.job_id = self._get_entity_id(build)
self.job = build
self.last_status = 'created'
if self.track_code:
self.log_code_ref()
if not settings.IS_MANAGED:
self._start()
self._set_health_url()
return self
def __init__(self,
project=None,
client=None,
track_logs=True,
track_code=True,
track_env=True,
outputs_store=None):
if not settings.IS_MANAGED and project is None:
raise PolyaxonClientException('Please provide a valid project.')
elif self.is_notebook_job:
job_info = self.get_notebook_job_info()
project = job_info['project_name']
self.last_status = None
self.client = client
self.user = None
if not (self.client or settings.IS_OFFLINE):
self.client = PolyaxonClient()
if self.client and not settings.IS_MANAGED:
self.user = (self.client.auth.get_user().username
if self.client.api_config.schema_response else
self.client.auth.get_user().get('username'))
username, project_name = get_project_info(current_user=self.user,
project=project)
self.track_logs = track_logs
self.track_code = track_code
self.track_env = track_env
self.project = project
self.username = username
self.project_name = project_name
self.outputs_store = outputs_store
self._entity_data = None
self._health_is_running = False
# Setup the outputs store
if outputs_store is None and settings.IS_MANAGED and self.REQUIRES_OUTPUTS:
self.set_outputs_store(outputs_path=get_outputs_path(),
set_env_vars=True)
def __init__(self,
host=None,
http_port=None,
ws_port=None,
token=None,
version=None,
authentication_type=None,
use_https=None,
verify_ssl=None,
in_cluster=None,
schema_response=None,
reraise=False,
timeout=None,
interval=None):
self.token = token or settings.SECRET_USER_TOKEN
self.host = host or settings.API_HOST
self.in_cluster = self._get_bool(in_cluster, settings.IN_CLUSTER)
self.use_https = self._get_bool(use_https, settings.USE_HTTPS)
self.verify_ssl = self._get_bool(verify_ssl, settings.VERIFY_SSL)
if not self.host and not self.in_cluster:
raise PolyaxonClientException(
'Api config requires at least a host if not running in-cluster.'
)
self.http_port = http_port or settings.HTTP_PORT or (
settings.DEFAULT_HTTPS_PORT
if self.use_https else settings.DEFAULT_HTTP_PORT)
self.ws_port = ws_port or settings.WS_PORT or (
settings.DEFAULT_HTTPS_PORT
if self.use_https else settings.DEFAULT_HTTP_PORT)
self.version = version or settings.API_VERSION
self.internal_header = None
if self.in_cluster:
if not settings.API_HTTP_HOST:
print(
'Could get api host info, '
'please make sure this is running inside a polyaxon job.')
self.http_host = settings.API_HTTP_HOST
self.ws_host = settings.API_WS_HOST
if all(
[settings.INTERNAL_HEADER, settings.INTERNAL_HEADER_SERVICE]):
self.internal_header = {
settings.INTERNAL_HEADER: settings.INTERNAL_HEADER_SERVICE
}
internal_token_cond = (self.internal_header and not self.token
and settings.SECRET_INTERNAL_TOKEN)
if internal_token_cond:
self.token = settings.SECRET_INTERNAL_TOKEN
else:
http_protocol = 'https' if self.use_https else 'http'
ws_protocol = 'wss' if self.use_https else 'ws'
self.http_host = '{}://{}:{}'.format(http_protocol, self.host,
self.http_port)
self.ws_host = '{}://{}:{}'.format(ws_protocol, self.host,
self.ws_port)
self.base_url = self.BASE_URL.format(self.http_host, self.version)
self.base_ws_url = self.BASE_WS_URL.format(self.ws_host, self.version)
self.authentication_type = (authentication_type
or settings.AUTHENTICATION_TYPE
or AuthenticationTypes.TOKEN)
self.schema_response = self._get_bool(schema_response,
settings.SCHEMA_RESPONSE)
self.reraise = reraise
self.timeout = timeout if timeout is not None else settings.TIMEOUT
self.interval = interval if timeout is not None else settings.INTERVAL
def create(self,
name=None,
framework=None,
backend=None,
tags=None,
description=None,
content=None,
build_id=None,
base_outputs_path=None):
experiment_config = {'run_env': get_run_env()} if self.track_env else {}
if name:
experiment_config['name'] = name
if tags:
experiment_config['tags'] = tags
if framework:
experiment_config['framework'] = framework
experiment_config['backend'] = OTHER_BACKEND
if backend:
experiment_config['backend'] = backend
if description:
experiment_config['description'] = description
if build_id:
experiment_config['build_job'] = str(build_id)
experiment_config['is_managed'] = settings.IS_MANAGED
experiment = None
if self.client:
if content:
experiment_config['content'] = self.client.project.validate_content(content=content)
experiment = self.client.project.create_experiment(
username=self.username,
project_name=self.project_name,
experiment_config=experiment_config,
group=self.group_id,
)
if not experiment:
raise PolyaxonClientException('Could not create experiment.')
if not settings.IS_MANAGED and self.track_logs:
setup_logging(send_logs=self.send_logs)
self.experiment_id = self._get_entity_id(experiment)
self.experiment = experiment
self.last_status = 'created'
# Setup the outputs store
base_outputs_path = base_outputs_path or get_base_outputs_path()
if self.outputs_store is None and base_outputs_path:
if self.group_id:
outputs_path = '{}/{}/{}/{}/{}'.format(
base_outputs_path,
self.username,
self.project_name,
self.group_id,
self.experiment_id)
else:
outputs_path = '{}/{}/{}/{}'.format(
base_outputs_path, self.username, self.project_name, self.experiment_id)
self.set_outputs_store(outputs_path=outputs_path)
if self.track_code:
self.log_code_ref()
if not settings.IS_MANAGED:
self._start()
self._set_health_url()
return self
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function
from polyaxon_client.exceptions import PolyaxonClientException
from polyaxon_client.tracking import Experiment
from polyaxon_client import settings
try:
from tensorflow.train import LoggingTensorHook
except ImportError:
raise PolyaxonClientException(
'tensorflow is required to use PolyaxonLoggingTensorHook')
class PolyaxonLoggingTensorHook(LoggingTensorHook):
"""Hook that logs data to console and Polyaxon"""
def __init__(self,
tensors,
experiment=None,
every_n_iter=None,
every_n_secs=None):
super(PolyaxonLoggingTensorHook,
self).__init__(tensors=tensors,
every_n_iter=every_n_iter,
every_n_secs=every_n_secs)
self.experiment = experiment
if settings.IS_MANAGED:
self.experiment = self.experiment or Experiment()
def _log_tensors(self, tensor_values):
super(PolyaxonLoggingTensorHook, self)._log_tensors(tensor_values)
def ensure_is_managed():
if not settings.IS_MANAGED and not settings.IN_CLUSTER:
raise PolyaxonClientException(
'This experiment/job is not managed by Polyaxon.')
def ensure_in_custer():
if not settings.IN_CLUSTER:
raise PolyaxonClientException(
'This experiment/job is not running inside a Polyaxon job.')
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function
from polyaxon_client import settings
from polyaxon_client.exceptions import PolyaxonClientException
from polyaxon_client.tracking import Experiment
try:
from keras.callbacks import Callback
except ImportError:
try:
from tensorflow.keras.callbacks import Callback
except ImportError:
raise PolyaxonClientException('Keras is required to use PolyaxonKeras')
class PolyaxonKeras(Callback):
def __init__(self, experiment=None, metrics=None):
self.experiment = experiment
if settings.IS_MANAGED:
self.experiment = self.experiment or Experiment()
self.metrics = metrics
def on_epoch_end(self, epoch, logs=None):
if not logs or not self.experiment:
return
if self.metrics:
metrics = {
metric: logs[metric]
for metric in self.metrics if metric in logs
}
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function
from polyaxon_client import settings
from polyaxon_client.exceptions import PolyaxonClientException
from polyaxon_client.tracking import Experiment
try:
from fastai.callbacks import TrackerCallback
except ImportError:
raise PolyaxonClientException('Fastai is required to use PolyaxonFastai')
class PolyaxonFastai(TrackerCallback):
def __init__(self,
learn,
experiment=None,
monitor='val_loss',
mode='auto'):
super(PolyaxonFastai, self).__init__(learn, monitor=monitor, mode=mode)
self.experiment = experiment
if settings.IS_MANAGED:
self.experiment = self.experiment or Experiment()
def on_epoch_end(self, epoch, smooth_loss, last_metrics, **kwargs):
if not self.experiment:
return
metrics = {
name: stat
for name, stat in list(
zip(self.learn.recorder.names, [epoch, smooth_loss] +
