def unbookmark(ctx):
"""Unbookmark build job.
Uses [Caching](/polyaxon_cli/introduction#Caching)
Examples:
\b
```bash
$ polyaxon build unbookmark
```
\b
```bash
$ polyaxon build -b 2 unbookmark
```
"""
user, project_name, _build = get_build_or_local(ctx.obj.get('project'),
ctx.obj.get('build'))
try:
PolyaxonClient().build_job.unbookmark(user, project_name, _build)
except (PolyaxonHTTPError, PolyaxonShouldExitError,
PolyaxonClientException) as e:
Printer.print_error(
'Could not unbookmark build job `{}`.'.format(_build))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
Printer.print_success("Build job unbookmarked.")
def url(ctx):
"""Prints the tensorboard url for this project.
Uses [Caching](/polyaxon_cli/introduction#Caching)
Example:
\b
```bash
$ polyaxon tensorboard url
```
"""
user, project_name = get_project_or_local(ctx.obj['project'])
try:
response = PolyaxonClients().project.get_project(user, project_name)
except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
Printer.print_error('Could not get project `{}`.'.format(project_name))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
if response.has_tensorboard:
click.echo(get_tensorboard_url(user, project_name))
else:
Printer.print_warning(
'This project `{}` does not have a running tensorboard.'.format(
project_name))
click.echo(
'You can start tensorboard with this command: polyaxon tensorboard start'
)
def url(ctx):
"""Prints the notebook url for this project.
Uses [Caching](/polyaxon_cli/introduction#Caching)
Example:
\b
```bash
$ polyaxon notebook url
```
"""
user, project_name = get_project_or_local(ctx.obj.get('project'))
try:
response = PolyaxonClient().project.get_project(user, project_name)
except (PolyaxonHTTPError, PolyaxonShouldExitError,
PolyaxonClientException) as e:
Printer.print_error('Could not get project `{}`.'.format(project_name))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
if response.has_notebook:
click.echo(get_notebook_url(user, project_name))
else:
Printer.print_warning(
'This project `{}` does not have a running notebook.'.format(
project_name))
click.echo(
'You can start a notebook with this command: polyaxon notebook start --help'
)
def stop(ctx, commit, yes):
"""Stops the notebook deployment for this project if it exists.
Uses [Caching](/polyaxon_cli/introduction#Caching)
"""
user, project_name = get_project_or_local(ctx.obj.get('project'))
if not yes and not click.confirm("Are sure you want to stop notebook "
"for project `{}/{}`".format(
user, project_name)):
click.echo('Existing without stopping notebook.')
sys.exit(1)
if commit is None:
commit = True
try:
PolyaxonClient().project.stop_notebook(user, project_name, commit)
Printer.print_success('Notebook is being deleted')
except (PolyaxonHTTPError, PolyaxonShouldExitError,
PolyaxonClientException) as e:
Printer.print_error(
'Could not stop notebook project `{}`.'.format(project_name))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
def bookmark(ctx):
"""Bookmark job.
Uses [Caching](/polyaxon_cli/introduction#Caching)
Examples:
\b
```bash
$ polyaxon job bookmark
```
\b
```bash
$ polyaxon job -xp 2 bookmark
```
"""
user, project_name, _job = get_job_or_local(ctx.obj.get('project'),
ctx.obj.get('job'))
try:
PolyaxonClient().job.bookmark(user, project_name, _job)
except (PolyaxonHTTPError, PolyaxonShouldExitError,
PolyaxonClientException) as e:
Printer.print_error('Could not bookmark job `{}`.'.format(_job))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
Printer.print_success("Job is bookmarked.")
def stop(ctx, yes):
"""Stop job.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples:
\b
```bash
$ polyaxon job stop
```
\b
```bash
$ polyaxon job -xp 2 stop
```
"""
user, project_name, _job = get_job_or_local(ctx.obj.get('project'),
ctx.obj.get('job'))
if not yes and not click.confirm("Are sure you want to stop "
"job `{}`".format(_job)):
click.echo('Existing without stopping job.')
sys.exit(0)
try:
PolyaxonClient().job.stop(user, project_name, _job)
except (PolyaxonHTTPError, PolyaxonShouldExitError,
PolyaxonClientException) as e:
Printer.print_error('Could not stop job `{}`.'.format(_job))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
Printer.print_success("Job is being stopped.")
def delete(ctx):
"""Delete build job.
Uses [Caching](/references/polyaxon-cli/#caching)
Example:
\b
```bash
$ polyaxon build delete
```
\b
```bash
$ polyaxon build -b 2 delete
```
"""
user, project_name, _build = get_build_or_local(ctx.obj.get('project'), ctx.obj.get('build'))
if not click.confirm("Are sure you want to delete build job `{}`".format(_build)):
click.echo('Existing without deleting build job.')
sys.exit(1)
try:
response = PolyaxonClient().build_job.delete_build(
user, project_name, _build)
# Purge caching
BuildJobManager.purge()
except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e:
Printer.print_error('Could not delete job `{}`.'.format(_build))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
if response.status_code == 204:
Printer.print_success("Build job `{}` was deleted successfully".format(_build))
def stop(ctx, yes):
"""Stop experiment.
Uses [Caching](/polyaxon_cli/introduction#Caching)
Examples:
\b
```bash
$ polyaxon experiment stop
```
\b
```bash
$ polyaxon experiment -xp 2 stop
```
"""
user, project_name, _experiment = get_project_experiment_or_local(
ctx.obj.get('project'), ctx.obj.get('experiment'))
if not yes and not click.confirm("Are sure you want to stop "
"experiment `{}`".format(_experiment)):
click.echo('Existing without stopping experiment.')
sys.exit(0)
try:
PolyaxonClient().experiment.stop(user, project_name, _experiment)
except (PolyaxonHTTPError, PolyaxonShouldExitError,
PolyaxonClientException) as e:
Printer.print_error(
'Could not stop experiment `{}`.'.format(_experiment))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
Printer.print_success("Experiment is being stopped.")
def delete(ctx):
"""Delete experiment.
Uses [Caching](/polyaxon_cli/introduction#Caching)
Example:
\b
```bash
$ polyaxon experiment delete
```
"""
user, project_name, _experiment = get_project_experiment_or_local(
ctx.obj.get('project'), ctx.obj.get('experiment'))
if not click.confirm(
"Are sure you want to delete experiment `{}`".format(_experiment)):
click.echo('Existing without deleting experiment.')
sys.exit(1)
try:
response = PolyaxonClient().experiment.delete_experiment(
user, project_name, _experiment)
# Purge caching
ExperimentManager.purge()
except (PolyaxonHTTPError, PolyaxonShouldExitError,
PolyaxonClientException) as e:
Printer.print_error(
'Could not delete experiment `{}`.'.format(_experiment))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
if response.status_code == 204:
Printer.print_success(
"Experiment `{}` was delete successfully".format(_experiment))
def resume(ctx, file, u): # pylint:disable=redefined-builtin
"""Resume experiment.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples:
\b
```bash
$ polyaxon experiment --experiment=1 resume
```
"""
config = None
update_code = None
if file:
config = rhea.read(file)
# Check if we need to upload
if u:
ctx.invoke(upload, sync=False)
update_code = True
user, project_name, _experiment = get_project_experiment_or_local(ctx.obj.get('project'),
ctx.obj.get('experiment'))
try:
response = PolyaxonClient().experiment.resume(
user, project_name, _experiment, config=config, update_code=update_code)
Printer.print_success('Experiment was resumed with id {}'.format(response.id))
except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e:
Printer.print_error('Could not resume experiment `{}`.'.format(_experiment))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
def unbookmark(ctx):
"""Unbookmark experiment.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples:
\b
```bash
$ polyaxon experiment unbookmark
```
\b
```bash
$ polyaxon experiment -xp 2 unbookmark
```
"""
user, project_name, _experiment = get_project_experiment_or_local(ctx.obj.get('project'),
ctx.obj.get('experiment'))
try:
PolyaxonClient().experiment.unbookmark(user, project_name, _experiment)
except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e:
Printer.print_error('Could not unbookmark experiment `{}`.'.format(_experiment))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
Printer.print_success("Experiment is unbookmarked.")
def outputs(ctx):
"""Download outputs for experiment.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples:
\b
```bash
$ polyaxon experiment -xp 1 outputs
```
"""
user, project_name, _experiment = get_project_experiment_or_local(
ctx.obj.get('project'), ctx.obj.get('experiment'))
try:
PolyaxonClient().experiment.download_outputs(user, project_name,
_experiment)
except (PolyaxonHTTPError, PolyaxonShouldExitError,
PolyaxonClientException) as e:
Printer.print_error(
'Could not download outputs for experiment `{}`.'.format(
_experiment))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
Printer.print_success('Files downloaded.')
def check(file, # pylint:disable=redefined-builtin
version,
definition,
params):
"""Check a polyaxonfile."""
file = file or 'polyaxonfile.yaml'
specification = check_polyaxonfile(file, params=params).specification
if version:
Printer.decorate_format_value('The version is: {}',
specification.version,
'yellow')
if definition:
job_condition = (specification.is_job or
specification.is_build or
specification.is_notebook or
specification.is_tensorboard)
if specification.is_experiment:
Printer.decorate_format_value('This polyaxon specification has {}',
'One experiment',
'yellow')
if job_condition:
Printer.decorate_format_value('This {} polyaxon specification is valid',
specification.kind,
'yellow')
if specification.is_group:
experiments_def = specification.experiments_def
click.echo(
'This polyaxon specification has experiment group with the following definition:')
get_group_experiments_info(**experiments_def)
return specification
def invalidate(ctx):
"""Invalidate build job.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples:
\b
```bash
$ polyaxon build invalidate
```
\b
```bash
$ polyaxon build -b 2 invalidate
```
"""
user, project_name, _build = get_build_or_local(ctx.obj.get('project'), ctx.obj.get('build'))
try:
PolyaxonClient().build_job.invalidate(user, project_name, _build)
except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e:
Printer.print_error('Could not invalidate build job `{}`.'.format(_build))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
Printer.print_success("Build job has being invalidated.")
def stop(ctx, yes):
"""Stop build job.
Uses [Caching](/polyaxon_cli/introduction#Caching)
Examples:
\b
```bash
$ polyaxon build stop
```
\b
```bash
$ polyaxon build -b 2 stop
```
"""
user, project_name, _build = get_build_or_local(ctx.obj.get('project'),
ctx.obj.get('build'))
if not yes and not click.confirm("Are sure you want to stop "
"job `{}`".format(_build)):
click.echo('Existing without stopping build job.')
sys.exit(0)
try:
PolyaxonClient().build_job.stop(user, project_name, _build)
except (PolyaxonHTTPError, PolyaxonShouldExitError,
PolyaxonClientException) as e:
Printer.print_error('Could not stop build job `{}`.'.format(_build))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
Printer.print_success("Build job is being stopped.")
def get(ctx):
"""Get experiment.
Uses [Caching](/polyaxon_cli/introduction#Caching)
Example:
\b
```bash
$ polyaxon experiment get # if experiment is cached
```
\b
```bash
$ polyaxon experiment --experiment=1 get
```
\b
```bash
$ polyaxon experiment -xp 1 --project=cats-vs-dogs get
```
\b
```bash
$ polyaxon experiment -xp 1 -p alain/cats-vs-dogs get
```
"""
user, project_name, _experiment = get_experiment_or_local(ctx.obj['project'],
ctx.obj['experiment'])
try:
response = PolyaxonClients().experiment.get_experiment(user, project_name, _experiment)
# Set caching only if we have an initialized project
if ProjectManager.is_initialized():
ExperimentManager.set_config(response)
except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
Printer.print_error('Could not load experiment `{}` info.'.format(_experiment))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
get_experiment_details(response)
def __init__(self):
host = GlobalConfigManager.get_value('host')
if not host:
Printer.print_error(
'Received an invalid config, you need to provide a valid host.'
)
sys.exit(1)
http_port = GlobalConfigManager.get_value('http_port')
ws_port = GlobalConfigManager.get_value('ws_port')
use_https = GlobalConfigManager.get_value('use_https')
verify_ssl = GlobalConfigManager.get_value('verify_ssl')
token = AuthConfigManager.get_value('token')
super(PolyaxonClient, self).__init__(
host=host,
http_port=http_port,
ws_port=ws_port,
use_https=use_https,
verify_ssl=verify_ssl,
token=token,
schema_response=True,
reraise=True,
)
def update(project, name, description):
"""Update project.
Example:
```
polyaxon update foobar --description=Image Classification with Deep Learning using TensorFlow
```
```
polyaxon update mike1/foobar --description=Image Classification with Deep Learning using TensorFlow
```
"""
user, project_name = get_project_or_local(project)
update_dict = {}
if name:
update_dict['name'] = name
if description:
update_dict['description'] = description
if not update_dict:
Printer.print_warning(
'No argument was provided to update the project.')
sys.exit(1)
try:
response = PolyaxonClients().project.update_project(
user, project_name, update_dict)
except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
Printer.print_error('Could not update project `{}`.'.format(project))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
Printer.print_success("Project updated.")
response = response.to_dict()
Printer.print_header("Project info:")
dict_tabulate(response)
def get_node_info(node_config):
if not node_config:
Printer.print_error('No node was found.')
sys.exit(1)
node = node_config.to_dict()
node.pop('gpus')
Printer.print_header("Node info:")
dict_tabulate(Printer.add_memory_unit(node, 'memory'))
gpus_items = list_dicts_to_tabulate([Printer.add_memory_unit(gpu.to_light_dict(), 'memory')
for gpu in node_config.gpus])
if gpus_items:
Printer.print_header("Node GPUs:")
dict_tabulate(gpus_items, is_list_dict=True)
def update(ctx, name, description, tags):
"""Update experiment.
Uses [Caching](/polyaxon_cli/introduction#Caching)
Examples:
\b
```bash
$ polyaxon experiment -xp 2 update --description="new description for my experiments"
```
\b
```bash
$ polyaxon experiment -xp 2 update --tags="foo, bar" --name="unique-name"
```
"""
user, project_name, _experiment = get_project_experiment_or_local(
ctx.obj.get('project'), ctx.obj.get('experiment'))
update_dict = {}
if name:
update_dict['name'] = name
if description:
update_dict['description'] = description
tags = validate_tags(tags)
if tags:
update_dict['tags'] = tags
if not update_dict:
Printer.print_warning(
'No argument was provided to update the experiment.')
sys.exit(0)
try:
response = PolyaxonClient().experiment.update_experiment(
user, project_name, _experiment, update_dict)
except (PolyaxonHTTPError, PolyaxonShouldExitError,
PolyaxonClientException) as e:
Printer.print_error(
'Could not update experiment `{}`.'.format(_experiment))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
Printer.print_success("Experiment updated.")
get_experiment_details(response)
def update(ctx, name, description, tags):
"""Update experiment group.
Uses [Caching](/references/polyaxon-cli/#caching)
Example:
\b
```bash
$ polyaxon group -g 2 update --description="new description for this group"
```
\b
```bash
$ polyaxon update --tags="foo, bar"
```
"""
user, project_name, _group = get_project_group_or_local(
ctx.obj.get('project'), ctx.obj.get('group'))
update_dict = {}
if name:
update_dict['name'] = name
if description:
update_dict['description'] = description
tags = validate_tags(tags)
if tags:
update_dict['tags'] = tags
if not update_dict:
Printer.print_warning(
'No argument was provided to update the experiment group.')
sys.exit(0)
try:
response = PolyaxonClient().experiment_group.update_experiment_group(
user, project_name, _group, update_dict)
except (PolyaxonHTTPError, PolyaxonShouldExitError,
PolyaxonClientException) as e:
Printer.print_error(
'Could not update experiment group `{}`.'.format(_group))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
Printer.print_success("Experiment group updated.")
get_group_details(response)
def create_polyaxonfile():
if os.path.isfile(constants.INIT_FILE):
try:
_ = PolyaxonFile(constants.INIT_FILE).specification # noqa
Printer.print_success(
"A valid polyaxonfile.yaml was found in the project.")
except Exception as e:
Printer.print_error(
"A Polyaxonfile was found but it is not valid.")
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
else:
create_init_file(constants.INIT_FILE_RUN)
# if we are here the file was not created
if not os.path.isfile(constants.INIT_FILE):
Printer.print_error(
"Something went wrong, init command did not create a file.\n"
"Possible reasons: you don't have enough rights to create the file."
)
sys.exit(1)
Printer.print_success("{} was created successfully.".format(
constants.INIT_FILE))
def get_job_info(log_line, job_regex):
log_search = job_regex.search(log_line)
if not log_search:
return log_line, False
job_info = log_search.group()
if job_info in job_to_color:
color = job_to_color[job_info]
else:
color = colors[0]
colors.rotate(-1)
job_to_color[job_info] = color
return re.sub(job_regex, Printer.add_color(job_info, color),
log_line), True
def restart(ctx, copy, file, u): # pylint:disable=redefined-builtin
"""Restart experiment.
Uses [Caching](/polyaxon_cli/introduction#Caching)
Examples:
\b
```bash
$ polyaxon experiment --experiment=1 restart
```
"""
config = None
update_code = None
if file:
config = reader.read(file)
# Check if we need to upload
if u:
ctx.invoke(upload, async=False)
update_code = True
user, project_name, _experiment = get_experiment_or_local(ctx.obj['project'],
ctx.obj['experiment'])
try:
if copy:
response = PolyaxonClients().experiment.copy(
user, project_name, _experiment, config=config, update_code=update_code)
else:
response = PolyaxonClients().experiment.restart(
user, project_name, _experiment, config=config, update_code=update_code)
except (PolyaxonHTTPError, PolyaxonShouldExitError) as e:
Printer.print_error('Could not restart experiment `{}`.'.format(_experiment))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
get_experiment_details(response)
def resume(ctx, file, u): # pylint:disable=redefined-builtin
"""Resume job.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples:
\b
```bash
$ polyaxon job --job=1 resume
```
"""
content = None
update_code = None
if file:
content = '{}'.format(rhea.read(file))
# Check if we need to upload
if u:
ctx.invoke(upload, sync=False)
update_code = True
user, project_name, _job = get_job_or_local(ctx.obj.get('project'),
ctx.obj.get('job'))
try:
response = PolyaxonClient().job.resume(user,
project_name,
_job,
content=content,
update_code=update_code)
except (PolyaxonHTTPError, PolyaxonShouldExitError,
PolyaxonClientException) as e:
Printer.print_error('Could not resume job `{}`.'.format(_job))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
get_job_details(response)
def code(ctx):
"""Download code for experiment.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples:
\b
```bash
$ polyaxon experiment -xp 1 code
```
"""
user, project_name, _experiment = get_project_experiment_or_local(
ctx.obj.get('project'), ctx.obj.get('experiment'))
try:
code_ref = PolyaxonClient().experiment.get_code_reference(
user, project_name, _experiment)
commit = None
if code_ref:
commit = code_ref.commit
Printer.print_header(
'Experiment has code ref: `{}`, downloading ...'.format(
commit))
else:
Printer.print_warning(
'Experiment has no code ref, downloading latest code...')
PolyaxonClient().project.download_repo(user,
project_name,
commit=commit)
except (PolyaxonHTTPError, PolyaxonShouldExitError,
PolyaxonClientException) as e:
Printer.print_error(
'Could not download outputs for experiment `{}`.'.format(
_experiment))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
Printer.print_success('Files downloaded.')
def logs(ctx, past, follow, hide_time):
"""Get build logs.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples:
\b
```bash
$ polyaxon build -b 2 logs
```
\b
```bash
$ polyaxon build logs
```
"""
user, project_name, _build = get_build_or_local(ctx.obj.get('project'), ctx.obj.get('build'))
if past:
try:
response = PolyaxonClient().build_job.logs(
user, project_name, _build, stream=False)
get_logs_handler(handle_job_info=False,
show_timestamp=not hide_time,
stream=False)(response.content.decode().split('\n'))
print()
if not follow:
return
except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e:
if not follow:
Printer.print_error('Could not get logs for job `{}`.'.format(_build))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
try:
PolyaxonClient().build_job.logs(
user,
project_name,
_build,
message_handler=get_logs_handler(handle_job_info=False, show_timestamp=not hide_time))
except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e:
Printer.print_error('Could not get logs for build job `{}`.'.format(_build))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
def get_project_or_local(project=None):
if not project and not ProjectManager.is_initialized():
Printer.print_error(
'Please provide a valid project, or init a new project.'
'{}'.format(constants.INIT_COMMAND))
sys.exit(1)
if project:
parts = project.split('/')
if len(parts) == 2:
user, project_name = parts
else:
user = AuthConfigManager.get_value('username')
project_name = project
else:
project = ProjectManager.get_config()
user, project_name = project.user, project.name
if not all([user, project_name]):
Printer.print_error(
'Please provide a valid project, or init a new project.'
'{}'.format(constants.INIT_COMMAND))
sys.exit(1)
return user, project_name
def download(ctx, commit):
"""Download code of the current project."""
user, project_name = get_project_or_local(ctx.obj.get('project'))
try:
PolyaxonClient().project.download_repo(user, project_name, commit=commit)
except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e:
Printer.print_error('Could not download code for project `{}`.'.format(project_name))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
Printer.print_success('Files downloaded.')
def disable_ci():
try:
PolyaxonClient().project.disable_ci(user, project_name)
except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e:
Printer.print_error('Could not disable CI on project `{}`.'.format(project_name))
Printer.print_error('Error message `{}`.'.format(e))
sys.exit(1)
Printer.print_success(
'Polyaxon CI was successfully disabled on project: `{}`.'.format(project_name))
