def get_experiment_logs(): if past: try: response = PolyaxonClient().experiment.logs( user, project_name, _experiment, stream=False) get_logs_handler(handle_job_info=True, show_timestamp=not hide_time, stream=False)(response.content.decode().split('\n')) print() if not follow: return except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: if not follow: Printer.print_error( 'Could not get logs for experiment `{}`.'.format(_experiment)) Printer.print_error( 'Error message `{}`.'.format(e)) sys.exit(1) try: PolyaxonClient().experiment.logs( user, project_name, _experiment, message_handler=get_logs_handler(handle_job_info=True, show_timestamp=not hide_time)) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not get logs for experiment `{}`.'.format(_experiment)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1)
def code(ctx): """Download code for job. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon job -j 1 code ``` """ user, project_name, _job = get_job_or_local(ctx.obj.get('project'), ctx.obj.get('job')) try: code_ref = PolyaxonClient().job.get_code_reference( user, project_name, _job) commit = None if code_ref: commit = code_ref.commit Printer.print_header( 'Job has code ref: `{}`, downloading ...'.format(commit)) else: Printer.print_warning( 'Job has no code ref, downloading latest code...') PolyaxonClient().project.download_repo(user, project_name, commit=commit) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not download outputs for experiment `{}`.'.format(_job)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success('Files downloaded.')
def get_tensorboard_url(user, project_name, experiment=None, group=None): if experiment: return "{}/tensorboard/{}/{}/experiments/{}/\n".format( PolyaxonClient().api_config.http_host, user, project_name, experiment) if group: return "{}/tensorboard/{}/{}/groups/{}/\n".format( PolyaxonClient().api_config.http_host, user, project_name, group) return "{}/tensorboard/{}/{}/\n".format( PolyaxonClient().api_config.http_host, user, project_name)
def logs(ctx, past, follow, hide_time): """Get build logs. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon build -b 2 logs ``` \b ```bash $ polyaxon build logs ``` """ user, project_name, _build = get_build_or_local(ctx.obj.get('project'), ctx.obj.get('build')) if past: try: response = PolyaxonClient().build_job.logs(user, project_name, _build, stream=False) get_logs_handler(handle_job_info=False, show_timestamp=not hide_time, stream=False)( response.content.decode().split('\n')) print() if not follow: return except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: if not follow: Printer.print_error( 'Could not get logs for job `{}`.'.format(_build)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) try: PolyaxonClient().build_job.logs(user, project_name, _build, message_handler=get_logs_handler( handle_job_info=False, show_timestamp=not hide_time)) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not get logs for build job `{}`.'.format(_build)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1)
def restart(ctx, copy, file, u): # pylint:disable=redefined-builtin """Restart experiment. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon experiment --experiment=1 restart ``` """ content = None update_code = None if file: content = '{}'.format(rhea.read(file)) # Check if we need to upload if u: ctx.invoke(upload, sync=False) update_code = True user, project_name, _experiment = get_project_experiment_or_local( ctx.obj.get('project'), ctx.obj.get('experiment')) try: if copy: response = PolyaxonClient().experiment.copy( user, project_name, _experiment, content=content, update_code=update_code) Printer.print_success('Experiment was copied with id {}'.format( response.id)) else: response = PolyaxonClient().experiment.restart( user, project_name, _experiment, content=content, update_code=update_code) Printer.print_success('Experiment was restarted with id {}'.format( response.id)) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not restart experiment `{}`.'.format(_experiment)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1)
def unbookmark(ctx): """Unbookmark group. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon group unbookmark ``` \b ```bash $ polyaxon group -g 2 unbookmark ``` """ user, project_name, _group = get_project_group_or_local( ctx.obj.get('project'), ctx.obj.get('group')) try: PolyaxonClient().experiment_group.unbookmark(user, project_name, _group) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not unbookmark group `{}`.'.format(_group)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success("Experiments group is unbookmarked.")
def get(ctx): """Get experiment group by uuid. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon group -g 13 get ``` """ user, project_name, _group = get_project_group_or_local( ctx.obj.get('project'), ctx.obj.get('group')) try: response = PolyaxonClient().experiment_group.get_experiment_group( user, project_name, _group) cache.cache(config_manager=GroupManager, response=response) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not get experiment group `{}`.'.format(_group)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) get_group_details(response)
def outputs(ctx): """Download outputs for experiment. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon experiment -xp 1 outputs ``` """ user, project_name, _experiment = get_project_experiment_or_local( ctx.obj.get('project'), ctx.obj.get('experiment')) try: PolyaxonClient().experiment.download_outputs(user, project_name, _experiment) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not download outputs for experiment `{}`.'.format( _experiment)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success('Files downloaded.')
def bookmark(ctx): """Bookmark experiment. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon experiment bookmark ``` \b ```bash $ polyaxon experiment -xp 2 bookmark ``` """ user, project_name, _experiment = get_project_experiment_or_local( ctx.obj.get('project'), ctx.obj.get('experiment')) try: PolyaxonClient().experiment.bookmark(user, project_name, _experiment) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not bookmark experiment `{}`.'.format(_experiment)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success("Experiment is bookmarked.")
def get(ctx): """Get job. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon job --job=1 get ``` \b ```bash $ polyaxon job --job=1 --project=project_name get ``` """ user, project_name, _job = get_job_or_local(ctx.obj.get('project'), ctx.obj.get('job')) try: response = PolyaxonClient().job.get_job(user, project_name, _job) cache.cache(config_manager=JobManager, response=response) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not get job `{}`.'.format(_job)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) get_job_details(response)
def upload(sync=True): # pylint:disable=assign-to-new-keyword """Upload code of the current directory while respecting the .polyaxonignore file.""" project = ProjectManager.get_config_or_raise() files = IgnoreManager.get_unignored_file_paths() try: with create_tarfile(files, project.name) as file_path: with get_files_in_current_directory('repo', [file_path]) as (files, files_size): try: PolyaxonClient().project.upload_repo(project.user, project.name, files, files_size, sync=sync) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not upload code for project `{}`.'.format(project.name)) Printer.print_error('Error message `{}`.'.format(e)) Printer.print_error( 'Check the project exists, ' 'and that you have access rights, ' 'this could happen as well when uploading large files. ' 'Please also make sure that you have enough space to upload the data.') sys.exit(1) Printer.print_success('Files uploaded.') except Exception as e: Printer.print_error("Could not upload the file.") Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1)
def url(ctx): """Prints the notebook url for this project. Uses [Caching](/polyaxon_cli/introduction#Caching) Example: \b ```bash $ polyaxon notebook url ``` """ user, project_name = get_project_or_local(ctx.obj.get('project')) try: response = PolyaxonClient().project.get_project(user, project_name) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not get project `{}`.'.format(project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) if response.has_notebook: click.echo(get_notebook_url(user, project_name)) else: Printer.print_warning( 'This project `{}` does not have a running notebook.'.format( project_name)) click.echo( 'You can start a notebook with this command: polyaxon notebook start --help' )
def resources(ctx, gpu): """Get job resources. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon job -j 2 resources ``` For GPU resources \b ```bash $ polyaxon job -j 2 resources --gpu ``` """ user, project_name, _job = get_job_or_local(ctx.obj.get('project'), ctx.obj.get('job')) try: message_handler = Printer.gpu_resources if gpu else Printer.resources PolyaxonClient().job.resources(user, project_name, _job, message_handler=message_handler) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not get resources for job `{}`.'.format(_job)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1)
def bookmark(ctx): """Bookmark build job. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon build bookmark ``` \b ```bash $ polyaxon build -b 2 bookmark ``` """ user, project_name, _build = get_build_or_local(ctx.obj.get('project'), ctx.obj.get('build')) try: PolyaxonClient().build_job.bookmark(user, project_name, _build) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not bookmark build job `{}`.'.format(_build)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success("Build job bookmarked.")
def stop(ctx, yes): """Stop build job. Uses [Caching](/polyaxon_cli/introduction#Caching) Examples: \b ```bash $ polyaxon build stop ``` \b ```bash $ polyaxon build -b 2 stop ``` """ user, project_name, _build = get_build_or_local(ctx.obj.get('project'), ctx.obj.get('build')) if not yes and not click.confirm("Are sure you want to stop " "job `{}`".format(_build)): click.echo('Existing without stopping build job.') sys.exit(0) try: PolyaxonClient().build_job.stop(user, project_name, _build) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not stop build job `{}`.'.format(_build)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success("Build job is being stopped.")
def stop(ctx, yes): """Stop experiment. Uses [Caching](/polyaxon_cli/introduction#Caching) Examples: \b ```bash $ polyaxon experiment stop ``` \b ```bash $ polyaxon experiment -xp 2 stop ``` """ user, project_name, _experiment = get_project_experiment_or_local( ctx.obj.get('project'), ctx.obj.get('experiment')) if not yes and not click.confirm("Are sure you want to stop " "experiment `{}`".format(_experiment)): click.echo('Existing without stopping experiment.') sys.exit(0) try: PolyaxonClient().experiment.stop(user, project_name, _experiment) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not stop experiment `{}`.'.format(_experiment)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success("Experiment is being stopped.")
def delete(ctx): """Delete experiment. Uses [Caching](/polyaxon_cli/introduction#Caching) Example: \b ```bash $ polyaxon experiment delete ``` """ user, project_name, _experiment = get_project_experiment_or_local( ctx.obj.get('project'), ctx.obj.get('experiment')) if not click.confirm( "Are sure you want to delete experiment `{}`".format(_experiment)): click.echo('Existing without deleting experiment.') sys.exit(1) try: response = PolyaxonClient().experiment.delete_experiment( user, project_name, _experiment) # Purge caching ExperimentManager.purge() except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not delete experiment `{}`.'.format(_experiment)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) if response.status_code == 204: Printer.print_success( "Experiment `{}` was delete successfully".format(_experiment))
def resources(ctx, gpu): """Get build job resources. Uses [Caching](/polyaxon_cli/introduction#Caching) Examples: \b ```bash $ polyaxon build -b 2 resources ``` For GPU resources \b ```bash $ polyaxon build -b 2 resources --gpu ``` """ user, project_name, _build = get_build_or_local(ctx.obj.get('project'), ctx.obj.get('build')) try: message_handler = Printer.gpu_resources if gpu else Printer.resources PolyaxonClient().build_job.resources(user, project_name, _build, message_handler=message_handler) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not get resources for build job `{}`.'.format(_build)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1)
def delete(ctx): """Delete experiment group. Uses [Caching](/references/polyaxon-cli/#caching) """ user, project_name, _group = get_project_group_or_local( ctx.obj.get('project'), ctx.obj.get('group')) if not click.confirm("Are sure you want to delete experiment group `{}`". format(_group)): click.echo('Existing without deleting experiment group.') sys.exit(0) try: response = PolyaxonClient().experiment_group.delete_experiment_group( user, project_name, _group) # Purge caching GroupManager.purge() except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not delete experiment group `{}`.'.format(_group)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) if response.status_code == 204: Printer.print_success( "Experiment group `{}` was delete successfully".format(_group))
def version(cli, platform): """Print the current version of the cli and platform.""" version_client = PolyaxonClient().version cli = cli or not any([cli, platform]) if cli: try: server_version = version_client.get_cli_version() except AuthorizationError: session_expired() sys.exit(1) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not get cli version.') Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) cli_version = get_current_version() Printer.print_header('Current cli version: {}.'.format(cli_version)) Printer.print_header('Supported cli versions:') dict_tabulate(server_version.to_dict()) if platform: try: platform_version = version_client.get_platform_version() except AuthorizationError: session_expired() sys.exit(1) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not get platform version.') Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) chart_version = version_client.get_chart_version() Printer.print_header('Current platform version: {}.'.format(chart_version.version)) Printer.print_header('Supported platform versions:') dict_tabulate(platform_version.to_dict())
def stop(ctx, yes): """Stop job. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon job stop ``` \b ```bash $ polyaxon job -xp 2 stop ``` """ user, project_name, _job = get_job_or_local(ctx.obj.get('project'), ctx.obj.get('job')) if not yes and not click.confirm("Are sure you want to stop " "job `{}`".format(_job)): click.echo('Existing without stopping job.') sys.exit(0) try: PolyaxonClient().job.stop(user, project_name, _job) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not stop job `{}`.'.format(_job)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success("Job is being stopped.")
def get(ctx): """Get info for current project, by project_name, or user/project_name. Uses [Caching](/references/polyaxon-cli/#caching) Examples: To get current project: \b ```bash $ polyaxon project get ``` To get a project by name \b ```bash $ polyaxon project get user/project ``` """ user, project_name = get_project_or_local(ctx.obj.get('project')) try: response = PolyaxonClient().project.get_project(user, project_name) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not get project `{}`.'.format(project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) get_project_details(response)
def unbookmark(ctx): """Unbookmark job. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon job unbookmark ``` \b ```bash $ polyaxon job -xp 2 unbookmark ``` """ user, project_name, _job = get_job_or_local(ctx.obj.get('project'), ctx.obj.get('job')) try: PolyaxonClient().job.unbookmark(user, project_name, _job) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not unbookmark job `{}`.'.format(_job)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success("Job is unbookmarked.")
def stop(ctx, commit, yes): """Stops the notebook deployment for this project if it exists. Uses [Caching](/polyaxon_cli/introduction#Caching) """ user, project_name = get_project_or_local(ctx.obj.get('project')) if not yes and not click.confirm("Are sure you want to stop notebook " "for project `{}/{}`".format( user, project_name)): click.echo('Existing without stopping notebook.') sys.exit(1) if commit is None: commit = True try: PolyaxonClient().project.stop_notebook(user, project_name, commit) Printer.print_success('Notebook is being deleted') except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not stop notebook project `{}`.'.format(project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1)
def delete(ctx): """Delete job. Uses [Caching](/references/polyaxon-cli/#caching) Example: \b ```bash $ polyaxon job delete ``` """ user, project_name, _job = get_job_or_local(ctx.obj.get('project'), ctx.obj.get('job')) if not click.confirm("Are sure you want to delete job `{}`".format(_job)): click.echo('Existing without deleting job.') sys.exit(1) try: response = PolyaxonClient().job.delete_job(user, project_name, _job) # Purge caching JobManager.purge() except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not delete job `{}`.'.format(_job)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) if response.status_code == 204: Printer.print_success("Job `{}` was delete successfully".format(_job))
def get(ctx): """Get build job. Uses [Caching](/polyaxon_cli/introduction#Caching) Examples: \b ```bash $ polyaxon build -b 1 get ``` \b ```bash $ polyaxon build --build=1 --project=project_name get ``` """ user, project_name, _build = get_build_or_local(ctx.obj.get('project'), ctx.obj.get('build')) try: response = PolyaxonClient().build_job.get_build( user, project_name, _build) cache.cache(config_manager=BuildJobManager, response=response) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not get build job `{}`.'.format(_build)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) get_build_details(response)
def upload(): # pylint:disable=assign-to-new-keyword """Upload code of the current directory while respecting the .polyaxonignore file.""" project = ProjectManager.get_config_or_raise() files = IgnoreManager.get_unignored_file_paths() with create_tarfile(files, project.name) as file_path: with get_files_in_current_directory('repo', [file_path]) as (files, files_size): try: PolyaxonClient().project.upload_repo(project.user, project.name, files, files_size) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not upload code for project `{}`.'.format( project.name)) Printer.print_error('Error message `{}`.'.format(e)) Printer.print_error( 'Check if you have access rights for this project and ' 'that you are not uploading large files.' 'If you are running a notebook, ' 'you should stop it before uploading.') sys.exit(1) Printer.print_success('Files uploaded.')
def get_experiment_job_statuses(): try: response = PolyaxonClient().experiment_job.get_statuses( user, project_name, _experiment, _job, page=page) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error( 'Could not get status for job `{}`.'.format(job)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) meta = get_meta_response(response) if meta: Printer.print_header('Statuses for Job `{}`.'.format(_job)) Printer.print_header('Navigation:') dict_tabulate(meta) else: Printer.print_header( 'No statuses found for job `{}`.'.format(_job)) objects = list_dicts_to_tabulate([ Printer.add_status_color(o.to_light_dict(humanize_values=True), status_key='status') for o in response['results'] ]) if objects: Printer.print_header("Statuses:") objects.pop('job', None) dict_tabulate(objects, is_list_dict=True)
def delete(ctx): """Delete project. Uses [Caching](/references/polyaxon-cli/#caching) """ user, project_name = get_project_or_local(ctx.obj.get('project')) if not click.confirm("Are sure you want to delete project `{}/{}`".format( user, project_name)): click.echo('Existing without deleting project.') sys.exit(1) try: response = PolyaxonClient().project.delete_project(user, project_name) local_project = ProjectManager.get_config() if local_project and (user, project_name) == (local_project.user, local_project.name): # Purge caching ProjectManager.purge() except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not delete project `{}/{}`.'.format( user, project_name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) if response.status_code == 204: Printer.print_success("Project `{}/{}` was delete successfully".format( user, project_name))
def create(ctx, name, description, tags, private, init): """Create a new project. Uses [Caching](/references/polyaxon-cli/#caching) Example: \b ```bash $ polyaxon project create --name=cats-vs-dogs --description="Image Classification with DL" ``` """ try: tags = tags.split(',') if tags else None project_dict = dict(name=name, description=description, is_public=not private, tags=tags) project_config = ProjectConfig.from_dict(project_dict) except ValidationError: Printer.print_error('Project name should contain only alpha numerical, "-", and "_".') sys.exit(1) try: _project = PolyaxonClient().project.create_project(project_config) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: Printer.print_error('Could not create project `{}`.'.format(name)) Printer.print_error('Error message `{}`.'.format(e)) sys.exit(1) Printer.print_success("Project `{}` was created successfully.".format(_project.name)) if init: ctx.obj = {} ctx.invoke(init_project, project=name)