def logs(ctx, follow, hide_time, all_info): """Get run or run job logs. Uses /docs/core/cli/#caching Examples for getting run logs: \b $ polyaxon run logs \b $ polyaxon ops -uid=8aac02e3a62a4f0aaa257c59da5eab80 -p mnist logs """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True, ) client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) get_run_logs( client=client, hide_time=hide_time, all_info=all_info, follow=follow, )
def delete(ctx): """Delete a run. Uses /docs/core/cli/#caching Example: \b $ polyaxon ops delete \b $ polyaxon ops --uid=8aac02e3a62a4f0aaa257c59da5eab80 delete # project is cached \b $ polyaxon ops --project=cats-vs-dogs -uid 8aac02e3a62a4f0aaa257c59da5eab80 delete """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True, ) if not click.confirm("Are sure you want to delete run `{}`".format(run_uuid)): click.echo("Existing without deleting the run.") sys.exit(1) try: polyaxon_client = RunClient( owner=owner, project=project_name, run_uuid=run_uuid ) polyaxon_client.delete() # Purge caching RunManager.purge() except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not delete run `{}`.".format(run_uuid)) sys.exit(1) Printer.print_success("Run `{}` was delete successfully".format(run_uuid))
def invalidate(ctx, project, uid): """Invalidate the run's cache state. Uses /docs/core/cli/#caching Examples: \b $ polyaxon ops invalidate """ owner, project_name, run_uuid = get_project_run_or_local( project or ctx.obj.get("project"), uid or ctx.obj.get("run_uuid"), is_cli=True, ) try: polyaxon_client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) response = polyaxon_client.invalidate() Printer.print_success("Run was invalidated with uid {}".format( response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not invalidate run `{}`.".format(run_uuid)) sys.exit(1)
def upload(ctx, project, uid, path_from, path_to, is_file, sync_failure): """Upload runs' artifacts. Uses /docs/core/cli/#caching Examples: \b $ polyaxon ops upload -uid=8aac02e3a62a4f0aaa257c59da5eab80 \b $ polyaxon ops upload -uid=8aac02e3a62a4f0aaa257c59da5eab80 path_from="path/to/upload" \b $ polyaxon ops upload -uid=8aac02e3a62a4f0aaa257c59da5eab80 path_to="path/to/upload/to" """ owner, project_name, run_uuid = get_project_run_or_local( project or ctx.obj.get("project"), uid or ctx.obj.get("run_uuid"), is_cli=True, ) try: client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) if is_file: response = client.upload_artifact(filepath=path_from, path=path_to, overwrite=True) else: response = client.upload_artifacts_dir( dirpath=path_from, path=path_to, overwrite=True, relative_to=path_from, ) except ( ApiException, HTTPError, PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException, ) as e: handle_cli_error( e, message="Could not upload artifacts for run `{}`".format(run_uuid)) sys.exit(1) if response.status_code == 200: Printer.print_success("Artifacts uploaded") else: if sync_failure: client.log_failed(reason="OperationCli", message="Operation failed uploading artifacts") Printer.print_error( "Error uploading artifacts. " "Status: {}. Error: {}.".format(response.status_code, response.content), sys_exit=True, )
def resume(ctx, polyaxonfile): """Resume run. Uses /docs/core/cli/#caching Examples: \b $ polyaxon ops --uid=8aac02e3a62a4f0aaa257c59da5eab80 resume """ content = None if polyaxonfile: content = "{}".format(ConfigSpec.read_from(polyaxonfile)) owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True, ) try: polyaxon_client = RunClient( owner=owner, project=project_name, run_uuid=run_uuid ) response = polyaxon_client.resume(override_config=content) Printer.print_success("Run was resumed with uid {}".format(response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not resume run `{}`.".format(run_uuid)) sys.exit(1)
def approve(ctx, project, uid): """Approve a run waiting for human validation. Uses /docs/core/cli/#caching Examples: \b $ polyaxon ops approve \b $ polyaxon ops approve --uid=8aac02e3a62a4f0aaa257c59da5eab80 """ owner, project_name, run_uuid = get_project_run_or_local( project or ctx.obj.get("project"), uid or ctx.obj.get("run_uuid"), is_cli=True, ) try: polyaxon_client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) polyaxon_client.approve() except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not approve run `{}`.".format(run_uuid)) sys.exit(1) Printer.print_success("Run is approved")
def restart(ctx, project, uid, copy, polyaxonfile): """Restart run. Uses /docs/core/cli/#caching Examples: \b $ polyaxon run --uid=8aac02e3a62a4f0aaa257c59da5eab80 restart """ content = None if polyaxonfile: content = OperationSpecification.read( polyaxonfile, is_preset=True).to_dict(dump=True) owner, project_name, run_uuid = get_project_run_or_local( project or ctx.obj.get("project"), uid or ctx.obj.get("run_uuid"), is_cli=True, ) try: polyaxon_client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) response = polyaxon_client.restart(override_config=content, copy=copy) Printer.print_success("Run was {} with uid {}".format( "copied" if copy else "restarted", response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not restart run `{}`.".format(run_uuid)) sys.exit(1)
def restart(ctx, copy, polyaxonfile, u): """Restart run. Uses /docs/core/cli/#caching Examples: \b $ polyaxon run --uid=8aac02e3a62a4f0aaa257c59da5eab80 restart """ content = None if polyaxonfile: content = "{}".format(ConfigSpec.read_from(polyaxonfile)) # Check if we need to upload if u: ctx.invoke(upload, sync=False) owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True, ) try: polyaxon_client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) response = polyaxon_client.restart(override_config=content, copy=copy) Printer.print_success("Run was {} with uid {}".format( "copied" if copy else "restarted", response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not restart run `{}`.".format(run_uuid)) sys.exit(1)
def artifacts(ctx): """Download outputs/artifacts for run. Uses /docs/core/cli/#caching Examples: \b $ polyaxon ops -uid=8aac02e3a62a4f0aaa257c59da5eab80 artifacts """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True, ) try: client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) client.download_artifacts() except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not download outputs for run `{}`.".format( run_uuid)) sys.exit(1) Printer.print_success("Files downloaded.")
def logs(ctx, follow, hide_time, all_info): """Get run or run job logs. Uses [Caching](/references/polyaxon-cli/#caching) Examples for getting run logs: \b ```bash $ polyaxon run logs ``` \b ```bash $ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 -p mnist logs ``` """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True) client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) get_run_logs( client=client, hide_time=hide_time, all_info=all_info, follow=follow, )
def stop(ctx, yes): """Stop run. Uses /docs/core/cli/#caching Examples: \b $ polyaxon ops stop \b $ polyaxon ops --uid=8aac02e3a62a4f0aaa257c59da5eab80 stop """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True, ) if not yes and not click.confirm( "Are sure you want to stop " "run `{}`".format(run_uuid) ): click.echo("Existing without stopping run.") sys.exit(0) try: polyaxon_client = RunClient( owner=owner, project=project_name, run_uuid=run_uuid ) polyaxon_client.stop() except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not stop run `{}`.".format(run_uuid)) sys.exit(1) Printer.print_success("Run is being stopped.")
def invalidate_run(ctx): """Invalidate runs' cache inside this project. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon invalidate_builds ``` """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True) try: polyaxon_client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) response = polyaxon_client.invalidate() Printer.print_success("Run was invalidated with uid {}".format( response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not invalidate run `{}`.".format(run_uuid)) sys.exit(1)
def resume(ctx, polyaxonfile, u): """Resume run. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 resume ``` """ content = None if polyaxonfile: content = "{}".format(reader.read(polyaxonfile)) # Check if we need to upload if u: ctx.invoke(upload, sync=False) owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True) try: polyaxon_client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) response = polyaxon_client.resume(override_config=content) Printer.print_success("Run was resumed with uid {}".format( response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not resume run `{}`.".format(run_uuid)) sys.exit(1)
def logs(ctx, follow, hide_time, all_info): """Get run or run job logs. Uses /docs/core/cli/#caching Examples for getting run logs: \b $ polyaxon run logs \b $ polyaxon ops -uid=8aac02e3a62a4f0aaa257c59da5eab80 -p mnist logs """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True, ) client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) try: get_run_logs( client=client, hide_time=hide_time, all_info=all_info, follow=follow, ) except (ApiException, HTTPError, PolyaxonClientException) as e: handle_cli_error( e, message="Could not get logs for run `{}`.".format(client.run_uuid), ) sys.exit(1)
def artifacts(ctx, path, path_to, no_untar): """Download outputs/artifacts for run. Uses /docs/core/cli/#caching Examples: \b $ polyaxon ops -uid=8aac02e3a62a4f0aaa257c59da5eab80 artifacts \b $ polyaxon ops -uid=8aac02e3a62a4f0aaa257c59da5eab80 artifacts path="events/only" \b $ polyaxon ops -uid=8aac02e3a62a4f0aaa257c59da5eab80 artifacts path_to="this/path" """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True, ) try: client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) download_path = client.download_artifacts( path=path or "", path_to=path_to, untar=not no_untar ) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not download outputs for run `{}`.".format(run_uuid) ) sys.exit(1) Printer.print_success("Files downloaded: path: {}".format(download_path))
def dashboard(ctx, yes, url): """Open this operation's dashboard details in browser.""" owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True, ) subpath = "{}/{}/runs/{}".format(owner, project_name, run_uuid) get_dashboard( dashboard_url=get_dashboard_url(subpath=subpath), url_only=url, yes=yes )
def statuses(ctx, watch): """Get run or run job statuses. Uses /docs/core/cli/#caching Examples getting run statuses: \b $ polyaxon ops statuses \b $ polyaxon ops -uid=8aac02e3a62a4f0aaa257c59da5eab80 statuses """ def _handle_run_statuses(): if not conditions: return Printer.print_header("Latest status:") latest_status = Printer.add_status_color( {"status": status}, status_key="status" ) click.echo("{}\n".format(latest_status["status"])) objects = list_dicts_to_tabulate( [ Printer.add_status_color(o.to_dict(), status_key="type") for o in conditions ] ) if objects: Printer.print_header("Conditions:") dict_tabulate(objects, is_list_dict=True) owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True, ) client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) if watch: try: for status, conditions in client.watch_statuses(): _handle_run_statuses() except (ApiException, HTTPError, PolyaxonClientException) as e: handle_cli_error( e, message="Could get status for run `{}`.".format(run_uuid) ) sys.exit(1) else: try: status, conditions = client.get_statuses() _handle_run_statuses() except (ApiException, HTTPError, PolyaxonClientException) as e: handle_cli_error( e, message="Could get status for run `{}`.".format(run_uuid) ) sys.exit(1)
def service(ctx, yes, external, url): """Open the operation service in browser. N.B. The operation must have a run kind service, otherwise it will raise an error. You can open the service embedded in Polyaxon UI or using the real service URL, please use the `--external` flag. """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True, ) client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) client.refresh_data() if client.run_data.kind != V1RunKind.SERVICE: Printer.print_warning("Command expected a operations of " "kind `service` received kind: {}!".format( client.run_data.kind)) sys.exit(1) dashboard_url = settings.CLIENT_CONFIG.host Printer.print_header("Waiting for running condition ...") client.wait_for_condition(statuses=[V1Statuses.RUNNING], print_status=True) client.refresh_data() namespace = client.run_data.settings.namespace run_url = "{}/{}/{}/runs/{}/service".format(dashboard_url, owner, project_name, run_uuid) service_endpoint = SERVICES_V1 if client.run_data.meta_info.get("rewrite_path", False): service_endpoint = REWRITE_SERVICES_V1 external_run_url = "{}/{}/{}/{}/{}/runs/{}/".format( dashboard_url, service_endpoint, namespace, owner, project_name, run_uuid) if url: Printer.print_header( "The service will be available at: {}".format(run_url)) Printer.print_header( "You can also view it in an external link at: {}".format( external_run_url)) sys.exit(0) if not yes: click.confirm( "Dashboard page will now open in your browser. Continue?", abort=True, default=True, ) if external: click.launch(external_run_url) sys.exit(0) click.launch(run_url)
def update(ctx, name, description, tags): """Update run. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 update --description="new description for my runs" ``` \b ```bash $ polyaxon runs --project=cats-vs-dogs -id 8aac02e3a62a4f0aaa257c59da5eab80 update --tags="foo, bar" --name="unique-name" ``` """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True) update_dict = {} if name: update_dict["name"] = name if description: update_dict["description"] = description tags = validate_tags(tags) if tags: update_dict["tags"] = tags if not update_dict: Printer.print_warning("No argument was provided to update the run.") sys.exit(0) try: polyaxon_client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) response = polyaxon_client.update(update_dict) except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not update run `{}`.".format(run_uuid)) sys.exit(1) Printer.print_success("Run updated.") get_run_details(response)
def get(ctx, project, uid): """Get run. Uses /docs/core/cli/#caching Examples for getting a run: \b $ polyaxon ops get # if run is cached \b $ polyaxon ops get --uid=8aac02e3a62a4f0aaa257c59da5eab80 # project is cached \b $ polyaxon ops get --project=cats-vs-dogs -uid 8aac02e3a62a4f0aaa257c59da5eab80 \b $ polyaxon ops get -p alain/cats-vs-dogs --uid=8aac02e3a62a4f0aaa257c59da5eab80 """ owner, project_name, run_uuid = get_project_run_or_local( project or ctx.obj.get("project"), uid or ctx.obj.get("run_uuid"), is_cli=True, ) try: polyaxon_client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) polyaxon_client.refresh_data() config = polyaxon_client.client.sanitize_for_serialization( polyaxon_client.run_data) cache.cache( config_manager=RunConfigManager, config=config, owner=owner, project=project_name, ) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not load run `{}/{}/{}` info.".format( owner, project_name, run_uuid), ) sys.exit(1) get_run_details(polyaxon_client.run_data)
def dashboard(ctx, yes, url): """Open this operation's dashboard details in browser.""" owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True, ) dashboard_url = clean_host(settings.CLIENT_CONFIG.host) run_url = "{}/{}/{}/runs/{}/".format(dashboard_url, owner, project_name, run_uuid) if url: Printer.print_header("The dashboard is available at: {}".format(run_url)) sys.exit(0) if not yes: click.confirm( "Dashboard page will now open in your browser. Continue?", abort=True, default=True, ) click.launch(run_url)
def get(ctx): """Get run. Uses [Caching](/references/polyaxon-cli/#caching) Examples for getting a run: \b ```bash $ polyaxon runs get # if run is cached ``` \b ```bash $ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 get # project is cached ``` \b ```bash $ polyaxon runs --project=cats-vs-dogs -id 8aac02e3a62a4f0aaa257c59da5eab80 get ``` \b ```bash $ polyaxon runs -p alain/cats-vs-dogs --uid=8aac02e3a62a4f0aaa257c59da5eab80 get ``` """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True) try: polyaxon_client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) polyaxon_client.refresh_data() config = polyaxon_client.client.sanitize_for_serialization( polyaxon_client.run_data) cache.cache(config_manager=RunManager, config=config) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not load run `{}` info.".format(run_uuid)) sys.exit(1) get_run_details(polyaxon_client.run_data)
def test_get_project_run_or_local(self, get_run_or_local_mock, get_project_or_local_mock): get_project_or_local_mock.return_value = ("owner", "project") get_run_or_local_mock.return_value = "uuid" assert get_project_run_or_local() == ("owner", "project", "uuid")
def service(ctx, yes, external, url): """Open the operation service in browser. N.B. The operation must have a run kind service, otherwise it will raise an error. You can open the service embedded in Polyaxon UI or using the real service URL, please use the `--external` flag. """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True, ) client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) client.refresh_data() if client.run_data.kind != V1RunKind.SERVICE: Printer.print_warning( "Command expected an operation of " "kind `service` received kind: `{}`!".format(client.run_data.kind) ) sys.exit(1) Printer.print_header("Waiting for running condition ...") client.wait_for_condition( statuses={V1Statuses.RUNNING} | LifeCycle.DONE_VALUES, print_status=True ) client.refresh_data() if LifeCycle.is_done(client.run_data.status): Printer.print_header("The operations reached a done statuses.") latest_status = Printer.add_status_color( {"status": client.run_data.status}, status_key="status" ) click.echo("{}\n".format(latest_status["status"])) run_url = get_dashboard_url( subpath="{}/{}/runs/{}/service".format(owner, project_name, run_uuid) ) namespace = client.run_data.settings.namespace service_endpoint = SERVICES_V1 if client.run_data.meta_info.get("rewrite_path", False): service_endpoint = REWRITE_SERVICES_V1 external_run_url = get_dashboard_url( base=service_endpoint, subpath="{}/{}/{}/runs/{}/".format(namespace, owner, project_name, run_uuid), ) if url: Printer.print_header("The service will be available at: {}".format(run_url)) Printer.print_header( "You can also view it in an external link at: {}".format(external_run_url) ) sys.exit(0) if not yes: click.confirm( "Dashboard page will now open in your browser. Continue?", abort=True, default=True, ) if external: click.launch(external_run_url) sys.exit(0) click.launch(run_url)
def get(ctx, project, uid, offline, offline_path): """Get run. Uses /docs/core/cli/#caching Examples for getting a run: \b $ polyaxon ops get # if run is cached \b $ polyaxon ops get --uid=8aac02e3a62a4f0aaa257c59da5eab80 # project is cached \b $ polyaxon ops get --project=cats-vs-dogs -uid 8aac02e3a62a4f0aaa257c59da5eab80 \b $ polyaxon ops get -p alain/cats-vs-dogs --uid=8aac02e3a62a4f0aaa257c59da5eab80 """ uid = uid or ctx.obj.get("run_uuid") if offline: offline_path = offline_path or container_contexts.CONTEXT_OFFLINE_ROOT offline_path = "{}/{}/run_data.json".format(offline_path, uid) if not os.path.exists(offline_path): Printer.print_error( f"Could not get offline run, the path `{offline_path}` " f"does not exist.") sys.exit(1) run_data = RunConfigManager.read_from_path(offline_path) else: owner, project_name, run_uuid = get_project_run_or_local( project or ctx.obj.get("project"), uid, is_cli=True, ) try: polyaxon_client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) polyaxon_client.refresh_data() config = polyaxon_client.client.sanitize_for_serialization( polyaxon_client.run_data) cache.cache( config_manager=RunConfigManager, config=config, owner=owner, project=project_name, ) run_data = polyaxon_client.run_data except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not load run `{}/{}/{}` info.".format( owner, project_name, run_uuid), ) sys.exit(1) get_run_details(run_data)