def resume(ctx, polyaxonfile, u): """Resume run. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 resume ``` """ content = None if polyaxonfile: content = "{}".format(rhea.read(polyaxonfile)) # Check if we need to upload if u: ctx.invoke(upload, sync=False) owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) try: polyaxon_client = PolyaxonClient() body = V1Run(content=content) response = polyaxon_client.runs_v1.resume_run(owner, project_name, run_uuid, body) Printer.print_success("Run was resumed with uid {}".format( response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not resume run `{}`.".format(run_uuid)) sys.exit(1)
def unbookmark(ctx): """Unbookmark run. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon runs unbookmark ``` \b ```bash $ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 unbookmark ``` """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) try: polyaxon_client = PolyaxonClient() polyaxon_client.runs_v1.unbookmark_run(owner, project_name, run_uuid) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not unbookmark run `{}`.".format(run_uuid)) sys.exit(1) Printer.print_success("Run is unbookmarked.")
def invalidate_run(ctx): """Invalidate runs' cache inside this project. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon invalidate_builds ``` """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) try: polyaxon_client = PolyaxonClient() response = polyaxon_client.runs_v1.invalidate_run( owner, project_name, run_uuid) Printer.print_success("Run was invalidated with uid {}".format( response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not invalidate run `{}`.".format(run_uuid)) sys.exit(1)
def stop(ctx, yes): """Stop run. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon runs stop ``` \b ```bash $ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 stop ``` """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) if not yes and not click.confirm("Are sure you want to stop " "run `{}`".format(run_uuid)): click.echo("Existing without stopping run.") sys.exit(0) try: polyaxon_client = PolyaxonClient() polyaxon_client.runs_v1.stop_run(owner, project_name, run_uuid) except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not stop run `{}`.".format(run_uuid)) sys.exit(1) Printer.print_success("Run is being stopped.")
def logs(ctx, past, follow, hide_time): """Get run or run job logs. Uses [Caching](/references/polyaxon-cli/#caching) Examples for getting run logs: \b ```bash $ polyaxon run logs ``` \b ```bash $ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 -p mnist logs ``` """ def get_run_logs(): if past: try: response = PolyaxonClient().run.logs(owner, project_name, run_uuid, stream=False) get_logs_handler(handle_job_info=True, show_timestamp=not hide_time, stream=False)( response.content.decode().split("\n")) print() if not follow: return except (ApiException, HTTPError) as e: if not follow: handle_cli_error( e, message="Could not get logs for run `{}`.".format( run_uuid)) sys.exit(1) try: PolyaxonClient().run.logs( owner, project_name, run_uuid, message_handler=get_logs_handler(handle_job_info=True, show_timestamp=not hide_time), ) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not get logs for run `{}`.".format(run_uuid)) sys.exit(1) owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) get_run_logs()
def update(ctx, name, description, tags): """Update run. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 update --description="new description for my runs" ``` \b ```bash $ polyaxon runs --project=cats-vs-dogs -id 8aac02e3a62a4f0aaa257c59da5eab80 update --tags="foo, bar" --name="unique-name" ``` """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) update_dict = {} if name: update_dict["name"] = name if description: update_dict["description"] = description tags = validate_tags(tags) if tags: update_dict["tags"] = tags if not update_dict: Printer.print_warning("No argument was provided to update the run.") sys.exit(0) try: polyaxon_client = PolyaxonClient() response = polyaxon_client.runs_v1.patch_run(owner, project_name, run_uuid, update_dict) except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not update run `{}`.".format(run_uuid)) sys.exit(1) Printer.print_success("Run updated.") get_run_details(response)
def statuses(ctx, watch): """Get run or run job statuses. Uses [Caching](/references/polyaxon-cli/#caching) Examples getting run statuses: \b ```bash $ polyaxon runs statuses ``` \b ```bash $ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 statuses ``` """ def _get_run_statuses(): try: for status, conditions in get_run_statuses(owner, project_name, run_uuid, watch): if not conditions: continue Printer.print_header("Latest status:") latest_status = Printer.add_status_color({"status": status}, status_key="status") click.echo("{}\n".format(latest_status["status"])) objects = list_dicts_to_tabulate([ Printer.add_status_color(o.to_dict(), status_key="type") for o in conditions ]) if objects: Printer.print_header("Conditions:") dict_tabulate(objects, is_list_dict=True) except (ApiException, HTTPError, PolyaxonClientException) as e: handle_cli_error( e, message="Could get status for run `{}`.".format(run_uuid)) sys.exit(1) owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) _get_run_statuses()
def get(ctx): """Get run. Uses [Caching](/references/polyaxon-cli/#caching) Examples for getting a run: \b ```bash $ polyaxon runs get # if run is cached ``` \b ```bash $ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 get # project is cached ``` \b ```bash $ polyaxon runs --project=cats-vs-dogs -id 8aac02e3a62a4f0aaa257c59da5eab80 get ``` \b ```bash $ polyaxon runs -p alain/cats-vs-dogs --uid=8aac02e3a62a4f0aaa257c59da5eab80 get ``` """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) try: polyaxon_client = PolyaxonClient() response = polyaxon_client.runs_v1.get_run(owner, project_name, run_uuid) config = polyaxon_client.api_client.sanitize_for_serialization( response) cache.cache(config_manager=RunManager, response=config) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not load run `{}` info.".format(run_uuid)) sys.exit(1) get_run_details(response)
def delete(ctx): """Delete a run. Uses [Caching](/references/polyaxon-cli/#caching) Example: \b ```bash $ polyaxon runs delete ``` \b ```bash $ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 delete # project is cached ``` \b ```bash $ polyaxon runs --project=cats-vs-dogs -id 8aac02e3a62a4f0aaa257c59da5eab80 delete ``` """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) if not click.confirm( "Are sure you want to delete run `{}`".format(run_uuid)): click.echo("Existing without deleting the run.") sys.exit(1) try: polyaxon_client = PolyaxonClient() polyaxon_client.runs_v1.delete_run(owner, project_name, run_uuid) # Purge caching RunManager.purge() except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not delete run `{}`.".format(run_uuid)) sys.exit(1) Printer.print_success("Run `{}` was delete successfully".format(run_uuid))
def outputs(ctx): """Download outputs for run. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 outputs ``` """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) try: PolyaxonClient().run.download_outputs(owner, project_name, run_uuid) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not download outputs for run `{}`.".format( run_uuid)) sys.exit(1) Printer.print_success("Files downloaded.")
def resources(ctx, gpu): """Get run or run job resources. Uses [Caching](/references/polyaxon-cli/#caching) Examples for getting run resources: \b ```bash $ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 resources ``` For GPU resources \b ```bash $ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 resources --gpu ``` """ def get_run_resources(): try: message_handler = Printer.gpu_resources if gpu else Printer.resources PolyaxonClient().run.resources(owner, project_name, run_uuid, message_handler=message_handler) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not get resources for run `{}`.".format( run_uuid)) sys.exit(1) owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) get_run_resources()
def code(ctx): """Download code for run. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 code ``` """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) try: polyaxon_client = PolyaxonClient() code_ref = polyaxon_client.runs_v1.get_run_code_refs( owner, project_name, run_uuid) commit = None if code_ref: commit = code_ref.commit Printer.print_header( "Run has code ref: `{}`, downloading ...".format(commit)) else: Printer.print_warning( "Run has no code ref, downloading latest code...") PolyaxonClient().project.download_repo(owner, project_name, commit=commit) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not download outputs for run `{}`.".format( run_uuid)) sys.exit(1) Printer.print_success("Files downloaded.")
def logs(ctx, past, follow, hide_time): """Get run or run job logs. Uses [Caching](/references/polyaxon-cli/#caching) Examples for getting run logs: \b ```bash $ polyaxon run logs ``` \b ```bash $ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 -p mnist logs ``` """ def show_run_logs(): is_done = False last_logs_filepath = None while not is_done: response_tree = get_run_tree() for filepath in response_tree.files: if filepath == last_logs_filepath: continue last_logs_filepath = filepath get_run_logs(filepath) is_done = response_tree.is_done if not is_done: time.sleep(5) def get_run_logs(path): try: response = polyaxon_client.runs_v1.get_run_logs_file(owner, project_name, run_uuid, path=path) get_logs_handler(handle_job_info=True, show_timestamp=not hide_time, stream=False)(response.split("\n")) print() except (ApiException, HTTPError) as e: if not follow: handle_cli_error( e, message="Could not get logs for run `{}`.".format( run_uuid)) sys.exit(1) def get_run_tree(): try: return polyaxon_client.runs_v1.get_run_logs_tree( owner, project_name, run_uuid) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not get logs for run `{}`.".format(run_uuid)) sys.exit(1) owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) polyaxon_client = PolyaxonClient() show_run_logs()