def resume(ctx, polyaxonfile, u):
"""Resume run.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples:
\b
```bash
$ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 resume
```
"""
content = None
if polyaxonfile:
content = "{}".format(rhea.read(polyaxonfile))
# Check if we need to upload
if u:
ctx.invoke(upload, sync=False)
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"), ctx.obj.get("run_uuid"))
try:
polyaxon_client = PolyaxonClient()
body = V1Run(content=content)
response = polyaxon_client.runs_v1.resume_run(owner, project_name,
run_uuid, body)
Printer.print_success("Run was resumed with uid {}".format(
response.uuid))
except (ApiException, HTTPError) as e:
handle_cli_error(e,
message="Could not resume run `{}`.".format(run_uuid))
sys.exit(1)
def unbookmark(ctx):
"""Unbookmark run.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples:
\b
```bash
$ polyaxon runs unbookmark
```
\b
```bash
$ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 unbookmark
```
"""
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"), ctx.obj.get("run_uuid"))
try:
polyaxon_client = PolyaxonClient()
polyaxon_client.runs_v1.unbookmark_run(owner, project_name, run_uuid)
except (ApiException, HTTPError) as e:
handle_cli_error(
e, message="Could not unbookmark run `{}`.".format(run_uuid))
sys.exit(1)
Printer.print_success("Run is unbookmarked.")
def invalidate_run(ctx):
"""Invalidate runs' cache inside this project.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples:
\b
```bash
$ polyaxon invalidate_builds
```
"""
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"), ctx.obj.get("run_uuid"))
try:
polyaxon_client = PolyaxonClient()
response = polyaxon_client.runs_v1.invalidate_run(
owner, project_name, run_uuid)
Printer.print_success("Run was invalidated with uid {}".format(
response.uuid))
except (ApiException, HTTPError) as e:
handle_cli_error(
e, message="Could not invalidate run `{}`.".format(run_uuid))
sys.exit(1)
def stop(ctx, yes):
"""Stop run.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples:
\b
```bash
$ polyaxon runs stop
```
\b
```bash
$ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 stop
```
"""
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"), ctx.obj.get("run_uuid"))
if not yes and not click.confirm("Are sure you want to stop "
"run `{}`".format(run_uuid)):
click.echo("Existing without stopping run.")
sys.exit(0)
try:
polyaxon_client = PolyaxonClient()
polyaxon_client.runs_v1.stop_run(owner, project_name, run_uuid)
except (ApiException, HTTPError) as e:
handle_cli_error(e,
message="Could not stop run `{}`.".format(run_uuid))
sys.exit(1)
Printer.print_success("Run is being stopped.")
def logs(ctx, past, follow, hide_time):
"""Get run or run job logs.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples for getting run logs:
\b
```bash
$ polyaxon run logs
```
\b
```bash
$ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 -p mnist logs
```
"""
def get_run_logs():
if past:
try:
response = PolyaxonClient().run.logs(owner,
project_name,
run_uuid,
stream=False)
get_logs_handler(handle_job_info=True,
show_timestamp=not hide_time,
stream=False)(
response.content.decode().split("\n"))
print()
if not follow:
return
except (ApiException, HTTPError) as e:
if not follow:
handle_cli_error(
e,
message="Could not get logs for run `{}`.".format(
run_uuid))
sys.exit(1)
try:
PolyaxonClient().run.logs(
owner,
project_name,
run_uuid,
message_handler=get_logs_handler(handle_job_info=True,
show_timestamp=not hide_time),
)
except (ApiException, HTTPError) as e:
handle_cli_error(
e, message="Could not get logs for run `{}`.".format(run_uuid))
sys.exit(1)
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"), ctx.obj.get("run_uuid"))
get_run_logs()
def update(ctx, name, description, tags):
"""Update run.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples:
\b
```bash
$ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 update
--description="new description for my runs"
```
\b
```bash
$ polyaxon runs --project=cats-vs-dogs -id 8aac02e3a62a4f0aaa257c59da5eab80 update
--tags="foo, bar" --name="unique-name"
```
"""
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"), ctx.obj.get("run_uuid"))
update_dict = {}
if name:
update_dict["name"] = name
if description:
update_dict["description"] = description
tags = validate_tags(tags)
if tags:
update_dict["tags"] = tags
if not update_dict:
Printer.print_warning("No argument was provided to update the run.")
sys.exit(0)
try:
polyaxon_client = PolyaxonClient()
response = polyaxon_client.runs_v1.patch_run(owner, project_name,
run_uuid, update_dict)
except (ApiException, HTTPError) as e:
handle_cli_error(e,
message="Could not update run `{}`.".format(run_uuid))
sys.exit(1)
Printer.print_success("Run updated.")
get_run_details(response)
def statuses(ctx, watch):
"""Get run or run job statuses.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples getting run statuses:
\b
```bash
$ polyaxon runs statuses
```
\b
```bash
$ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 statuses
```
"""
def _get_run_statuses():
try:
for status, conditions in get_run_statuses(owner, project_name,
run_uuid, watch):
if not conditions:
continue
Printer.print_header("Latest status:")
latest_status = Printer.add_status_color({"status": status},
status_key="status")
click.echo("{}\n".format(latest_status["status"]))
objects = list_dicts_to_tabulate([
Printer.add_status_color(o.to_dict(), status_key="type")
for o in conditions
])
if objects:
Printer.print_header("Conditions:")
dict_tabulate(objects, is_list_dict=True)
except (ApiException, HTTPError, PolyaxonClientException) as e:
handle_cli_error(
e, message="Could get status for run `{}`.".format(run_uuid))
sys.exit(1)
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"), ctx.obj.get("run_uuid"))
_get_run_statuses()
def get(ctx):
"""Get run.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples for getting a run:
\b
```bash
$ polyaxon runs get # if run is cached
```
\b
```bash
$ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 get # project is cached
```
\b
```bash
$ polyaxon runs --project=cats-vs-dogs -id 8aac02e3a62a4f0aaa257c59da5eab80 get
```
\b
```bash
$ polyaxon runs -p alain/cats-vs-dogs --uid=8aac02e3a62a4f0aaa257c59da5eab80 get
```
"""
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"), ctx.obj.get("run_uuid"))
try:
polyaxon_client = PolyaxonClient()
response = polyaxon_client.runs_v1.get_run(owner, project_name,
run_uuid)
config = polyaxon_client.api_client.sanitize_for_serialization(
response)
cache.cache(config_manager=RunManager, response=config)
except (ApiException, HTTPError) as e:
handle_cli_error(
e, message="Could not load run `{}` info.".format(run_uuid))
sys.exit(1)
get_run_details(response)
def delete(ctx):
"""Delete a run.
Uses [Caching](/references/polyaxon-cli/#caching)
Example:
\b
```bash
$ polyaxon runs delete
```
\b
```bash
$ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 delete # project is cached
```
\b
```bash
$ polyaxon runs --project=cats-vs-dogs -id 8aac02e3a62a4f0aaa257c59da5eab80 delete
```
"""
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"), ctx.obj.get("run_uuid"))
if not click.confirm(
"Are sure you want to delete run `{}`".format(run_uuid)):
click.echo("Existing without deleting the run.")
sys.exit(1)
try:
polyaxon_client = PolyaxonClient()
polyaxon_client.runs_v1.delete_run(owner, project_name, run_uuid)
# Purge caching
RunManager.purge()
except (ApiException, HTTPError) as e:
handle_cli_error(e,
message="Could not delete run `{}`.".format(run_uuid))
sys.exit(1)
Printer.print_success("Run `{}` was delete successfully".format(run_uuid))
def outputs(ctx):
"""Download outputs for run.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples:
\b
```bash
$ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 outputs
```
"""
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"), ctx.obj.get("run_uuid"))
try:
PolyaxonClient().run.download_outputs(owner, project_name, run_uuid)
except (ApiException, HTTPError) as e:
handle_cli_error(
e,
message="Could not download outputs for run `{}`.".format(
run_uuid))
sys.exit(1)
Printer.print_success("Files downloaded.")
def resources(ctx, gpu):
"""Get run or run job resources.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples for getting run resources:
\b
```bash
$ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 resources
```
For GPU resources
\b
```bash
$ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 resources --gpu
```
"""
def get_run_resources():
try:
message_handler = Printer.gpu_resources if gpu else Printer.resources
PolyaxonClient().run.resources(owner,
project_name,
run_uuid,
message_handler=message_handler)
except (ApiException, HTTPError) as e:
handle_cli_error(
e,
message="Could not get resources for run `{}`.".format(
run_uuid))
sys.exit(1)
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"), ctx.obj.get("run_uuid"))
get_run_resources()
def code(ctx):
"""Download code for run.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples:
\b
```bash
$ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 code
```
"""
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"), ctx.obj.get("run_uuid"))
try:
polyaxon_client = PolyaxonClient()
code_ref = polyaxon_client.runs_v1.get_run_code_refs(
owner, project_name, run_uuid)
commit = None
if code_ref:
commit = code_ref.commit
Printer.print_header(
"Run has code ref: `{}`, downloading ...".format(commit))
else:
Printer.print_warning(
"Run has no code ref, downloading latest code...")
PolyaxonClient().project.download_repo(owner,
project_name,
commit=commit)
except (ApiException, HTTPError) as e:
handle_cli_error(
e,
message="Could not download outputs for run `{}`.".format(
run_uuid))
sys.exit(1)
Printer.print_success("Files downloaded.")
def logs(ctx, past, follow, hide_time):
"""Get run or run job logs.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples for getting run logs:
\b
```bash
$ polyaxon run logs
```
\b
```bash
$ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 -p mnist logs
```
"""
def show_run_logs():
is_done = False
last_logs_filepath = None
while not is_done:
response_tree = get_run_tree()
for filepath in response_tree.files:
if filepath == last_logs_filepath:
continue
last_logs_filepath = filepath
get_run_logs(filepath)
is_done = response_tree.is_done
if not is_done:
time.sleep(5)
def get_run_logs(path):
try:
response = polyaxon_client.runs_v1.get_run_logs_file(owner,
project_name,
run_uuid,
path=path)
get_logs_handler(handle_job_info=True,
show_timestamp=not hide_time,
stream=False)(response.split("\n"))
print()
except (ApiException, HTTPError) as e:
if not follow:
handle_cli_error(
e,
message="Could not get logs for run `{}`.".format(
run_uuid))
sys.exit(1)
def get_run_tree():
try:
return polyaxon_client.runs_v1.get_run_logs_tree(
owner, project_name, run_uuid)
except (ApiException, HTTPError) as e:
handle_cli_error(
e, message="Could not get logs for run `{}`.".format(run_uuid))
sys.exit(1)
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"), ctx.obj.get("run_uuid"))
polyaxon_client = PolyaxonClient()
show_run_logs()
