def get_project_or_local(project=None, is_cli: bool = False):
from polyaxon import settings
if not project and not ProjectConfigManager.is_initialized():
if is_cli:
Printer.print_error("Please provide a valid project.")
sys.exit(1)
else:
raise PolyaxonClientException("Please provide a valid project.")
if project:
owner, project_name = get_project_info(project)
else:
project = ProjectConfigManager.get_config()
owner, project_name = project.owner, project.name
if not owner and (not settings.CLI_CONFIG or settings.CLI_CONFIG.is_ce):
owner = DEFAULT
if not all([owner, project_name]):
if is_cli:
Printer.print_error("Please provide a valid project.")
sys.exit(1)
else:
raise PolyaxonClientException("Please provide a valid project.")
return owner, project_name
def get_config_or_raise(cls):
run = cls.get_config()
if not run:
Printer.print_error("No run was provided.")
sys.exit(1)
return run
def get_project_or_local(project=None, is_cli: bool = False):
from polyaxon import settings
if not project and not ProjectConfigManager.is_initialized():
error_message = "Please provide a valid project or initialize a project in the current path."
if is_cli:
Printer.print_error(error_message)
sys.exit(1)
else:
raise PolyaxonClientException(error_message)
if project:
owner, project_name = get_entity_info(project)
else:
project = get_local_project()
owner, project_name = project.owner, project.name
if not owner:
owner = get_local_owner(is_cli=is_cli)
if not owner and (not settings.CLI_CONFIG or settings.CLI_CONFIG.is_ce):
owner = DEFAULT
if not all([owner, project_name]):
error_message = get_project_error_message(owner, project_name)
if is_cli:
Printer.print_error(error_message)
sys.exit(1)
else:
raise PolyaxonClientException(error_message)
return owner, project_name
def get_local_owner(is_cli: bool = False):
from polyaxon import settings
owner = None
if UserConfigManager.is_initialized():
try:
user_config = UserConfigManager.get_config()
owner = user_config.organization
except TypeError:
Printer.print_error(
"Found an invalid user config or user config cache, "
"if you are using Polyaxon CLI please run: "
"`polyaxon config purge --cache-only`",
sys_exit=True,
)
if not owner and (not settings.CLI_CONFIG or settings.CLI_CONFIG.is_ce):
owner = DEFAULT
if not owner:
error = "An context owner (user or organization) is required."
if is_cli:
Printer.print_error(error)
sys.exit(1)
else:
raise PolyaxonClientException(error)
return owner
def upgrade(config_file, deployment_type, manager_path, check, dry_run):
"""Upgrade a Polyaxon deployment."""
config = read_deployment_config(config_file, command="upgrade")
manager = DeployConfigManager(
config=config,
filepath=config_file,
deployment_type=deployment_type,
manager_path=manager_path,
dry_run=dry_run,
)
exception = None
if config:
Printer.print_success("Polyaxon `{}` deployment file is valid.".format(
config.deployment_chart))
if check:
try:
manager.check()
except Exception as e:
handle_cli_error(e,
message="Polyaxon deployment manager error.",
sys_exit=True)
else:
try:
manager.upgrade()
except Exception as e:
Printer.print_error("Polyaxon could not upgrade the deployment.")
exception = e
if exception:
Printer.print_error("Error message: {}.".format(exception))
def get_config_or_raise(cls):
user = cls.get_config()
if not user:
Printer.print_error("User configuration was not found.")
sys.exit(1)
return user
def deploy(config_file, deployment_type, manager_path, check, dry_run):
"""Deploy polyaxon."""
config = read_deployment_config(config_file)
manager = DeployManager(
config=config,
filepath=config_file,
deployment_type=deployment_type,
manager_path=manager_path,
dry_run=dry_run,
)
exception = None
if config:
Printer.print_success("Polyaxon `{}` deployment file is valid.".format(
config.deployment_chart))
if check:
try:
manager.check()
except Exception as e:
handle_cli_error(e,
message="Polyaxon deployment manager error.",
sys_exit=True)
else:
try:
manager.install()
except Exception as e:
Printer.print_error("Polyaxon could not be installed.")
exception = e
if exception:
Printer.print_error("Error message: {}.".format(exception),
sys_exit=True)
def upload(sync=True): # pylint:disable=assign-to-new-keyword
"""Upload code of the current directory while respecting the .polyaxonignore file."""
project = ProjectManager.get_config_or_raise()
files = IgnoreManager.get_unignored_filepaths()
try:
with create_project_tarfile(files, project.name) as filepath:
with get_files_by_paths("repo", [filepath]) as (files, files_size):
try:
PolyaxonClient().project.upload_repo(project.user,
project.name,
files,
files_size,
sync=sync)
except (
PolyaxonHTTPError,
PolyaxonShouldExitError,
PolyaxonClientException,
) as e:
handle_cli_error(
e,
message="Could not upload code for project `{}`.".
format(project.name),
)
Printer.print_error(
"Check the project exists, "
"and that you have access rights, "
"this could happen as well when uploading large files. "
"Please also make sure that you have enough space to upload the data."
)
sys.exit(1)
Printer.print_success("Files uploaded.")
except Exception as e:
handle_cli_error(e, message="Could not upload the file.")
sys.exit(1)
def check_polyaxonfile(polyaxonfile,
params=None,
profile=None,
queue=None,
nocache=None,
log=True):
if not polyaxonfile:
polyaxonfile = PolyaxonFile.check_default_path(path=".")
if not polyaxonfile:
polyaxonfile = ""
polyaxonfile = to_list(polyaxonfile)
exists = [os.path.isfile(f) for f in polyaxonfile]
parsed_params = None
if params:
parsed_params = parse_params(params)
if not any(exists):
Printer.print_error("Polyaxonfile is not present, "
"please run {}".format(constants.INIT_COMMAND))
sys.exit(1)
try:
plx_file = PolyaxonFile(polyaxonfile)
plx_file = plx_file.get_op_specification(params=parsed_params,
profile=profile,
queue=queue,
nocache=nocache)
if log:
Printer.print_success("Polyaxonfile valid")
return plx_file
except Exception as e:
handle_cli_error(e, message="Polyaxonfile is not valid.")
sys.exit(1)
def get_info(model: str = None, version: str = None):
if not any([model, version]):
Printer.print_error(
"A model registry or a model version is required.", sys_exit=True
)
if all([model, version]):
Printer.print_error(
"Only a model registry or a model version is required, not both.",
sys_exit=True,
)
if model:
entity = model
entity_name = "model registry"
is_version = False
else:
entity = version
entity_name = "model version"
is_version = True
try:
owner, model_registry, model_version = get_model_info(
entity=entity, entity_name=entity_name, is_cli=True
)
return owner, model_registry, model_version, is_version
except PolyaxonException as e:
handle_cli_error(
e,
message="Could not resolve the {} from the value `{}`.".format(
entity_name, entity
),
sys_exit=True,
)
def get(ctx, save, filename):
"""Get a component info by component_name, or owner/component_name.
Uses /docs/core/cli/#caching
Examples:
To get a component by name
\b
$ polyaxon hub get component_name
To get a component by owner/name
\b
$ polyaxon hub get owner/component_name
"""
name = ctx.obj.get("name")
if not name:
Printer.print_error("Please provide a valid component name!")
sys.exit(0)
try:
polyaxonfile = ConfigSpec.get_from(name, "hub").read()
except Exception as e:
handle_cli_error(e, message="Could not get component `{}`.".format(name))
sys.exit(1)
specification = get_specification(data=polyaxonfile)
polyaxonfile = yaml.dump(polyaxonfile)
get_component_details(polyaxonfile=polyaxonfile, specification=specification)
if save:
filename = filename or "polyaxonfile.yaml"
with open(filename, "w") as env_file:
env_file.write(polyaxonfile)
def upload(ctx, project, uid, path_from, path_to, is_file, sync_failure):
"""Upload runs' artifacts.
Uses /docs/core/cli/#caching
Examples:
\b
$ polyaxon ops upload -uid=8aac02e3a62a4f0aaa257c59da5eab80
\b
$ polyaxon ops upload -uid=8aac02e3a62a4f0aaa257c59da5eab80 path_from="path/to/upload"
\b
$ polyaxon ops upload -uid=8aac02e3a62a4f0aaa257c59da5eab80 path_to="path/to/upload/to"
"""
owner, project_name, run_uuid = get_project_run_or_local(
project or ctx.obj.get("project"),
uid or ctx.obj.get("run_uuid"),
is_cli=True,
)
try:
client = RunClient(owner=owner,
project=project_name,
run_uuid=run_uuid)
if is_file:
response = client.upload_artifact(filepath=path_from,
path=path_to,
overwrite=True)
else:
response = client.upload_artifacts_dir(
dirpath=path_from,
path=path_to,
overwrite=True,
relative_to=path_from,
)
except (
ApiException,
HTTPError,
PolyaxonHTTPError,
PolyaxonShouldExitError,
PolyaxonClientException,
) as e:
handle_cli_error(
e,
message="Could not upload artifacts for run `{}`".format(run_uuid))
sys.exit(1)
if response.status_code == 200:
Printer.print_success("Artifacts uploaded")
else:
if sync_failure:
client.log_failed(reason="OperationCli",
message="Operation failed uploading artifacts")
Printer.print_error(
"Error uploading artifacts. "
"Status: {}. Error: {}.".format(response.status_code,
response.content),
sys_exit=True,
)
def file(file_context, filepath, copy_path, track):
"""Create auth context."""
from polyaxon.init.file import create_file_lineage
from polyaxon.utils.hashing import hash_value
try:
file_context = V1FileType.from_dict(ConfigSpec.read_from(file_context))
except (PolyaxonSchemaError, ValidationError) as e:
Printer.print_error("received a non valid file context.")
Printer.print_error("Error message: {}.".format(e))
sys.exit(1)
filepath = os.path.join(filepath, file_context.filename)
check_or_create_path(filepath, is_dir=False)
# Clean any previous file on that path
if os.path.exists(filepath):
os.remove(filepath)
with open(filepath, "w") as generated_file:
generated_file.write(file_context.content)
if file_context.chmod:
subprocess.check_call(["chmod", file_context.chmod, filepath])
if copy_path:
filepath = copy_file(filepath, copy_path)
if track:
create_file_lineage(
filepath=filepath,
summary={"hash": hash_value(file_context.content)},
kind=file_context.kind,
)
Printer.print_success("File is initialized, path: `{}`".format(filepath))
def deploy(config_file, manager_path, check, dry_run):
"""Deploy polyaxon."""
config = read_deployment_config(config_file)
manager = DeployManager(config=config,
filepath=config_file,
manager_path=manager_path,
dry_run=dry_run)
exception = None
if check:
try:
manager.check()
except Exception as e:
handle_cli_error(e,
message="Polyaxon deployment file is not valid.")
sys.exit(1)
Printer.print_success("Polyaxon deployment file is valid.")
else:
try:
manager.install()
except Exception as e:
Printer.print_error("Polyaxon could not be installed.")
exception = e
if exception:
Printer.print_error("Error message: {}.".format(exception))
def create(ctx, name, owner, description, private, init):
"""Create a new project.
Uses /docs/core/cli/#caching
Example:
\b
$ polyaxon project create --name=cats-vs-dogs --description="Image Classification with DL"
"""
owner = owner or AuthConfigManager.get_value("username")
if not owner:
Printer.print_error(
"Please login first or provide a valid owner --owner. "
"`polyaxon login --help`")
sys.exit(1)
try:
project_config = V1Project(name=name,
description=description,
is_public=not private)
_project = ProjectClient(owner=owner).create(project_config)
except (ApiException, HTTPError) as e:
handle_cli_error(e,
message="Could not create project `{}`.".format(name))
sys.exit(1)
Printer.print_success("Project `{}` was created successfully.".format(
_project.name))
if init:
ctx.obj = {}
ctx.invoke(init_project, project=name)
def get_info(component: str = None, version: str = None):
if not any([component, version]):
Printer.print_error(
"A component or a component version is required.", sys_exit=True
)
if all([component, version]):
Printer.print_error(
"Only a component or a component version is required, not both.",
sys_exit=True,
)
if component:
entity = component
entity_name = "component"
is_version = False
else:
entity = version
entity_name = "component version"
is_version = True
try:
owner, component_hub, component_version = get_component_info(entity)
return owner, component_hub, component_version, is_version
except PolyaxonException as e:
handle_cli_error(
e,
message="Could not resolve the {} from the value `{}`.".format(
entity_name, entity
),
sys_exit=True,
)
def ls(owner, limit, offset):
"""List projects.
Uses /docs/core/cli/#caching
"""
owner = owner or AuthConfigManager.get_value("username")
if not owner:
Printer.print_error(
"Please login first or provide a valid owner --owner. "
"`polyaxon login --help`")
sys.exit(1)
try:
polyaxon_client = ProjectClient(owner=owner)
response = polyaxon_client.list(limit=limit, offset=offset)
except (ApiException, HTTPError) as e:
handle_cli_error(e, message="Could not get list of projects.")
sys.exit(1)
meta = get_meta_response(response)
if meta:
Printer.print_header("Projects for current user")
Printer.print_header("Navigation:")
dict_tabulate(meta)
else:
Printer.print_header("No projects found for current user")
objects = list_dicts_to_tabulate(
[o.to_dict() for o in response.results],
humanize_values=True,
exclude_attrs=["uuid", "description"],
)
if objects:
Printer.print_header("Projects:")
dict_tabulate(objects, is_list_dict=True)
def get_eager_matrix_operations(
content: str, compiled_operation: V1CompiledOperation, is_cli: bool = False,
) -> List[V1Operation]:
is_supported_in_eager_mode(compiled_operation)
try:
import numpy as np
except ImportError as e:
if is_cli:
Printer.print_error("numpy is required for this operation", sys_exit=True)
raise e
from polyaxon.polytune.search_managers.grid_search.manager import GridSearchManager
from polyaxon.polytune.search_managers.mapping.manager import MappingManager
from polyaxon.polytune.search_managers.random_search.manager import (
RandomSearchManager,
)
if compiled_operation.has_random_search_matrix:
suggestions = RandomSearchManager(compiled_operation.matrix).get_suggestions()
elif compiled_operation.has_grid_search_matrix:
suggestions = GridSearchManager(compiled_operation.matrix).get_suggestions()
elif compiled_operation.has_mapping_matrix:
suggestions = MappingManager(compiled_operation.matrix).get_suggestions()
else:
raise PolyaxonSchemaError(
"Received a bad configuration, eager mode not supported, "
"I should not be here!"
)
return get_ops_from_suggestions(
content=content, compiled_operation=compiled_operation, suggestions=suggestions
)
def check_polyaxonfile_kind(specification, kind):
if specification.kind != kind:
Printer.print_error(
"Your polyaxonfile must be of kind: `{}`, "
"received: `{}`.".format(kind, specification.kind)
)
sys.exit(-1)
def get_compatibility(key: str, service: str, version: str, is_cli=True):
if not key:
key = uuid.uuid4().hex
try:
version = version.lstrip("v").replace(".", "-")[:5]
except Exception as e:
CliConfigManager.reset(last_check=now())
if is_cli:
handle_cli_error(
e,
message="Could parse the version {}.".format(version),
)
polyaxon_client = PolyaxonClient(config=ClientConfig(), token=NO_AUTH)
try:
return polyaxon_client.versions_v1.get_compatibility(uuid=key,
service=service,
version=version)
except ApiException as e:
if e.status == 403:
session_expired()
CliConfigManager.reset(last_check=now())
if is_cli:
handle_cli_error(
e,
message="Could not reach the compatibility API.",
)
except HTTPError:
CliConfigManager.reset(last_check=now())
if is_cli:
Printer.print_error(
"Could not connect to remote server to fetch compatibility versions.",
)
def parse_params(params, is_cli: bool = True):
if isinstance(params, Mapping):
return params
parsed_params = {}
for param in params:
index = param.find("=")
if index == -1:
message = (
"Invalid format for -P parameter: '%s'. Use -P name=value." %
param)
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
name = param[:index]
value = param[index + 1:]
if name in parsed_params:
message = "Repeated parameter: '%s'" % name
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
parsed_params[name] = {"value": value}
return parsed_params
def get_model_info(entity: str, entity_name: str, is_cli: bool = False):
from polyaxon import settings
if not entity:
message = "Please provide a valid {}!".format(entity_name)
if is_cli:
Printer.print_error(message)
sys.exit(1)
else:
raise PolyaxonClientException(message)
owner = get_local_owner(is_cli=is_cli)
if not owner and (not settings.CLI_CONFIG or settings.CLI_CONFIG.is_ce):
owner = DEFAULT
owner, entity_namespace, version = get_versioned_entity_info(
entity=entity, entity_name=entity_name, default_owner=owner
)
owner = owner or settings.AUTH_CONFIG.username
if not all([owner, entity_name]):
message = "Please provide a valid {}.".format(entity_name)
if is_cli:
Printer.print_error(message)
sys.exit(1)
else:
raise PolyaxonClientException(message)
return owner, entity_namespace, version
def get_local_project(is_cli: bool = False):
try:
return ProjectConfigManager.get_config()
except Exception: # noqa
if is_cli:
Printer.print_error(CACHE_ERROR, sys_exit=True)
else:
raise PolyaxonSchemaError(CACHE_ERROR)
def check_old_packages():
pkg = "polyaxon-cli"
if get_version(package=pkg, show_error=False):
Printer.print_error(
"Legacy package `{pkg}` is installed. Please run `pip uninstall {pkg}`"
.format(pkg=pkg),
sys_exit=True,
)
def create(ctx, name, description, tags, public, init):
"""Create a new project.
Uses /docs/core/cli/#caching
Example:
\b
$ polyaxon project create --name=cats-vs-dogs --description="Image Classification with DL"
\b
$ polyaxon project create --name=owner/name --description="Project Description"
"""
if not name:
Printer.print_error(
"Please provide a valid name to create a project.",
command_help="project create",
sys_exit=True,
)
owner, project_name = resolve_entity_info(name or ctx.obj.get("project"),
is_cli=True,
entity_name="project")
tags = validate_tags(tags)
if not owner:
Printer.print_error(
"Please provide a valid name with an owner namespace: --name=owner/project."
)
sys.exit(1)
try:
project_config = V1Project(name=project_name,
description=description,
tags=tags,
is_public=public)
polyaxon_client = ProjectClient(owner=owner)
_project = polyaxon_client.create(project_config)
config = polyaxon_client.client.sanitize_for_serialization(_project)
cache.cache(config_manager=ProjectConfigManager, config=config)
except (ApiException, HTTPError) as e:
handle_cli_error(
e, message="Could not create project `{}`.".format(project_name))
sys.exit(1)
Printer.print_success("Project `{}` was created successfully.".format(
_project.name))
click.echo("You can view this project on Polyaxon UI: {}".format(
get_dashboard_url(subpath="{}/{}".format(owner, _project.name))))
if init:
ctx.obj = {}
ctx.invoke(
init_project,
project="{}/{}".format(owner, project_name),
polyaxonignore=True,
)
def get_config_or_raise(cls):
project = cls.get_config()
if not project:
Printer.print_error(
"No project was found, please initialize a project."
" {}".format(constants.INIT_COMMAND))
sys.exit(1)
return project
def get_config_or_raise(cls):
project = cls.get_config()
if not project:
Printer.print_error(
"Please initialize your project before uploading any code."
" {}".format(constants.INIT_COMMAND))
sys.exit(1)
return project
def run(ctx, name, owner, project_name, description, tags, specification, log):
try:
_run(ctx, name, owner, project_name, description, tags, specification, log)
except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e:
handle_cli_error(e, message="Could start local run.")
sys.exit(1)
except Exception as e:
Printer.print_error("Could start local run.")
Printer.print_error("Unexpected Error: `{}`.".format(e))
sys.exit(1)
def create(ctx, name, description, tags, private, init):
"""Create a new project.
Uses /docs/core/cli/#caching
Example:
\b
$ polyaxon project create --project=cats-vs-dogs --description="Image Classification with DL"
\b
$ polyaxon project create --project=owner/name --description="Project Description"
"""
if not name:
Printer.print_error(
"Please login provide a name to create a project.",
command_help="project create",
sys_exit=True,
)
owner, project_name = get_project_or_local(name or ctx.obj.get("project"),
is_cli=True)
owner = owner or settings.AUTH_CONFIG.username
if not owner and (not settings.CLI_CONFIG or settings.CLI_CONFIG.is_ce):
owner = DEFAULT
tags = validate_tags(tags)
if not owner:
Printer.print_error(
"Please login first or provide a valid name with owner --name=owner/project. "
"`polyaxon login --help`")
sys.exit(1)
try:
project_config = V1Project(name=project_name,
description=description,
tags=tags,
is_public=not private)
polyaxon_client = ProjectClient(owner=owner)
_project = polyaxon_client.create(project_config)
config = polyaxon_client.client.sanitize_for_serialization(_project)
cache.cache(config_manager=ProjectConfigManager, config=config)
except (ApiException, HTTPError) as e:
handle_cli_error(
e, message="Could not create project `{}`.".format(project_name))
sys.exit(1)
Printer.print_success("Project `{}` was created successfully.".format(
_project.name))
click.echo("You can view this project on Polyaxon UI: {}".format(
get_dashboard_url(subpath="{}/{}".format(owner, _project.name))))
if init:
ctx.obj = {}
ctx.invoke(init_project, project="{}/{}".format(owner, project_name))
def run(
ctx,
name: str,
owner: str,
project_name: str,
description: str,
tags: List[str],
op_spec: V1Operation,
upload: Callable,
log: bool,
watch: bool,
can_upload: bool,
):
def create_run():
click.echo("Creating a run.")
body = V1OperationBody(
name=name,
description=description,
tags=tags,
content=op_spec.to_dict(dump=True),
)
try:
polyaxon_client = PolyaxonClient()
response = polyaxon_client.runs_v1.create_run(
owner, project_name, body)
config = polyaxon_client.sanitize_for_serialization(response)
cache.cache(config_manager=RunManager, config=config)
Printer.print_success("A new run `{}` was created".format(
response.uuid))
except (ApiException, HTTPError) as e:
handle_cli_error(e, message="Could not create a run.")
sys.exit(1)
# Check if we need to upload
if upload:
if can_upload:
Printer.print_error(
"Uploading is not supported when switching project context!")
click.echo(
"Please, either omit the `-u` option or `-p` / `--project=` option."
)
sys.exit(1)
ctx.invoke(upload_cmd, sync=False)
create_run()
# Check if we need to invoke logs
if watch:
ctx.obj = {"project": "{}/{}".format(owner, project_name)}
ctx.invoke(statuses, watch=True)
# Check if we need to invoke logs
if log:
ctx.obj = {"project": "{}/{}".format(owner, project_name)}
ctx.invoke(run_logs)
