def _read_from_polyaxon_hub(hub: str): from polyaxon.client import PolyaxonClient from polyaxon.constants.globals import DEFAULT_HUB, NO_AUTH from polyaxon.env_vars.getters import get_component_info from polyaxon.schemas.cli.client_config import ClientConfig owner, component, version = get_component_info(hub) try: if owner == DEFAULT_HUB: config = ClientConfig() client = PolyaxonClient( config=config, token=NO_AUTH, ) else: client = PolyaxonClient() response = client.component_hub_v1.get_component_version( owner, component, version ) return _read_from_stream(response.content) except (ApiException, HTTPError) as e: raise PolyaxonClientException( "Component `{}` could not be fetched, " "an error was encountered".format(hub, e) )
def get_run_logs(): if past: try: response = PolyaxonClient().run.logs(owner, project_name, run_uuid, stream=False) get_logs_handler(handle_job_info=True, show_timestamp=not hide_time, stream=False)( response.content.decode().split("\n")) print() if not follow: return except (ApiException, HTTPError) as e: if not follow: handle_cli_error( e, message="Could not get logs for run `{}`.".format( run_uuid)) sys.exit(1) try: PolyaxonClient().run.logs( owner, project_name, run_uuid, message_handler=get_logs_handler(handle_job_info=True, show_timestamp=not hide_time), ) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not get logs for run `{}`.".format(run_uuid)) sys.exit(1)
def impersonate(owner, project, run_uuid): try: response = PolyaxonClient().runs_v1.impersonate_token( owner, project, run_uuid) polyaxon_client = PolyaxonClient(token=response.token) user = polyaxon_client.users_v1.get_user() access_token = AccessTokenConfig(username=user.username, token=response.token) create_context_auth(access_token) except (ApiException, HTTPError) as e: raise PolyaxonClientException( "This worker is not allowed to run this job %s." % e)
def create_run(): click.echo("Creating a run.") body = V1OperationBody(content=op_spec.to_dict(dump=True)) try: polyaxon_client = PolyaxonClient() response = polyaxon_client.runs_v1.create_run( owner, project_name, body) config = polyaxon_client.sanitize_for_serialization(response) cache.cache(config_manager=RunManager, config=config) Printer.print_success("A new run `{}` was created".format( response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not create a run.") sys.exit(1)
def invalidate_run(ctx): """Invalidate runs' cache inside this project. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon invalidate_builds ``` """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) try: polyaxon_client = PolyaxonClient() response = polyaxon_client.runs_v1.invalidate_run( owner, project_name, run_uuid) Printer.print_success("Run was invalidated with uid {}".format( response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not invalidate run `{}`.".format(run_uuid)) sys.exit(1)
def resume(ctx, polyaxonfile, u): """Resume run. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 resume ``` """ content = None if polyaxonfile: content = "{}".format(rhea.read(polyaxonfile)) # Check if we need to upload if u: ctx.invoke(upload, sync=False) owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) try: polyaxon_client = PolyaxonClient() body = V1Run(content=content) response = polyaxon_client.runs_v1.resume_run(owner, project_name, run_uuid, body) Printer.print_success("Run was resumed with uid {}".format( response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not resume run `{}`.".format(run_uuid)) sys.exit(1)
def stop(ctx, yes): """Stop run. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon runs stop ``` \b ```bash $ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 stop ``` """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) if not yes and not click.confirm("Are sure you want to stop " "run `{}`".format(run_uuid)): click.echo("Existing without stopping run.") sys.exit(0) try: polyaxon_client = PolyaxonClient() polyaxon_client.runs_v1.stop_run(owner, project_name, run_uuid) except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not stop run `{}`.".format(run_uuid)) sys.exit(1) Printer.print_success("Run is being stopped.")
def update(component, version, name, description, tags, private): """Update component hub. Uses /docs/core/cli/#caching Example: \b $ polyaxon hub update foobar --description="Image Classification with DL using TensorFlow" \b $ polyaxon hub update mike1/foobar --description="Image Classification with DL using TensorFlow" \b $ polyaxon hub update --tags="foo, bar" """ owner, component_hub, component_version, is_version = get_info( component, version) full_entity = ("{}/{}:{}".format(owner, component_hub, component_version) if is_version else "{}/{}".format(owner, component_hub)) update_dict = {} if name: update_dict["name"] = name if description: update_dict["description"] = description tags = validate_tags(tags) if tags: update_dict["tags"] = tags if private is not None: update_dict["is_public"] = not private if not update_dict: Printer.print_warning( "No argument was provided to update the component {}.".format( "version" if is_version else "hub")) sys.exit(1) try: polyaxon_client = PolyaxonClient() if is_version: response = polyaxon_client.component_hub_v1.patch_component_version( owner, component_hub, component_version, body=update_dict) Printer.print_success("Component version updated.") get_component_version_details(response) else: response = polyaxon_client.component_hub_v1.patch_component_hub( owner, component_hub, body=update_dict) Printer.print_success("Component updated.") get_entity_details(response, "Component hub") except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not update component {} `{}`.".format( "version" if is_version else "hub", full_entity), ) sys.exit(1)
def get_compatibility(key: str, service: str, version: str, is_cli=True): if not key: key = uuid.uuid4().hex try: version = version.lstrip("v").replace(".", "-")[:5] except Exception as e: CliConfigManager.reset(last_check=now()) if is_cli: handle_cli_error( e, message="Could parse the version {}.".format(version), ) polyaxon_client = PolyaxonClient(config=ClientConfig(), token=NO_AUTH) try: return polyaxon_client.versions_v1.get_compatibility(uuid=key, service=service, version=version) except ApiException as e: if e.status == 403: session_expired() CliConfigManager.reset(last_check=now()) if is_cli: handle_cli_error( e, message="Could not reach the compatibility API.", ) except HTTPError: CliConfigManager.reset(last_check=now()) if is_cli: Printer.print_error( "Could not connect to remote server to fetch compatibility versions.", )
def delete(ctx): """Delete project. Uses [Caching](/references/polyaxon-cli/#caching) """ owner, project_name = get_project_or_local(ctx.obj.get("project")) if not click.confirm("Are sure you want to delete project `{}/{}`".format( owner, project_name)): click.echo("Existing without deleting project.") sys.exit(1) try: polyaxon_client = PolyaxonClient() response = polyaxon_client.projects_v1.delete_project( owner, project_name) local_project = ProjectManager.get_config() if local_project and (owner, project_name) == ( local_project.user, local_project.name, ): # Purge caching ProjectManager.purge() except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not delete project `{}/{}`.".format( owner, project_name)) sys.exit(1) if response.status_code == 204: Printer.print_success("Project `{}/{}` was delete successfully".format( owner, project_name))
def get(ctx): """Get info for current project, by project_name, or user/project_name. Uses [Caching](/references/polyaxon-cli/#caching) Examples: To get current project: \b ```bash $ polyaxon project get ``` To get a project by name \b ```bash $ polyaxon project get user/project ``` """ owner, project_name = get_project_or_local(ctx.obj.get("project")) try: polyaxon_client = PolyaxonClient() response = polyaxon_client.projects_v1.get_project(owner, project_name) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not get project `{}`.".format(project_name)) sys.exit(1) get_project_details(response)
def delete(model, version): """Delete a model registry or a model version.""" owner, model_registry, model_version, is_version = get_info(model, version) full_entity = ("{}/{}:{}".format(owner, model_registry, model_version) if is_version else "{}/{}".format(owner, model_registry)) if not click.confirm("Are sure you want to delete model {} `{}`".format( "version" if is_version else "registry", full_entity)): click.echo("Existing without deleting model registry.") sys.exit(1) try: polyaxon_client = PolyaxonClient() if is_version: polyaxon_client.model_registry_v1.delete_model_version( owner, model_registry, model_version) else: polyaxon_client.model_registry_v1.delete_model_registry( owner, model_registry) Printer.print_success("Model {} `{}` was delete successfully".format( "version" if is_version else "registry", full_entity)) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not delete model {} `{}`.".format( "version" if is_version else "registry", full_entity), ) sys.exit(1)
def __init__( self, owner: str = None, project: str = None, run_uuid: str = None, client: PolyaxonClient = None, ): try: owner, project = get_project_or_local( get_project_full_name(owner=owner, project=project)) except PolyaxonClientException: pass if project is None: if settings.CLIENT_CONFIG.is_managed: owner, project, _run_uuid = get_run_info() run_uuid = run_uuid or _run_uuid else: raise PolyaxonClientException( "Please provide a valid project.") if not owner or not project: raise PolyaxonClientException( "Please provide a valid project with owner.") self.client = client if not (self.client or settings.CLIENT_CONFIG.is_offline): self.client = PolyaxonClient() self._owner = owner self._project = project self._run_uuid = get_run_or_local(run_uuid) self._run_data = polyaxon_sdk.V1Run() self._namespace = None
def set_versions_config( polyaxon_client=None, set_installation: bool = True, set_compatibility: bool = True, set_handler: bool = False, service=PolyaxonServices.CLI, version=pkg.VERSION, key: str = None, ): polyaxon_client = polyaxon_client or PolyaxonClient() server_installation = None if set_installation: server_installation = get_server_installation( polyaxon_client=polyaxon_client) if not key and server_installation and server_installation.key: key = server_installation.key compatibility = None if set_compatibility: compatibility = get_compatibility(key=key, service=service, version=version) log_handler = None if set_handler: log_handler = get_log_handler(polyaxon_client=polyaxon_client) return CliConfigManager.reset( last_check=now(), current_version=version, installation=server_installation.to_dict() if server_installation else {}, compatibility=compatibility.to_dict() if compatibility else {}, log_handler=log_handler.to_dict() if log_handler else {}, )
def upload(sync=True): # pylint:disable=assign-to-new-keyword """Upload code of the current directory while respecting the .polyaxonignore file.""" project = ProjectManager.get_config_or_raise() files = IgnoreManager.get_unignored_filepaths() try: with create_project_tarfile(files, project.name) as filepath: with get_files_by_paths("repo", [filepath]) as (files, files_size): try: PolyaxonClient().project.upload_repo(project.user, project.name, files, files_size, sync=sync) except ( PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException, ) as e: handle_cli_error( e, message="Could not upload code for project `{}`.". format(project.name), ) Printer.print_error( "Check the project exists, " "and that you have access rights, " "this could happen as well when uploading large files. " "Please also make sure that you have enough space to upload the data." ) sys.exit(1) Printer.print_success("Files uploaded.") except Exception as e: handle_cli_error(e, message="Could not upload the file.") sys.exit(1)
def delete(component, version): """Delete a component hub or a component version.""" owner, component_hub, component_version, is_version = get_info( component, version) full_entity = ("{}/{}:{}".format(owner, component_hub, component_version) if is_version else "{}/{}".format(owner, component_hub)) if not click.confirm( "Are sure you want to delete component {} `{}`".format( "version" if is_version else "hub", full_entity)): click.echo("Existing without deleting component hub.") sys.exit(1) try: polyaxon_client = PolyaxonClient() if is_version: polyaxon_client.component_hub_v1.delete_component_version( owner, component_hub, component_version) else: polyaxon_client.component_hub_v1.delete_component_hub( owner, component_hub) Printer.print_success( "Component {} `{}` was delete successfully".format( "version" if is_version else "hub", full_entity)) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not delete component {} `{}`.".format( "version" if is_version else "hub", full_entity), ) sys.exit(1)
def unbookmark(ctx): """Unbookmark run. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon runs unbookmark ``` \b ```bash $ polyaxon runs -uid=8aac02e3a62a4f0aaa257c59da5eab80 unbookmark ``` """ owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid")) try: polyaxon_client = PolyaxonClient() polyaxon_client.runs_v1.unbookmark_run(owner, project_name, run_uuid) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not unbookmark run `{}`.".format(run_uuid)) sys.exit(1) Printer.print_success("Run is unbookmarked.")
def get_log_handler(polyaxon_client=None): polyaxon_client = polyaxon_client or PolyaxonClient() try: return polyaxon_client.versions_v1.get_log_handler() except ApiException as e: if e.status == 403: session_expired() CliConfigManager.reset(last_check=now()) handle_cli_error(e, message="Could not get cli version.") except HTTPError: CliConfigManager.reset(last_check=now()) Printer.print_error("Could not connect to remote server to fetch log handler.")
def ci(ctx, enable, disable): # pylint:disable=assign-to-new-keyword """Enable/Disable CI on this project. Uses [Caching](/references/polyaxon-cli/#caching) Example: \b ```bash $ polyaxon project ci --enable ``` \b ```bash $ polyaxon project ci --disable ``` """ owner, project_name = get_project_or_local(ctx.obj.get("project")) polyaxon_client = PolyaxonClient() def enable_ci(): try: polyaxon_client.projects_v1.enable_ci(owner, project_name) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not enable CI on project `{}`.".format( project_name)) sys.exit(1) Printer.print_success( "Polyaxon CI was successfully enabled on project: `{}`.".format( project_name)) def disable_ci(): try: polyaxon_client.projects_v1.disable_ci(owner, project_name) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not disable CI on project `{}`.".format( project_name)) sys.exit(1) Printer.print_success( "Polyaxon CI was successfully disabled on project: `{}`.".format( project_name)) if enable: enable_ci() if disable: disable_ci()
def get_compatibility( key: str, service: str, version: str, is_cli: bool = True, set_config: bool = True, ): if not key: installation = CliConfigManager.get_value("installation") or {} key = installation.get("key") or uuid.uuid4().hex try: version = clean_version_for_compatibility(version) except Exception as e: if set_config: CliConfigManager.reset(last_check=now()) if is_cli: handle_cli_error( e, message="Could parse the version {}.".format(version), ) polyaxon_client = PolyaxonClient(config=ClientConfig(), token=NO_AUTH) try: return polyaxon_client.versions_v1.get_compatibility( uuid=key, service=service, version=version, _request_timeout=2, ) except ApiException as e: if e.status == 403 and is_cli: session_expired() if set_config: CliConfigManager.reset(last_check=now()) if is_cli: handle_cli_error( e, message="Could not reach the compatibility API.", ) except HTTPError: if set_config: CliConfigManager.reset(last_check=now()) if is_cli: Printer.print_error( "Could not connect to remote server to fetch compatibility versions.", ) except Exception as e: if set_config: CliConfigManager.reset(last_check=now()) if is_cli: Printer.print_error( "Unexpected error %s, " "could not connect to remote server to fetch compatibility versions." % e, )
def get_server_installation(polyaxon_client=None): polyaxon_client = polyaxon_client or PolyaxonClient() try: return polyaxon_client.versions_v1.get_installation() except ApiException as e: if e.status == 403: session_expired() handle_cli_error(e, message="Could not get server version.") except HTTPError: Printer.print_error( "Could not connect to remote server to fetch installation version.", )
def _read_from_polyaxon_hub(hub: str): from polyaxon.client import PolyaxonClient, ProjectClient from polyaxon.env_vars.getters.versioned_entity import get_component_info from polyaxon.schemas.cli.client_config import ClientConfig from polyaxon_schemas.constants import DEFAULT_HUB, NO_AUTH from polyaxon_schemas.lifecycle import V1ProjectVersionKind owner, component, version = get_component_info(hub) try: if owner == DEFAULT_HUB: config = ClientConfig() client = PolyaxonClient( config=config, token=NO_AUTH, ) else: client = PolyaxonClient() client = ProjectClient(owner=owner, project=component, client=client) response = client.get_version( kind=V1ProjectVersionKind.COMPONENT, version=version ) return _read_from_stream(response.content) except (ApiException, HTTPError) as e: raise PolyaxonClientException( "Component `{}` could not be fetched, " "an error was encountered".format(hub, e) )
def create_run(): click.echo("Creating a run.") run = V1Run(content=specification.config_dump) try: polyaxon_client = PolyaxonClient() response = polyaxon_client.runs_v1.create_run( owner, project_name, run) cache.cache(config_manager=RunManager, response=response) Printer.print_success("A new run `{}` was created".format( response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not create a run.") sys.exit(1)
def get_log_handler(polyaxon_client=None): polyaxon_client = polyaxon_client or PolyaxonClient() try: return polyaxon_client.versions_v1.get_log_handler() except ApiException as e: if e.status == 403: session_expired() sys.exit(1) handle_cli_error(e, message="Could not get cli version.") sys.exit(1) except HTTPError: Printer.print_error("Could not connect to remote server.") sys.exit(1)
def get_run_resources(): try: message_handler = Printer.gpu_resources if gpu else Printer.resources PolyaxonClient().run.resources(owner, project_name, run_uuid, message_handler=message_handler) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not get resources for run `{}`.".format( run_uuid)) sys.exit(1)
def whoami(): """Show current logged Polyaxon user.""" try: polyaxon_client = PolyaxonClient() user = polyaxon_client.users_v1.get_user() except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not load user info.") sys.exit(1) response = dict_to_tabulate(user.to_dict(), exclude_attrs=["role"]) Printer.print_header("User info:") dict_tabulate(response)
def upload(): """N.B. This is not available in all distributions. Upload code of the current directory while respecting the .polyaxonignore file. """ import sys from polyaxon.cli.errors import handle_cli_error from polyaxon.client import PolyaxonClient from polyaxon.exceptions import ( PolyaxonClientException, PolyaxonHTTPError, PolyaxonShouldExitError, ) from polyaxon.managers.ignore import IgnoreManager from polyaxon.managers.project import ProjectManager from polyaxon.utils.formatting import Printer from polyaxon.utils.path_utils import create_project_tarfile, get_files_by_paths project = ProjectManager.get_config_or_raise() files = IgnoreManager.get_unignored_filepaths() try: with create_project_tarfile(files, project.name) as filepath: with get_files_by_paths("repo", [filepath]) as (files, files_size): try: PolyaxonClient().project.upload_repo(project.user, project.name, files, files_size, sync=sync) except ( PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException, ) as e: handle_cli_error( e, message="Could not upload code for project `{}`.". format(project.name), ) Printer.print_error( "Check the project exists, " "and that you have access rights, " "this could happen as well when uploading large files. " "Please also make sure that you have enough space to upload the data." ) sys.exit(1) Printer.print_success("Files uploaded.") except Exception as e: handle_cli_error(e, message="Could not upload the file.") sys.exit(1)
def update(ctx, name, description, private): """Update project. Uses [Caching](/references/polyaxon-cli/#caching) Example: \b ```bash $ polyaxon update foobar --description="Image Classification with DL using TensorFlow" ``` \b ```bash $ polyaxon update mike1/foobar --description="Image Classification with DL using TensorFlow" ``` \b ```bash $ polyaxon update --tags="foo, bar" ``` """ owner, project_name = get_project_or_local(ctx.obj.get("project")) update_dict = {} if name: update_dict["name"] = name if description: update_dict["description"] = description if private is not None: update_dict["is_public"] = not private if not update_dict: Printer.print_warning( "No argument was provided to update the project.") sys.exit(1) try: polyaxon_client = PolyaxonClient() response = polyaxon_client.projects_v1.patch_project( owner, project_name, update_dict) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not update project `{}`.".format(project_name)) sys.exit(1) Printer.print_success("Project updated.") get_project_details(response)
def __init__( self, owner=None, project=None, run_uuid=None, client=None, track_logs=True, track_code=True, track_env=False, outputs_store=None, ): owner, project = get_project_info(owner=owner, project=project) if project is None: if settings.CLIENT_CONFIG.is_managed: owner, project, _run_uuid = self.get_run_info() run_uuid = run_uuid or _run_uuid else: raise PolyaxonClientException( "Please provide a valid project.") self.status = None self.client = client if not (self.client or settings.CLIENT_CONFIG.is_offline): self.client = PolyaxonClient() self.track_logs = track_logs self.track_code = track_code self.track_env = track_env self._owner = owner self._project = project self._run_uuid = run_uuid self.outputs_store = outputs_store # Setup the outputs store if outputs_store is None and settings.CLIENT_CONFIG.is_managed: self.set_outputs_store(outputs_path=get_outputs_path(), set_env_vars=True) self._run = polyaxon_sdk.V1Run() if settings.CLIENT_CONFIG.is_offline: return if self._run_uuid: self.refresh_data() # Track run env if settings.CLIENT_CONFIG.is_managed and self.track_env: self.log_run_env()
def create(name, description, tags, public): """Create a new model. Example: \b $ polyaxon registry create --name=kaniko --description="Tool to build container images" \b $ polyaxon registry create --name=owner/name --description="Model description" """ if not name: Printer.print_error( "Please provide a name to create a model registry.", command_help="registry create", sys_exit=True, ) owner, registry_name, _, _ = get_info(name, None) tags = validate_tags(tags) if not owner or not registry_name: Printer.print_error( "Please provide a valid model name with --name=owner/registry-name. " ) sys.exit(1) try: registry_config = V1ModelRegistry( name=registry_name, description=description, tags=tags, is_public=public ) polyaxon_client = PolyaxonClient() _registry = polyaxon_client.model_registry_v1.create_model_registry( owner, registry_config ) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not create model registry `{}`.".format(registry_name) ) sys.exit(1) Printer.print_success( "Model registry `{}` was created successfully.".format(_registry.name) ) click.echo( "You can view this model registry on Polyaxon UI: {}".format( get_dashboard_url(subpath="{}/registry/{}".format(owner, _registry.name)) ) )