def get_project_or_local(project=None, is_cli: bool = False): from polyaxon import settings if not project and not ProjectConfigManager.is_initialized(): if is_cli: Printer.print_error("Please provide a valid project.") sys.exit(1) else: raise PolyaxonClientException("Please provide a valid project.") if project: owner, project_name = get_project_info(project) else: project = ProjectConfigManager.get_config() owner, project_name = project.owner, project.name if not owner and (not settings.CLI_CONFIG or settings.CLI_CONFIG.is_ce): owner = DEFAULT if not all([owner, project_name]): if is_cli: Printer.print_error("Please provide a valid project.") sys.exit(1) else: raise PolyaxonClientException("Please provide a valid project.") return owner, project_name
def get_config_or_raise(cls): run = cls.get_config() if not run: Printer.print_error("No run was provided.") sys.exit(1) return run
def get_project_or_local(project=None, is_cli: bool = False): from polyaxon import settings if not project and not ProjectConfigManager.is_initialized(): error_message = "Please provide a valid project or initialize a project in the current path." if is_cli: Printer.print_error(error_message) sys.exit(1) else: raise PolyaxonClientException(error_message) if project: owner, project_name = get_entity_info(project) else: project = get_local_project() owner, project_name = project.owner, project.name if not owner: owner = get_local_owner(is_cli=is_cli) if not owner and (not settings.CLI_CONFIG or settings.CLI_CONFIG.is_ce): owner = DEFAULT if not all([owner, project_name]): error_message = get_project_error_message(owner, project_name) if is_cli: Printer.print_error(error_message) sys.exit(1) else: raise PolyaxonClientException(error_message) return owner, project_name
def get_local_owner(is_cli: bool = False): from polyaxon import settings owner = None if UserConfigManager.is_initialized(): try: user_config = UserConfigManager.get_config() owner = user_config.organization except TypeError: Printer.print_error( "Found an invalid user config or user config cache, " "if you are using Polyaxon CLI please run: " "`polyaxon config purge --cache-only`", sys_exit=True, ) if not owner and (not settings.CLI_CONFIG or settings.CLI_CONFIG.is_ce): owner = DEFAULT if not owner: error = "An context owner (user or organization) is required." if is_cli: Printer.print_error(error) sys.exit(1) else: raise PolyaxonClientException(error) return owner
def upgrade(config_file, deployment_type, manager_path, check, dry_run): """Upgrade a Polyaxon deployment.""" config = read_deployment_config(config_file, command="upgrade") manager = DeployConfigManager( config=config, filepath=config_file, deployment_type=deployment_type, manager_path=manager_path, dry_run=dry_run, ) exception = None if config: Printer.print_success("Polyaxon `{}` deployment file is valid.".format( config.deployment_chart)) if check: try: manager.check() except Exception as e: handle_cli_error(e, message="Polyaxon deployment manager error.", sys_exit=True) else: try: manager.upgrade() except Exception as e: Printer.print_error("Polyaxon could not upgrade the deployment.") exception = e if exception: Printer.print_error("Error message: {}.".format(exception))
def get_config_or_raise(cls): user = cls.get_config() if not user: Printer.print_error("User configuration was not found.") sys.exit(1) return user
def deploy(config_file, deployment_type, manager_path, check, dry_run): """Deploy polyaxon.""" config = read_deployment_config(config_file) manager = DeployManager( config=config, filepath=config_file, deployment_type=deployment_type, manager_path=manager_path, dry_run=dry_run, ) exception = None if config: Printer.print_success("Polyaxon `{}` deployment file is valid.".format( config.deployment_chart)) if check: try: manager.check() except Exception as e: handle_cli_error(e, message="Polyaxon deployment manager error.", sys_exit=True) else: try: manager.install() except Exception as e: Printer.print_error("Polyaxon could not be installed.") exception = e if exception: Printer.print_error("Error message: {}.".format(exception), sys_exit=True)
def upload(sync=True): # pylint:disable=assign-to-new-keyword """Upload code of the current directory while respecting the .polyaxonignore file.""" project = ProjectManager.get_config_or_raise() files = IgnoreManager.get_unignored_filepaths() try: with create_project_tarfile(files, project.name) as filepath: with get_files_by_paths("repo", [filepath]) as (files, files_size): try: PolyaxonClient().project.upload_repo(project.user, project.name, files, files_size, sync=sync) except ( PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException, ) as e: handle_cli_error( e, message="Could not upload code for project `{}`.". format(project.name), ) Printer.print_error( "Check the project exists, " "and that you have access rights, " "this could happen as well when uploading large files. " "Please also make sure that you have enough space to upload the data." ) sys.exit(1) Printer.print_success("Files uploaded.") except Exception as e: handle_cli_error(e, message="Could not upload the file.") sys.exit(1)
def check_polyaxonfile(polyaxonfile, params=None, profile=None, queue=None, nocache=None, log=True): if not polyaxonfile: polyaxonfile = PolyaxonFile.check_default_path(path=".") if not polyaxonfile: polyaxonfile = "" polyaxonfile = to_list(polyaxonfile) exists = [os.path.isfile(f) for f in polyaxonfile] parsed_params = None if params: parsed_params = parse_params(params) if not any(exists): Printer.print_error("Polyaxonfile is not present, " "please run {}".format(constants.INIT_COMMAND)) sys.exit(1) try: plx_file = PolyaxonFile(polyaxonfile) plx_file = plx_file.get_op_specification(params=parsed_params, profile=profile, queue=queue, nocache=nocache) if log: Printer.print_success("Polyaxonfile valid") return plx_file except Exception as e: handle_cli_error(e, message="Polyaxonfile is not valid.") sys.exit(1)
def get_info(model: str = None, version: str = None): if not any([model, version]): Printer.print_error( "A model registry or a model version is required.", sys_exit=True ) if all([model, version]): Printer.print_error( "Only a model registry or a model version is required, not both.", sys_exit=True, ) if model: entity = model entity_name = "model registry" is_version = False else: entity = version entity_name = "model version" is_version = True try: owner, model_registry, model_version = get_model_info( entity=entity, entity_name=entity_name, is_cli=True ) return owner, model_registry, model_version, is_version except PolyaxonException as e: handle_cli_error( e, message="Could not resolve the {} from the value `{}`.".format( entity_name, entity ), sys_exit=True, )
def get(ctx, save, filename): """Get a component info by component_name, or owner/component_name. Uses /docs/core/cli/#caching Examples: To get a component by name \b $ polyaxon hub get component_name To get a component by owner/name \b $ polyaxon hub get owner/component_name """ name = ctx.obj.get("name") if not name: Printer.print_error("Please provide a valid component name!") sys.exit(0) try: polyaxonfile = ConfigSpec.get_from(name, "hub").read() except Exception as e: handle_cli_error(e, message="Could not get component `{}`.".format(name)) sys.exit(1) specification = get_specification(data=polyaxonfile) polyaxonfile = yaml.dump(polyaxonfile) get_component_details(polyaxonfile=polyaxonfile, specification=specification) if save: filename = filename or "polyaxonfile.yaml" with open(filename, "w") as env_file: env_file.write(polyaxonfile)
def upload(ctx, project, uid, path_from, path_to, is_file, sync_failure): """Upload runs' artifacts. Uses /docs/core/cli/#caching Examples: \b $ polyaxon ops upload -uid=8aac02e3a62a4f0aaa257c59da5eab80 \b $ polyaxon ops upload -uid=8aac02e3a62a4f0aaa257c59da5eab80 path_from="path/to/upload" \b $ polyaxon ops upload -uid=8aac02e3a62a4f0aaa257c59da5eab80 path_to="path/to/upload/to" """ owner, project_name, run_uuid = get_project_run_or_local( project or ctx.obj.get("project"), uid or ctx.obj.get("run_uuid"), is_cli=True, ) try: client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) if is_file: response = client.upload_artifact(filepath=path_from, path=path_to, overwrite=True) else: response = client.upload_artifacts_dir( dirpath=path_from, path=path_to, overwrite=True, relative_to=path_from, ) except ( ApiException, HTTPError, PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException, ) as e: handle_cli_error( e, message="Could not upload artifacts for run `{}`".format(run_uuid)) sys.exit(1) if response.status_code == 200: Printer.print_success("Artifacts uploaded") else: if sync_failure: client.log_failed(reason="OperationCli", message="Operation failed uploading artifacts") Printer.print_error( "Error uploading artifacts. " "Status: {}. Error: {}.".format(response.status_code, response.content), sys_exit=True, )
def file(file_context, filepath, copy_path, track): """Create auth context.""" from polyaxon.init.file import create_file_lineage from polyaxon.utils.hashing import hash_value try: file_context = V1FileType.from_dict(ConfigSpec.read_from(file_context)) except (PolyaxonSchemaError, ValidationError) as e: Printer.print_error("received a non valid file context.") Printer.print_error("Error message: {}.".format(e)) sys.exit(1) filepath = os.path.join(filepath, file_context.filename) check_or_create_path(filepath, is_dir=False) # Clean any previous file on that path if os.path.exists(filepath): os.remove(filepath) with open(filepath, "w") as generated_file: generated_file.write(file_context.content) if file_context.chmod: subprocess.check_call(["chmod", file_context.chmod, filepath]) if copy_path: filepath = copy_file(filepath, copy_path) if track: create_file_lineage( filepath=filepath, summary={"hash": hash_value(file_context.content)}, kind=file_context.kind, ) Printer.print_success("File is initialized, path: `{}`".format(filepath))
def deploy(config_file, manager_path, check, dry_run): """Deploy polyaxon.""" config = read_deployment_config(config_file) manager = DeployManager(config=config, filepath=config_file, manager_path=manager_path, dry_run=dry_run) exception = None if check: try: manager.check() except Exception as e: handle_cli_error(e, message="Polyaxon deployment file is not valid.") sys.exit(1) Printer.print_success("Polyaxon deployment file is valid.") else: try: manager.install() except Exception as e: Printer.print_error("Polyaxon could not be installed.") exception = e if exception: Printer.print_error("Error message: {}.".format(exception))
def create(ctx, name, owner, description, private, init): """Create a new project. Uses /docs/core/cli/#caching Example: \b $ polyaxon project create --name=cats-vs-dogs --description="Image Classification with DL" """ owner = owner or AuthConfigManager.get_value("username") if not owner: Printer.print_error( "Please login first or provide a valid owner --owner. " "`polyaxon login --help`") sys.exit(1) try: project_config = V1Project(name=name, description=description, is_public=not private) _project = ProjectClient(owner=owner).create(project_config) except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not create project `{}`.".format(name)) sys.exit(1) Printer.print_success("Project `{}` was created successfully.".format( _project.name)) if init: ctx.obj = {} ctx.invoke(init_project, project=name)
def get_info(component: str = None, version: str = None): if not any([component, version]): Printer.print_error( "A component or a component version is required.", sys_exit=True ) if all([component, version]): Printer.print_error( "Only a component or a component version is required, not both.", sys_exit=True, ) if component: entity = component entity_name = "component" is_version = False else: entity = version entity_name = "component version" is_version = True try: owner, component_hub, component_version = get_component_info(entity) return owner, component_hub, component_version, is_version except PolyaxonException as e: handle_cli_error( e, message="Could not resolve the {} from the value `{}`.".format( entity_name, entity ), sys_exit=True, )
def ls(owner, limit, offset): """List projects. Uses /docs/core/cli/#caching """ owner = owner or AuthConfigManager.get_value("username") if not owner: Printer.print_error( "Please login first or provide a valid owner --owner. " "`polyaxon login --help`") sys.exit(1) try: polyaxon_client = ProjectClient(owner=owner) response = polyaxon_client.list(limit=limit, offset=offset) except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not get list of projects.") sys.exit(1) meta = get_meta_response(response) if meta: Printer.print_header("Projects for current user") Printer.print_header("Navigation:") dict_tabulate(meta) else: Printer.print_header("No projects found for current user") objects = list_dicts_to_tabulate( [o.to_dict() for o in response.results], humanize_values=True, exclude_attrs=["uuid", "description"], ) if objects: Printer.print_header("Projects:") dict_tabulate(objects, is_list_dict=True)
def get_eager_matrix_operations( content: str, compiled_operation: V1CompiledOperation, is_cli: bool = False, ) -> List[V1Operation]: is_supported_in_eager_mode(compiled_operation) try: import numpy as np except ImportError as e: if is_cli: Printer.print_error("numpy is required for this operation", sys_exit=True) raise e from polyaxon.polytune.search_managers.grid_search.manager import GridSearchManager from polyaxon.polytune.search_managers.mapping.manager import MappingManager from polyaxon.polytune.search_managers.random_search.manager import ( RandomSearchManager, ) if compiled_operation.has_random_search_matrix: suggestions = RandomSearchManager(compiled_operation.matrix).get_suggestions() elif compiled_operation.has_grid_search_matrix: suggestions = GridSearchManager(compiled_operation.matrix).get_suggestions() elif compiled_operation.has_mapping_matrix: suggestions = MappingManager(compiled_operation.matrix).get_suggestions() else: raise PolyaxonSchemaError( "Received a bad configuration, eager mode not supported, " "I should not be here!" ) return get_ops_from_suggestions( content=content, compiled_operation=compiled_operation, suggestions=suggestions )
def check_polyaxonfile_kind(specification, kind): if specification.kind != kind: Printer.print_error( "Your polyaxonfile must be of kind: `{}`, " "received: `{}`.".format(kind, specification.kind) ) sys.exit(-1)
def get_compatibility(key: str, service: str, version: str, is_cli=True): if not key: key = uuid.uuid4().hex try: version = version.lstrip("v").replace(".", "-")[:5] except Exception as e: CliConfigManager.reset(last_check=now()) if is_cli: handle_cli_error( e, message="Could parse the version {}.".format(version), ) polyaxon_client = PolyaxonClient(config=ClientConfig(), token=NO_AUTH) try: return polyaxon_client.versions_v1.get_compatibility(uuid=key, service=service, version=version) except ApiException as e: if e.status == 403: session_expired() CliConfigManager.reset(last_check=now()) if is_cli: handle_cli_error( e, message="Could not reach the compatibility API.", ) except HTTPError: CliConfigManager.reset(last_check=now()) if is_cli: Printer.print_error( "Could not connect to remote server to fetch compatibility versions.", )
def parse_params(params, is_cli: bool = True): if isinstance(params, Mapping): return params parsed_params = {} for param in params: index = param.find("=") if index == -1: message = ( "Invalid format for -P parameter: '%s'. Use -P name=value." % param) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) name = param[:index] value = param[index + 1:] if name in parsed_params: message = "Repeated parameter: '%s'" % name if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) parsed_params[name] = {"value": value} return parsed_params
def get_model_info(entity: str, entity_name: str, is_cli: bool = False): from polyaxon import settings if not entity: message = "Please provide a valid {}!".format(entity_name) if is_cli: Printer.print_error(message) sys.exit(1) else: raise PolyaxonClientException(message) owner = get_local_owner(is_cli=is_cli) if not owner and (not settings.CLI_CONFIG or settings.CLI_CONFIG.is_ce): owner = DEFAULT owner, entity_namespace, version = get_versioned_entity_info( entity=entity, entity_name=entity_name, default_owner=owner ) owner = owner or settings.AUTH_CONFIG.username if not all([owner, entity_name]): message = "Please provide a valid {}.".format(entity_name) if is_cli: Printer.print_error(message) sys.exit(1) else: raise PolyaxonClientException(message) return owner, entity_namespace, version
def get_local_project(is_cli: bool = False): try: return ProjectConfigManager.get_config() except Exception: # noqa if is_cli: Printer.print_error(CACHE_ERROR, sys_exit=True) else: raise PolyaxonSchemaError(CACHE_ERROR)
def check_old_packages(): pkg = "polyaxon-cli" if get_version(package=pkg, show_error=False): Printer.print_error( "Legacy package `{pkg}` is installed. Please run `pip uninstall {pkg}`" .format(pkg=pkg), sys_exit=True, )
def create(ctx, name, description, tags, public, init): """Create a new project. Uses /docs/core/cli/#caching Example: \b $ polyaxon project create --name=cats-vs-dogs --description="Image Classification with DL" \b $ polyaxon project create --name=owner/name --description="Project Description" """ if not name: Printer.print_error( "Please provide a valid name to create a project.", command_help="project create", sys_exit=True, ) owner, project_name = resolve_entity_info(name or ctx.obj.get("project"), is_cli=True, entity_name="project") tags = validate_tags(tags) if not owner: Printer.print_error( "Please provide a valid name with an owner namespace: --name=owner/project." ) sys.exit(1) try: project_config = V1Project(name=project_name, description=description, tags=tags, is_public=public) polyaxon_client = ProjectClient(owner=owner) _project = polyaxon_client.create(project_config) config = polyaxon_client.client.sanitize_for_serialization(_project) cache.cache(config_manager=ProjectConfigManager, config=config) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not create project `{}`.".format(project_name)) sys.exit(1) Printer.print_success("Project `{}` was created successfully.".format( _project.name)) click.echo("You can view this project on Polyaxon UI: {}".format( get_dashboard_url(subpath="{}/{}".format(owner, _project.name)))) if init: ctx.obj = {} ctx.invoke( init_project, project="{}/{}".format(owner, project_name), polyaxonignore=True, )
def get_config_or_raise(cls): project = cls.get_config() if not project: Printer.print_error( "No project was found, please initialize a project." " {}".format(constants.INIT_COMMAND)) sys.exit(1) return project
def get_config_or_raise(cls): project = cls.get_config() if not project: Printer.print_error( "Please initialize your project before uploading any code." " {}".format(constants.INIT_COMMAND)) sys.exit(1) return project
def run(ctx, name, owner, project_name, description, tags, specification, log): try: _run(ctx, name, owner, project_name, description, tags, specification, log) except (PolyaxonHTTPError, PolyaxonShouldExitError, PolyaxonClientException) as e: handle_cli_error(e, message="Could start local run.") sys.exit(1) except Exception as e: Printer.print_error("Could start local run.") Printer.print_error("Unexpected Error: `{}`.".format(e)) sys.exit(1)
def create(ctx, name, description, tags, private, init): """Create a new project. Uses /docs/core/cli/#caching Example: \b $ polyaxon project create --project=cats-vs-dogs --description="Image Classification with DL" \b $ polyaxon project create --project=owner/name --description="Project Description" """ if not name: Printer.print_error( "Please login provide a name to create a project.", command_help="project create", sys_exit=True, ) owner, project_name = get_project_or_local(name or ctx.obj.get("project"), is_cli=True) owner = owner or settings.AUTH_CONFIG.username if not owner and (not settings.CLI_CONFIG or settings.CLI_CONFIG.is_ce): owner = DEFAULT tags = validate_tags(tags) if not owner: Printer.print_error( "Please login first or provide a valid name with owner --name=owner/project. " "`polyaxon login --help`") sys.exit(1) try: project_config = V1Project(name=project_name, description=description, tags=tags, is_public=not private) polyaxon_client = ProjectClient(owner=owner) _project = polyaxon_client.create(project_config) config = polyaxon_client.client.sanitize_for_serialization(_project) cache.cache(config_manager=ProjectConfigManager, config=config) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not create project `{}`.".format(project_name)) sys.exit(1) Printer.print_success("Project `{}` was created successfully.".format( _project.name)) click.echo("You can view this project on Polyaxon UI: {}".format( get_dashboard_url(subpath="{}/{}".format(owner, _project.name)))) if init: ctx.obj = {} ctx.invoke(init_project, project="{}/{}".format(owner, project_name))
def run( ctx, name: str, owner: str, project_name: str, description: str, tags: List[str], op_spec: V1Operation, upload: Callable, log: bool, watch: bool, can_upload: bool, ): def create_run(): click.echo("Creating a run.") body = V1OperationBody( name=name, description=description, tags=tags, content=op_spec.to_dict(dump=True), ) try: polyaxon_client = PolyaxonClient() response = polyaxon_client.runs_v1.create_run( owner, project_name, body) config = polyaxon_client.sanitize_for_serialization(response) cache.cache(config_manager=RunManager, config=config) Printer.print_success("A new run `{}` was created".format( response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not create a run.") sys.exit(1) # Check if we need to upload if upload: if can_upload: Printer.print_error( "Uploading is not supported when switching project context!") click.echo( "Please, either omit the `-u` option or `-p` / `--project=` option." ) sys.exit(1) ctx.invoke(upload_cmd, sync=False) create_run() # Check if we need to invoke logs if watch: ctx.obj = {"project": "{}/{}".format(owner, project_name)} ctx.invoke(statuses, watch=True) # Check if we need to invoke logs if log: ctx.obj = {"project": "{}/{}".format(owner, project_name)} ctx.invoke(run_logs)