def collect_references(config: V1Operation, path_context: str = None): if config.has_component_reference: return config elif config.has_hub_reference: component = ConfigSpec.get_from(config.hub_ref, "hub").read() elif config.has_url_reference: component = ConfigSpec.get_from(config.url_ref, "url").read() elif config.has_path_reference: path_ref = config.path_ref if path_context: path_ref = os.path.join( os.path.dirname(os.path.abspath(path_context)), path_ref) component = ConfigSpec.get_from(path_ref).read() else: raise PolyaxonfileError("Operation found without component") component = get_specification(data=component) if component.kind != kinds.COMPONENT: if config.has_url_reference: ref_type = "Url ref" ref = config.url_ref else: ref_type = "Path ref" ref = config.path_ref raise PolyaxonfileError( "the reference ({}) `{}` is of kind `{}`, it should be a `{}`". format(ref, ref_type, component.kind, kinds.COMPONENT)) config.component = component if component.is_dag_run: component.run.collect_components() return config
def get_config_from_env(cls, **kwargs) -> ClientConfig: tmp_path = os.path.join(CONTEXT_TMP_POLYAXON_PATH, cls.CONFIG_FILE_NAME) user_path = os.path.join(CONTEXT_USER_POLYAXON_PATH, cls.CONFIG_FILE_NAME) config = ConfigManager.read_configs( [ ConfigSpec(tmp_path, config_type=".json", check_if_exists=False), ConfigSpec(user_path, config_type=".json", check_if_exists=False), os.environ, ] ) return ClientConfig.from_dict(config.data)
def collect_references(config: V1Operation): if config.has_component_reference or config.has_hub_reference: return config elif config.has_url_reference: component = ConfigSpec.get_from(config.url_ref, "url").read() elif config.has_path_reference: component = ConfigSpec.get_from(config.path_ref).read() else: raise PolyaxonfileError("Operation found without component") component = get_specification(data=component) config.component = component return config
def test_reads_config_map(self): config = ConfigSpec.read_from([{"x": "y"}, {1: 2}, {"x": "override y"}]) assert config == {"x": "override y", 1: 2} config = ConfigSpec.read_from( [ {"x": "y"}, {1: 2}, {"x": "override y"}, "tests/fixtures/parsing/yaml_file.yml", "tests/fixtures/parsing/json_file.json", ] ) assert config == {"x": 1, "y": 2, 1: 2, "foo": "bar", "type": "json"}
def get_config_from_env(cls) -> AgentConfig: tmp_path = os.path.join(CONTEXT_TMP_POLYAXON_PATH, ".agent") user_path = os.path.join(CONTEXT_USER_POLYAXON_PATH, ".agent") config_paths = [ os.environ, ConfigSpec(tmp_path, config_type=".json", check_if_exists=False), ConfigSpec(user_path, config_type=".json", check_if_exists=False), { "dummy": "dummy" }, ] agent_config = ConfigManager.read_configs(config_paths) return AgentConfig.from_dict(agent_config.data)
def get(ctx, save, filename): """Get a component info by component_name, or owner/component_name. Uses /docs/core/cli/#caching Examples: To get a component by name \b $ polyaxon hub get component_name To get a component by owner/name \b $ polyaxon hub get owner/component_name """ name = ctx.obj.get("name") if not name: Printer.print_error("Please provide a valid component name!") sys.exit(0) try: polyaxonfile = ConfigSpec.get_from(name, "hub").read() except Exception as e: handle_cli_error(e, message="Could not get component `{}`.".format(name)) sys.exit(1) specification = get_specification(data=polyaxonfile) polyaxonfile = yaml.dump(polyaxonfile) get_component_details(polyaxonfile=polyaxonfile, specification=specification) if save: filename = filename or "polyaxonfile.yaml" with open(filename, "w") as env_file: env_file.write(polyaxonfile)
def resume(ctx, polyaxonfile, u): """Resume run. Uses [Caching](/references/polyaxon-cli/#caching) Examples: \b ```bash $ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 resume ``` """ content = None if polyaxonfile: content = "{}".format(ConfigSpec.read_from(polyaxonfile)) # Check if we need to upload if u: ctx.invoke(upload, sync=False) owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True) try: polyaxon_client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) response = polyaxon_client.resume(override_config=content) Printer.print_success("Run was resumed with uid {}".format( response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not resume run `{}`.".format(run_uuid)) sys.exit(1)
def restart(ctx, copy, polyaxonfile, u): """Restart run. Uses /docs/core/cli/#caching Examples: \b $ polyaxon run --uid=8aac02e3a62a4f0aaa257c59da5eab80 restart """ content = None if polyaxonfile: content = "{}".format(ConfigSpec.read_from(polyaxonfile)) # Check if we need to upload if u: ctx.invoke(upload, sync=False) owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True, ) try: polyaxon_client = RunClient(owner=owner, project=project_name, run_uuid=run_uuid) response = polyaxon_client.restart(override_config=content, copy=copy) Printer.print_success("Run was {} with uid {}".format( "copied" if copy else "restarted", response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error( e, message="Could not restart run `{}`.".format(run_uuid)) sys.exit(1)
def file(file_context, filepath, copy_path, track): """Create auth context.""" from polyaxon.init.file import create_file_lineage from polyaxon.utils.hashing import hash_value try: file_context = V1FileType.from_dict(ConfigSpec.read_from(file_context)) except (PolyaxonSchemaError, ValidationError) as e: Printer.print_error("received a non valid file context.") Printer.print_error("Error message: {}.".format(e)) sys.exit(1) filepath = os.path.join(filepath, file_context.filename) check_or_create_path(filepath, is_dir=False) # Clean any previous file on that path if os.path.exists(filepath): os.remove(filepath) with open(filepath, "w") as generated_file: generated_file.write(file_context.content) if file_context.chmod: subprocess.check_call(["chmod", file_context.chmod, filepath]) if copy_path: filepath = copy_file(filepath, copy_path) if track: create_file_lineage( filepath=filepath, summary={"hash": hash_value(file_context.content)}, kind=file_context.kind, ) Printer.print_success("File is initialized, path: `{}`".format(filepath))
def test_reads_yaml_stream(self): stream = """--- x: y 1: 2 """ config = ConfigSpec.read_from(stream) assert config == {"x": "y", 1: 2}
def resume(ctx, polyaxonfile): """Resume run. Uses /docs/core/cli/#caching Examples: \b $ polyaxon ops --uid=8aac02e3a62a4f0aaa257c59da5eab80 resume """ content = None if polyaxonfile: content = "{}".format(ConfigSpec.read_from(polyaxonfile)) owner, project_name, run_uuid = get_project_run_or_local( ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True, ) try: polyaxon_client = RunClient( owner=owner, project=project_name, run_uuid=run_uuid ) response = polyaxon_client.resume(override_config=content) Printer.print_success("Run was resumed with uid {}".format(response.uuid)) except (ApiException, HTTPError) as e: handle_cli_error(e, message="Could not resume run `{}`.".format(run_uuid)) sys.exit(1)
def get_config_from_env(cls) -> AccessTokenConfig: tmp_path = os.path.join(CONTEXT_TMP_POLYAXON_PATH, cls.CONFIG_FILE_NAME) user_path = os.path.join(CONTEXT_USER_POLYAXON_PATH, cls.CONFIG_FILE_NAME) auth_config = ConfigManager.read_configs([ os.environ, ConfigSpec(tmp_path, config_type=".json", check_if_exists=False), ConfigSpec(user_path, config_type=".json", check_if_exists=False), ConfigSpec(CONTEXT_MOUNT_AUTH, config_type=".json", check_if_exists=False), { "dummy": "dummy" }, ]) return AccessTokenConfig.from_dict(auth_config.data)
def generate(polyaxonfile, python_module, build_context, destination, copy_path, params, track): """Generate a dockerfile given the polyaxonfile.""" from polyaxon.init.dockerfile import create_dockerfile_lineage from polyaxon.utils.hashing import hash_value if all([polyaxonfile, build_context]): Printer.print_error( "Only a polyaxonfile or a build context option is required.") sys.exit(1) if build_context: try: build_context = [ V1DockerfileType.from_dict(ConfigSpec.read_from(build_context)) ] except (PolyaxonSchemaError, ValidationError) as e: Printer.print_error("received a non valid build context.") Printer.print_error("Error message: {}.".format(e)) sys.exit(1) else: specification = check_polyaxonfile( polyaxonfile=polyaxonfile, python_module=python_module, params=params, verbose=False, ) try: compiled_operation = OperationSpecification.compile_operation( specification) compiled_operation.apply_params(params=specification.config.params) compiled_operation = CompiledOperationSpecification.apply_operation_contexts( compiled_operation) except PolyaxonSchemaError: Printer.print_error( "Could not run this polyaxonfile locally, " "a context is required to resolve it dependencies.") sys.exit(1) build_context = compiled_operation.init_dockerfiles for init_dockerfile in build_context: generator = DockerFileGenerator(build_context=init_dockerfile, destination=destination or ".") generator.create() Printer.print_success("Dockerfile was generated, path: `{}`".format( generator.dockerfile_path)) dockerfile_path = generator.dockerfile_path if copy_path: dockerfile_path = copy_file(dockerfile_path, copy_path) if track: hash_content = hash_value(init_dockerfile.to_dict()) create_dockerfile_lineage(dockerfile_path, summary={"hash": hash_content})
def test_reads_non_existing_file(self): # Raises by default with self.assertRaises(PolyaxonSchemaError): reader.read("tests/fixtures/parsing/no_file.yml") with self.assertRaises(PolyaxonSchemaError): reader.read("tests/fixtures/parsing/no_file.json") with self.assertRaises(PolyaxonSchemaError): reader.read(ConfigSpec("tests/fixtures/parsing/no_file")) with self.assertRaises(PolyaxonSchemaError): reader.read(ConfigSpec("tests/fixtures/parsing/no_file.yml")) with self.assertRaises(PolyaxonSchemaError): reader.read(ConfigSpec("tests/fixtures/parsing/no_file.json")) # Does not raise if set to ignore assert (reader.read( ConfigSpec("tests/fixtures/parsing/no_file", check_if_exists=False)) == {}) assert (reader.read( ConfigSpec("tests/fixtures/parsing/no_file.yml", check_if_exists=False)) == {}) assert (reader.read( ConfigSpec("tests/fixtures/parsing/no_file.json", check_if_exists=False)) == {})
def read(cls, values): if isinstance(values, cls.CONFIG): return values values = to_list(values) data = ConfigSpec.read_from([{"kind": cls._SPEC_KIND}] + values) try: config = cls.CONFIG.from_dict(copy.deepcopy(data)) except TypeError as e: raise ValidationError( "Received a non valid config `{}`: `{}`".format( cls._SPEC_KIND, e)) cls.check_data(data) return config
def read(cls, values): if isinstance(values, cls.CONFIG): return values if not isinstance(values, Mapping) or Sections.KIND not in values: values = to_list(values) values = ConfigSpec.read_from([{Sections.KIND: cls._SPEC_KIND}] + values) cls.check_data(values) try: config = cls.CONFIG.from_dict(copy.deepcopy(values)) except TypeError as e: raise ValidationError( "Received a non valid config `{}`: `{}`".format(cls._SPEC_KIND, e) ) return config
def generate(polyaxonfile, python_module, build_context, destination, copy_path, params): """Generate a dockerfile given the polyaxonfile.""" if all([polyaxonfile, build_context]): Printer.print_error( "Only a polyaxonfile or a build context option is required.") sys.exit(1) if build_context: try: build_context = [ V1DockerfileType.from_dict(ConfigSpec.read_from(build_context)) ] except (PolyaxonSchemaError, ValidationError) as e: Printer.print_error("received a non valid build context.") Printer.print_error("Error message: {}.".format(e)) sys.exit(1) else: specification = check_polyaxonfile( polyaxonfile=polyaxonfile, python_module=python_module, params=params, log=False, ) try: compiled_operation = specification.compile_operation() compiled_operation.apply_params(params=specification.config.params) compiled_operation = CompiledOperationSpecification.apply_context( compiled_operation) except PolyaxonSchemaError: Printer.print_error( "Could not run this polyaxonfile locally, " "a context is required to resolve it dependencies.") sys.exit(1) build_context = compiled_operation.init_dockerfiles for init_dockerfile in build_context: generator = DockerFileGenerator(build_context=init_dockerfile, destination=destination or ".") generator.create() Printer.print_success("Dockerfile was generated, path: `{}`".format( generator.dockerfile_path)) if copy_path: copy_file(generator.dockerfile_path, copy_path)
def check_polyaxonfile( polyaxonfile=None, python_module=None, params=None, profile=None, queue=None, nocache=None, log=True, ): if not any([polyaxonfile, python_module]): polyaxonfile = PolyaxonFile.check_default_path(path=".") if not any([polyaxonfile, python_module]): polyaxonfile = "" polyaxonfile = to_list(polyaxonfile) exists = [os.path.isfile(f) for f in polyaxonfile] parsed_params = None if params: parsed_params = parse_params(params) if not any(exists) and not python_module: Printer.print_error( "Polyaxonfile is not present, " "please run {}".format(constants.INIT_COMMAND) ) sys.exit(1) if python_module: config = ConfigSpec.get_from(python_module) return config.read() try: plx_file = PolyaxonFile(polyaxonfile) plx_file = plx_file.get_op_specification( params=parsed_params, profile=profile, queue=queue, nocache=nocache ) if log: Printer.print_success("Polyaxonfile valid") return plx_file except Exception as e: handle_cli_error(e, message="Polyaxonfile is not valid.") sys.exit(1)
def read(config_values): """Reads an ordered list of configuration values and deep merge the values in reverse order.""" if not config_values: raise PolyaxonSchemaError( "Cannot read config_value: `{}`".format(config_values)) config_values = to_list(config_values) config = {} for config_value in config_values: config_value = ConfigSpec.get_from(value=config_value) config_value.check_type() config_results = config_value.read() if config_results and isinstance(config_results, Mapping): config = deep_update(config, config_results) elif config_value.check_if_exists: raise PolyaxonSchemaError( "Cannot read config_value: `{}`".format(config_value)) return config
def read_configs(cls, config_values): # pylint:disable=redefined-outer-name config = ConfigSpec.read_from(config_values) # pylint:disable=redefined-outer-name return cls(**config) if config else None
def check_polyaxonfile( polyaxonfile: str = None, python_module: str = None, url: str = None, hub: str = None, params=None, profile=None, queue=None, nocache=None, log=True, is_cli: bool = True, to_op: bool = True, ): if sum([1 for i in [polyaxonfile, python_module, url, hub] if i]) > 1: message = ( "You can only use one and only one option: " "hub, url, module, or path ro polyaxonfile.".format(hub) ) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) if not any([polyaxonfile, python_module, url, hub]): polyaxonfile = check_default_path(path=".") if not any([polyaxonfile, python_module, url, hub]): polyaxonfile = "" if hub and not to_op: message = "Something went wrong, calling hub component `{}` without operation.".format(hub) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) polyaxonfile = to_list(polyaxonfile, check_none=True) parsed_params = None if params: parsed_params = parse_params(params, is_cli=is_cli) if not any([os.path.isfile(f) for f in polyaxonfile]) and not any( [python_module, url, hub] ): message = ( "Please pass a valid polyaxonfile, a python module, url, or component name" ) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) try: plx_file = None if not hub: if python_module: plx_file = ConfigSpec.get_from(python_module, config_type=".py").read() elif url: plx_file = ConfigSpec.get_from(url, "url").read() else: plx_file = ConfigSpec.read_from(polyaxonfile) plx_file = get_specification(data=plx_file) if plx_file.kind == kinds.OPERATION: plx_file = collect_references(plx_file) if to_op or hub: plx_file = get_op_specification( hub=hub, config=plx_file, params=parsed_params, profile=profile, queue=queue, nocache=nocache, ) if log and not is_cli: Printer.print_success("Polyaxonfile valid") return plx_file except Exception as e: message = "Polyaxonfile is not valid." if is_cli: handle_cli_error(e, message=message, sys_exit=True) else: raise PolyaxonfileError(message) from e
def test_reads_non_valid_yaml_stream(self): stream = ";sdfsd;sdff" with self.assertRaises(PolyaxonSchemaError): ConfigSpec.read_from(stream)
def read(cls, values, unknown=None, config_type=None): values = ConfigSpec.read_from(values, config_type=config_type) return cls.from_dict(values, unknown=unknown)
def test_reads_json_stream(self): stream = """--- {x: y, 1: 2} """ config = ConfigSpec.read_from(stream) assert config is not None
def check_polyaxonfile( polyaxonfile: str = None, python_module: str = None, url: str = None, hub: str = None, params: Dict = None, presets: List[str] = None, queue: str = None, nocache: bool = None, cache: bool = None, verbose: bool = True, is_cli: bool = True, to_op: bool = True, validate_params: bool = True, eager: bool = False, git_init: V1Init = None, ignore_template: bool = False, ): if sum([1 for i in [python_module, url, hub] if i]) > 1: message = ("You can only use one and only one option: " "hub, url, or a python module.".format(hub)) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) if not any([polyaxonfile, python_module, url, hub]): polyaxonfile = check_default_path(path=".") if not any([polyaxonfile, python_module, url, hub]): message = ( "Something went wrong, `check_polyaxonfile` was called without a polyaxonfile, " "a hub component reference, a url or a python module.") if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) if hub and not to_op: message = "Something went wrong, calling hub component `{}` without operation.".format( hub) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) polyaxonfile = to_list(polyaxonfile, check_none=True) parsed_params = None if params: parsed_params = parse_params(params, is_cli=is_cli) if not any([os.path.isfile(f) for f in polyaxonfile]) and not any([python_module, url, hub]): message = "Please pass a valid polyaxonfile, a python module, a url, or a component name" if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) try: path_context = None if python_module: path_context = python_module plx_file = (ConfigSpec.get_from(python_module, config_type=".py").read().to_dict( include_kind=True, include_version=True)) elif url: plx_file = ConfigSpec.get_from(url, "url").read() elif hub: plx_file = ConfigSpec.get_from(hub, "hub").read() else: path_context = polyaxonfile.pop(0) plx_file = ConfigSpec.read_from(path_context) plx_file = get_specification(data=plx_file) if plx_file.kind == kinds.OPERATION: plx_file = collect_references(plx_file, path_context) plx_component = plx_file.component else: plx_component = plx_file if plx_component.is_dag_run: collect_dag_components(plx_component.run, path_context) if to_op or hub: plx_file = get_op_specification( hub=hub, config=plx_file, params=parsed_params, presets=presets, queue=queue, nocache=nocache, cache=cache, validate_params=validate_params, preset_files=polyaxonfile, git_init=git_init, ) if verbose and is_cli: Printer.print_success("Polyaxonfile valid") if ignore_template: plx_file.disable_template() if plx_file.is_template(): template_message = "This polyaxonfile was marked as template by the owner:" if plx_file.template.description: template_message += "\ntemplate description: {}".format( plx_file.template.description) if plx_file.template.fields: template_message += "\ntemplate fields that need changes: {}".format( plx_file.template.fields) Printer.print_warning(template_message) if eager: is_supported_in_eager_mode(spec=plx_file) return plx_file except Exception as e: message = "Polyaxonfile is not valid." if is_cli: handle_cli_error(e, message=message, sys_exit=True) else: raise PolyaxonfileError(message) from e
def test_reads_yaml_files(self): config = ConfigSpec.read_from("tests/fixtures/parsing/yaml_file.yml") assert config == {"x": 10, "y": 20, "foo": "bar", "type": "yaml"}
def get_specification(data): if not isinstance(data, Mapping): data = ConfigSpec.read_from(data) kind = BaseSpecification.get_kind(data=data) return SPECIFICATION_BY_KIND[kind].read(data)
def test_reads_json_files(self): config = ConfigSpec.read_from("tests/fixtures/parsing/json_file.json") assert config == {"x": 1, "y": 2, "foo": "bar", "type": "json"}
def test_reads_yaml_files_without_extension(self): config = reader.read( ConfigSpec("tests/fixtures/parsing/yaml_file", config_type=".yml")) assert config == {"x": 10, "y": 20, "foo": "bar", "type": "yaml"}
def test_reads_json_files_without_extension(self): config = reader.read( ConfigSpec("tests/fixtures/parsing/json_file", config_type=".json")) assert config == {"x": 1, "y": 2, "foo": "bar", "type": "json"}