def get_op_specification( self, params=None, profile=None, queue=None, nocache=None ) -> V1Operation: job_data = {"version": self.config.version, "kind": kinds.OPERATION} if params: if not isinstance(params, Mapping): raise PolyaxonfileError( "Params: `{}` must be a valid mapping".format(params) ) job_data["params"] = params if profile: job_data["profile"] = profile if queue: job_data["queue"] = queue if nocache is not None: job_data["cache"] = {"disable": nocache} if self.config.kind == kinds.OPERATION: config = get_specification(data=[self.config.to_dict(), job_data]) else: job_data["component"] = self.config.to_dict() config = get_specification(data=[job_data]) params = copy.deepcopy(config.params) # Sanity check if params were passed run_config = OperationSpecification.compile_operation(config) run_config.validate_params(params=params, is_template=False) if run_config.is_dag_run: CompiledOperationSpecification.apply_context(run_config) return config
def get_op_specification( config: Union[V1Component, V1Operation] = None, hub: str = None, params: Dict = None, profile: str = None, queue: str = None, nocache: bool = None, path_context: str = None, validate_params: bool = True, ) -> V1Operation: job_data = { "version": config.version if config else pkg.SCHEMA_VERSION, "kind": kinds.OPERATION, } if params: if not isinstance(params, Mapping): raise PolyaxonfileError( "Params: `{}` must be a valid mapping".format(params) ) job_data["params"] = params if profile: job_data["profile"] = profile if queue: # Check only get_queue_info(queue) job_data["queue"] = queue if nocache is not None: job_data["cache"] = {"disable": nocache} if config and config.kind == kinds.COMPONENT: job_data["component"] = config.to_dict() config = get_specification(data=[job_data]) elif config and config.kind == kinds.OPERATION: config = get_specification(data=[config.to_dict(), job_data]) elif hub: job_data["hubRef"] = hub config = get_specification(data=[job_data]) if hub and config.hub_ref is None: config.hub_ref = hub hub = config.hub_ref public_hub = config.has_public_hub_reference params = copy.deepcopy(config.params) # Sanity check if params were passed and we are not dealing with a hub component if validate_params and not (hub and not public_hub): run_config = OperationSpecification.compile_operation(config) run_config.validate_params(params=params, is_template=False) if run_config.is_dag_run: run_config.run.set_path_context(path_context) CompiledOperationSpecification.apply_operation_contexts(run_config) return config
def collect_references(config: V1Operation, path_context: str = None): if config.has_component_reference: return config elif config.has_hub_reference: component = ConfigSpec.get_from(config.hub_ref, "hub").read() elif config.has_url_reference: component = ConfigSpec.get_from(config.url_ref, "url").read() elif config.has_path_reference: path_ref = config.path_ref if path_context: path_ref = os.path.join( os.path.dirname(os.path.abspath(path_context)), path_ref) component = ConfigSpec.get_from(path_ref).read() else: raise PolyaxonfileError("Operation found without component") component = get_specification(data=component) if component.kind != kinds.COMPONENT: if config.has_url_reference: ref_type = "Url ref" ref = config.url_ref else: ref_type = "Path ref" ref = config.path_ref raise PolyaxonfileError( "the reference ({}) `{}` is of kind `{}`, it should be a `{}`". format(ref, ref_type, component.kind, kinds.COMPONENT)) config.component = component if component.is_dag_run: component.run.collect_components() return config
def __init__(self, filepaths): filepaths = to_list(filepaths) for filepath in filepaths: if not os.path.isfile(filepath): raise PolyaxonfileError("`{}` must be a valid file".format(filepath)) self._filenames = [os.path.basename(filepath) for filepath in filepaths] self.config = get_specification(data=reader.read(filepaths))
def get_op_specification( config: Union[V1Component, V1Operation] = None, hub=None, params=None, profile=None, queue=None, nocache=None, ) -> V1Operation: job_data = { "version": config.version if config else pkg.SCHEMA_VERSION, "kind": kinds.OPERATION } if params: if not isinstance(params, Mapping): raise PolyaxonfileError( "Params: `{}` must be a valid mapping".format(params)) job_data["params"] = params if profile: job_data["profile"] = profile if queue: job_data["queue"] = queue if nocache is not None: job_data["cache"] = {"disable": nocache} if hub: job_data["hubRef"] = hub config = get_specification(data=[job_data]) elif config.kind == kinds.OPERATION: config = get_specification(data=[config.to_dict(), job_data]) else: job_data["component"] = config.to_dict() config = get_specification(data=[job_data]) params = copy.deepcopy(config.params) # Sanity check if params were passed and we are not dealing with a hub component if not hub: run_config = OperationSpecification.compile_operation(config) run_config.validate_params(params=params, is_template=False) if run_config.is_dag_run: CompiledOperationSpecification.apply_context(run_config) return config
def collect_references(config: V1Operation): if config.has_component_reference or config.has_hub_reference: return config elif config.has_url_reference: component = ConfigSpec.get_from(config.url_ref, "url").read() elif config.has_path_reference: component = ConfigSpec.get_from(config.path_ref).read() else: raise PolyaxonfileError("Operation found without component") component = get_specification(data=component) config.component = component return config
def check_polyaxonfile( polyaxonfile: str = None, python_module: str = None, url: str = None, hub: str = None, params=None, profile=None, queue=None, nocache=None, log=True, is_cli: bool = True, to_op: bool = True, ): if sum([1 for i in [polyaxonfile, python_module, url, hub] if i]) > 1: message = ( "You can only use one and only one option: " "hub, url, module, or path ro polyaxonfile.".format(hub) ) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) if not any([polyaxonfile, python_module, url, hub]): polyaxonfile = check_default_path(path=".") if not any([polyaxonfile, python_module, url, hub]): polyaxonfile = "" if hub and not to_op: message = "Something went wrong, calling hub component `{}` without operation.".format(hub) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) polyaxonfile = to_list(polyaxonfile, check_none=True) parsed_params = None if params: parsed_params = parse_params(params, is_cli=is_cli) if not any([os.path.isfile(f) for f in polyaxonfile]) and not any( [python_module, url, hub] ): message = ( "Please pass a valid polyaxonfile, a python module, url, or component name" ) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) try: plx_file = None if not hub: if python_module: plx_file = ConfigSpec.get_from(python_module, config_type=".py").read() elif url: plx_file = ConfigSpec.get_from(url, "url").read() else: plx_file = ConfigSpec.read_from(polyaxonfile) plx_file = get_specification(data=plx_file) if plx_file.kind == kinds.OPERATION: plx_file = collect_references(plx_file) if to_op or hub: plx_file = get_op_specification( hub=hub, config=plx_file, params=parsed_params, profile=profile, queue=queue, nocache=nocache, ) if log and not is_cli: Printer.print_success("Polyaxonfile valid") return plx_file except Exception as e: message = "Polyaxonfile is not valid." if is_cli: handle_cli_error(e, message=message, sys_exit=True) else: raise PolyaxonfileError(message) from e
def check_polyaxonfile( polyaxonfile: str = None, python_module: str = None, url: str = None, hub: str = None, params: Dict = None, presets: List[str] = None, queue: str = None, nocache: bool = None, cache: bool = None, verbose: bool = True, is_cli: bool = True, to_op: bool = True, validate_params: bool = True, eager: bool = False, git_init: V1Init = None, ignore_template: bool = False, ): if sum([1 for i in [python_module, url, hub] if i]) > 1: message = ("You can only use one and only one option: " "hub, url, or a python module.".format(hub)) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) if not any([polyaxonfile, python_module, url, hub]): polyaxonfile = check_default_path(path=".") if not any([polyaxonfile, python_module, url, hub]): message = ( "Something went wrong, `check_polyaxonfile` was called without a polyaxonfile, " "a hub component reference, a url or a python module.") if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) if hub and not to_op: message = "Something went wrong, calling hub component `{}` without operation.".format( hub) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) polyaxonfile = to_list(polyaxonfile, check_none=True) parsed_params = None if params: parsed_params = parse_params(params, is_cli=is_cli) if not any([os.path.isfile(f) for f in polyaxonfile]) and not any([python_module, url, hub]): message = "Please pass a valid polyaxonfile, a python module, a url, or a component name" if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) try: path_context = None if python_module: path_context = python_module plx_file = (ConfigSpec.get_from(python_module, config_type=".py").read().to_dict( include_kind=True, include_version=True)) elif url: plx_file = ConfigSpec.get_from(url, "url").read() elif hub: plx_file = ConfigSpec.get_from(hub, "hub").read() else: path_context = polyaxonfile.pop(0) plx_file = ConfigSpec.read_from(path_context) plx_file = get_specification(data=plx_file) if plx_file.kind == kinds.OPERATION: plx_file = collect_references(plx_file, path_context) plx_component = plx_file.component else: plx_component = plx_file if plx_component.is_dag_run: collect_dag_components(plx_component.run, path_context) if to_op or hub: plx_file = get_op_specification( hub=hub, config=plx_file, params=parsed_params, presets=presets, queue=queue, nocache=nocache, cache=cache, validate_params=validate_params, preset_files=polyaxonfile, git_init=git_init, ) if verbose and is_cli: Printer.print_success("Polyaxonfile valid") if ignore_template: plx_file.disable_template() if plx_file.is_template(): template_message = "This polyaxonfile was marked as template by the owner:" if plx_file.template.description: template_message += "\ntemplate description: {}".format( plx_file.template.description) if plx_file.template.fields: template_message += "\ntemplate fields that need changes: {}".format( plx_file.template.fields) Printer.print_warning(template_message) if eager: is_supported_in_eager_mode(spec=plx_file) return plx_file except Exception as e: message = "Polyaxonfile is not valid." if is_cli: handle_cli_error(e, message=message, sys_exit=True) else: raise PolyaxonfileError(message) from e
def get_op_specification( config: Union[V1Component, V1Operation] = None, hub: str = None, params: Dict = None, presets: List[str] = None, queue: str = None, nocache: bool = None, cache: bool = None, validate_params: bool = True, preset_files: List[str] = None, git_init: V1Init = None, ) -> V1Operation: if cache and nocache: raise PolyaxonfileError("Received both cache and nocache") job_data = { "version": config.version if config else pkg.SCHEMA_VERSION, "kind": kinds.OPERATION, } if params: if not isinstance(params, Mapping): raise PolyaxonfileError( "Params: `{}` must be a valid mapping".format(params)) job_data["params"] = params if presets: job_data["presets"] = presets if queue: # Check only get_queue_info(queue) job_data["queue"] = queue if cache: job_data["cache"] = {"disable": False} if nocache: job_data["cache"] = {"disable": True} if config and config.kind == kinds.COMPONENT: job_data["component"] = config.to_dict() config = get_specification(data=[job_data]) elif config and config.kind == kinds.OPERATION: config = get_specification(data=[config.to_dict(), job_data]) elif hub: job_data["hubRef"] = hub config = get_specification(data=[job_data]) if hub and config.hub_ref is None: config.hub_ref = hub # Check if there's presets for preset_plx_file in preset_files: preset_plx_file = OperationSpecification.read(preset_plx_file, is_preset=True) config = config.patch(preset_plx_file, strategy=preset_plx_file.patch_strategy) # Turn git_init to a pre_merge preset if git_init: git_preset = V1Operation(run_patch={"init": [git_init.to_dict()]}, is_preset=True) config = config.patch(git_preset, strategy=V1PatchStrategy.PRE_MERGE) # Sanity check if params were passed and we are not dealing with a hub component params = copy.deepcopy(config.params) if validate_params: # Avoid in-place patch run_config = get_specification(config.to_dict()) run_config = OperationSpecification.compile_operation(run_config) run_config.validate_params(params=params, is_template=False) if run_config.is_dag_run: CompiledOperationSpecification.apply_operation_contexts(run_config) return config
def check_polyaxonfile( polyaxonfile: str = None, python_module: str = None, url: str = None, hub: str = None, params: Dict = None, profile: str = None, queue: str = None, nocache: bool = None, verbose: bool = True, eager_hub: bool = True, is_cli: bool = True, to_op: bool = True, validate_params: bool = True, eager: bool = False, ): if sum([1 for i in [polyaxonfile, python_module, url, hub] if i]) > 1: message = ("You can only use one and only one option: " "hub, url, module, or path or polyaxonfile.".format(hub)) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) if not any([polyaxonfile, python_module, url, hub]): polyaxonfile = check_default_path(path=".") if not any([polyaxonfile, python_module, url, hub]): polyaxonfile = "" if hub and not to_op: message = "Something went wrong, calling hub component `{}` without operation.".format( hub) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) polyaxonfile = to_list(polyaxonfile, check_none=True) parsed_params = None if params: parsed_params = parse_params(params, is_cli=is_cli) if not any([os.path.isfile(f) for f in polyaxonfile]) and not any([python_module, url, hub]): message = ( "Please pass a valid polyaxonfile, a python module, url, or component name" ) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) try: plx_file = None path_context = None public_hub = hub and "/" not in hub if not hub or (public_hub and eager_hub): if python_module: path_context = python_module plx_file = (ConfigSpec.get_from( python_module, config_type=".py").read().to_dict(include_kind=True, include_version=True)) elif url: plx_file = ConfigSpec.get_from(url, "url").read() elif hub: plx_file = ConfigSpec.get_from(hub, "hub").read() else: path_context = polyaxonfile[0] plx_file = ConfigSpec.read_from(polyaxonfile) plx_file = get_specification(data=plx_file) if plx_file.kind == kinds.OPERATION: plx_file = collect_references(plx_file, path_context) if to_op or hub: plx_file = get_op_specification( hub=hub, config=plx_file, params=parsed_params, profile=profile, queue=queue, nocache=nocache, path_context=path_context, validate_params=validate_params, ) if verbose and is_cli: Printer.print_success("Polyaxonfile valid") if eager: is_supported_in_eager_mode(spec=plx_file) return plx_file except Exception as e: message = "Polyaxonfile is not valid." if is_cli: handle_cli_error(e, message=message, sys_exit=True) else: raise PolyaxonfileError(message) from e