def collect_references(config: V1Operation, path_context: str = None): if config.has_component_reference: return config elif config.has_hub_reference: component = ConfigSpec.get_from(config.hub_ref, "hub").read() elif config.has_url_reference: component = ConfigSpec.get_from(config.url_ref, "url").read() elif config.has_path_reference: path_ref = config.path_ref if path_context: path_ref = os.path.join( os.path.dirname(os.path.abspath(path_context)), path_ref) component = ConfigSpec.get_from(path_ref).read() else: raise PolyaxonfileError("Operation found without component") component = get_specification(data=component) if component.kind != kinds.COMPONENT: if config.has_url_reference: ref_type = "Url ref" ref = config.url_ref else: ref_type = "Path ref" ref = config.path_ref raise PolyaxonfileError( "the reference ({}) `{}` is of kind `{}`, it should be a `{}`". format(ref, ref_type, component.kind, kinds.COMPONENT)) config.component = component if component.is_dag_run: component.run.collect_components() return config
def collect_references(config: V1Operation): if config.has_component_reference or config.has_hub_reference: return config elif config.has_url_reference: component = ConfigSpec.get_from(config.url_ref, "url").read() elif config.has_path_reference: component = ConfigSpec.get_from(config.path_ref).read() else: raise PolyaxonfileError("Operation found without component") component = get_specification(data=component) config.component = component return config
def get(ctx, save, filename): """Get a component info by component_name, or owner/component_name. Uses /docs/core/cli/#caching Examples: To get a component by name \b $ polyaxon hub get component_name To get a component by owner/name \b $ polyaxon hub get owner/component_name """ name = ctx.obj.get("name") if not name: Printer.print_error("Please provide a valid component name!") sys.exit(0) try: polyaxonfile = ConfigSpec.get_from(name, "hub").read() except Exception as e: handle_cli_error(e, message="Could not get component `{}`.".format(name)) sys.exit(1) specification = get_specification(data=polyaxonfile) polyaxonfile = yaml.dump(polyaxonfile) get_component_details(polyaxonfile=polyaxonfile, specification=specification) if save: filename = filename or "polyaxonfile.yaml" with open(filename, "w") as env_file: env_file.write(polyaxonfile)
def check_polyaxonfile( polyaxonfile=None, python_module=None, params=None, profile=None, queue=None, nocache=None, log=True, ): if not any([polyaxonfile, python_module]): polyaxonfile = PolyaxonFile.check_default_path(path=".") if not any([polyaxonfile, python_module]): polyaxonfile = "" polyaxonfile = to_list(polyaxonfile) exists = [os.path.isfile(f) for f in polyaxonfile] parsed_params = None if params: parsed_params = parse_params(params) if not any(exists) and not python_module: Printer.print_error( "Polyaxonfile is not present, " "please run {}".format(constants.INIT_COMMAND) ) sys.exit(1) if python_module: config = ConfigSpec.get_from(python_module) return config.read() try: plx_file = PolyaxonFile(polyaxonfile) plx_file = plx_file.get_op_specification( params=parsed_params, profile=profile, queue=queue, nocache=nocache ) if log: Printer.print_success("Polyaxonfile valid") return plx_file except Exception as e: handle_cli_error(e, message="Polyaxonfile is not valid.") sys.exit(1)
def read(config_values): """Reads an ordered list of configuration values and deep merge the values in reverse order.""" if not config_values: raise PolyaxonSchemaError( "Cannot read config_value: `{}`".format(config_values)) config_values = to_list(config_values) config = {} for config_value in config_values: config_value = ConfigSpec.get_from(value=config_value) config_value.check_type() config_results = config_value.read() if config_results and isinstance(config_results, Mapping): config = deep_update(config, config_results) elif config_value.check_if_exists: raise PolyaxonSchemaError( "Cannot read config_value: `{}`".format(config_value)) return config
def check_polyaxonfile( polyaxonfile: str = None, python_module: str = None, url: str = None, hub: str = None, params=None, profile=None, queue=None, nocache=None, log=True, is_cli: bool = True, to_op: bool = True, ): if sum([1 for i in [polyaxonfile, python_module, url, hub] if i]) > 1: message = ( "You can only use one and only one option: " "hub, url, module, or path ro polyaxonfile.".format(hub) ) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) if not any([polyaxonfile, python_module, url, hub]): polyaxonfile = check_default_path(path=".") if not any([polyaxonfile, python_module, url, hub]): polyaxonfile = "" if hub and not to_op: message = "Something went wrong, calling hub component `{}` without operation.".format(hub) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) polyaxonfile = to_list(polyaxonfile, check_none=True) parsed_params = None if params: parsed_params = parse_params(params, is_cli=is_cli) if not any([os.path.isfile(f) for f in polyaxonfile]) and not any( [python_module, url, hub] ): message = ( "Please pass a valid polyaxonfile, a python module, url, or component name" ) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) try: plx_file = None if not hub: if python_module: plx_file = ConfigSpec.get_from(python_module, config_type=".py").read() elif url: plx_file = ConfigSpec.get_from(url, "url").read() else: plx_file = ConfigSpec.read_from(polyaxonfile) plx_file = get_specification(data=plx_file) if plx_file.kind == kinds.OPERATION: plx_file = collect_references(plx_file) if to_op or hub: plx_file = get_op_specification( hub=hub, config=plx_file, params=parsed_params, profile=profile, queue=queue, nocache=nocache, ) if log and not is_cli: Printer.print_success("Polyaxonfile valid") return plx_file except Exception as e: message = "Polyaxonfile is not valid." if is_cli: handle_cli_error(e, message=message, sys_exit=True) else: raise PolyaxonfileError(message) from e
def check_polyaxonfile( polyaxonfile: str = None, python_module: str = None, url: str = None, hub: str = None, params: Dict = None, presets: List[str] = None, queue: str = None, nocache: bool = None, cache: bool = None, verbose: bool = True, is_cli: bool = True, to_op: bool = True, validate_params: bool = True, eager: bool = False, git_init: V1Init = None, ignore_template: bool = False, ): if sum([1 for i in [python_module, url, hub] if i]) > 1: message = ("You can only use one and only one option: " "hub, url, or a python module.".format(hub)) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) if not any([polyaxonfile, python_module, url, hub]): polyaxonfile = check_default_path(path=".") if not any([polyaxonfile, python_module, url, hub]): message = ( "Something went wrong, `check_polyaxonfile` was called without a polyaxonfile, " "a hub component reference, a url or a python module.") if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) if hub and not to_op: message = "Something went wrong, calling hub component `{}` without operation.".format( hub) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) polyaxonfile = to_list(polyaxonfile, check_none=True) parsed_params = None if params: parsed_params = parse_params(params, is_cli=is_cli) if not any([os.path.isfile(f) for f in polyaxonfile]) and not any([python_module, url, hub]): message = "Please pass a valid polyaxonfile, a python module, a url, or a component name" if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) try: path_context = None if python_module: path_context = python_module plx_file = (ConfigSpec.get_from(python_module, config_type=".py").read().to_dict( include_kind=True, include_version=True)) elif url: plx_file = ConfigSpec.get_from(url, "url").read() elif hub: plx_file = ConfigSpec.get_from(hub, "hub").read() else: path_context = polyaxonfile.pop(0) plx_file = ConfigSpec.read_from(path_context) plx_file = get_specification(data=plx_file) if plx_file.kind == kinds.OPERATION: plx_file = collect_references(plx_file, path_context) plx_component = plx_file.component else: plx_component = plx_file if plx_component.is_dag_run: collect_dag_components(plx_component.run, path_context) if to_op or hub: plx_file = get_op_specification( hub=hub, config=plx_file, params=parsed_params, presets=presets, queue=queue, nocache=nocache, cache=cache, validate_params=validate_params, preset_files=polyaxonfile, git_init=git_init, ) if verbose and is_cli: Printer.print_success("Polyaxonfile valid") if ignore_template: plx_file.disable_template() if plx_file.is_template(): template_message = "This polyaxonfile was marked as template by the owner:" if plx_file.template.description: template_message += "\ntemplate description: {}".format( plx_file.template.description) if plx_file.template.fields: template_message += "\ntemplate fields that need changes: {}".format( plx_file.template.fields) Printer.print_warning(template_message) if eager: is_supported_in_eager_mode(spec=plx_file) return plx_file except Exception as e: message = "Polyaxonfile is not valid." if is_cli: handle_cli_error(e, message=message, sys_exit=True) else: raise PolyaxonfileError(message) from e
def check_polyaxonfile( polyaxonfile: str = None, python_module: str = None, url: str = None, hub: str = None, params: Dict = None, profile: str = None, queue: str = None, nocache: bool = None, verbose: bool = True, eager_hub: bool = True, is_cli: bool = True, to_op: bool = True, validate_params: bool = True, eager: bool = False, ): if sum([1 for i in [polyaxonfile, python_module, url, hub] if i]) > 1: message = ("You can only use one and only one option: " "hub, url, module, or path or polyaxonfile.".format(hub)) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) if not any([polyaxonfile, python_module, url, hub]): polyaxonfile = check_default_path(path=".") if not any([polyaxonfile, python_module, url, hub]): polyaxonfile = "" if hub and not to_op: message = "Something went wrong, calling hub component `{}` without operation.".format( hub) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) polyaxonfile = to_list(polyaxonfile, check_none=True) parsed_params = None if params: parsed_params = parse_params(params, is_cli=is_cli) if not any([os.path.isfile(f) for f in polyaxonfile]) and not any([python_module, url, hub]): message = ( "Please pass a valid polyaxonfile, a python module, url, or component name" ) if is_cli: Printer.print_error(message, sys_exit=True) else: raise PolyaxonfileError(message) try: plx_file = None path_context = None public_hub = hub and "/" not in hub if not hub or (public_hub and eager_hub): if python_module: path_context = python_module plx_file = (ConfigSpec.get_from( python_module, config_type=".py").read().to_dict(include_kind=True, include_version=True)) elif url: plx_file = ConfigSpec.get_from(url, "url").read() elif hub: plx_file = ConfigSpec.get_from(hub, "hub").read() else: path_context = polyaxonfile[0] plx_file = ConfigSpec.read_from(polyaxonfile) plx_file = get_specification(data=plx_file) if plx_file.kind == kinds.OPERATION: plx_file = collect_references(plx_file, path_context) if to_op or hub: plx_file = get_op_specification( hub=hub, config=plx_file, params=parsed_params, profile=profile, queue=queue, nocache=nocache, path_context=path_context, validate_params=validate_params, ) if verbose and is_cli: Printer.print_success("Polyaxonfile valid") if eager: is_supported_in_eager_mode(spec=plx_file) return plx_file except Exception as e: message = "Polyaxonfile is not valid." if is_cli: handle_cli_error(e, message=message, sys_exit=True) else: raise PolyaxonfileError(message) from e