def get_versioned_entity_info( entity: str, entity_name: str, default_owner: str ) -> Tuple[str, str, str]: if not entity: raise PolyaxonSchemaError( "Received an invalid {} reference: `{}`".format(entity_name, entity) ) entity_values = entity.split(":") if len(entity_values) > 2: raise PolyaxonSchemaError( "Received an invalid {} reference: `{}`".format(entity_name, entity) ) if len(entity_values) == 2: entity_name, version = entity_values else: entity_name, version = entity_values[0], "latest" version = version or "latest" parts = entity_name.replace(".", "/").split("/") owner = default_owner if not parts or len(parts) > 2: raise PolyaxonSchemaError( "Received an invalid {} reference: `{}`".format(entity_name, entity) ) if len(parts) == 2: owner, entity_namespace = parts else: entity_namespace = entity_name return owner, entity_namespace, version
def read(self): if isinstance(self.value, Mapping): return self.value # try a python file if isinstance(self.value, str) and ( self.config_type == ".py" or ".py" in self.value ): _f_path, _f_module = _get_python_file_def(self.value) if _f_path and _f_module: return _read_from_python(_f_path, _f_module) if os.path.isfile(self.value): return _read_from_file(self.value, self.config_type) # try reading a stream of yaml or json if not self.config_type or self.config_type in (".json", ".yml", ".yaml"): try: return _read_from_stream(self.value) except (ScannerError, ParserError): raise PolyaxonSchemaError( "Received an invalid yaml stream: `{}`".format(self.value) ) if self.config_type == "url": return _read_from_url(self.value) if self.config_type == "hub": if os.environ.get(POLYAXON_KEYS_USE_GIT_REGISTRY, False): return _read_from_public_hub(self.value) return _read_from_polyaxon_hub(self.value) raise PolyaxonSchemaError( "Received an invalid configuration: `{}`".format(self.value) )
def read_from(cls, config_values, config_type=None): """ Reads an ordered list of configuration values and deep merge the values in reverse order. """ if not config_values: raise PolyaxonSchemaError( "Cannot read config_value: `{}`".format(config_values) ) config_values = to_list(config_values, check_none=True) config = {} for config_value in config_values: config_value = ConfigSpec.get_from( value=config_value, config_type=config_type ) config_value.check_type() config_results = config_value.read() if config_results and isinstance(config_results, Mapping): config = deep_update(config, config_results) elif config_value.check_if_exists: raise PolyaxonSchemaError( "Cannot read config_value: `{}`".format(config_value.value) ) return config
def parse_uri_spec(uri_spec) -> V1UriType: if isinstance(uri_spec, V1UriType): return uri_spec if isinstance(uri_spec, Mapping): return V1UriType.from_dict(uri_spec) try: auth_spec = convert_to_dict(uri_spec, "") return V1UriType.from_dict(auth_spec) except (PolyaxonSchemaError, JSONDecodeError): pass parts = uri_spec.split("@") if len(parts) != 2: raise PolyaxonSchemaError( "Received invalid uri_spec `{}`. " "The uri must be in the format `user:pass@host`".format(uri_spec)) user_pass, host = parts user_pass = user_pass.split(":") if len(user_pass) != 2: raise PolyaxonSchemaError( "Received invalid uri_spec `{}`. `user:host` is not conform." "The uri must be in the format `user:pass@host`".format(uri_spec)) return V1UriType(user=user_pass[0], password=user_pass[1], host=host)
def _get_typed_list_value(key, value, target_type, type_convert, is_optional=False, default=None, options=None): """ Return the value corresponding to the key converted first to list than each element to the given type. Args: key: the dict key. target_type: The type we expect the variable or key to be in. type_convert: A lambda expression that converts the key to the desired type. is_optional: To raise an error if key was not found. default: default value if is_optional is True. options: list/tuple if provided, the value must be one of these values. """ value = _get_typed_value( key=key, value=value, target_type=list, type_convert=json.loads, is_optional=is_optional, default=default, options=options, ) if not value: return default raise_type = "dict" if target_type == Mapping else target_type if not isinstance(value, list): raise PolyaxonSchemaError("Cannot convert value `{}` (key: `{}`) " "to `{}`".format(value, key, raise_type)) # If we are here the value must be a list result = [] for v in value: if isinstance(v, six.string_types): try: result.append(type_convert(v)) except ValueError: raise PolyaxonSchemaError( "Cannot convert value `{}` (found in list key: `{}`) " "to `{}`".format(v, key, raise_type)) elif isinstance(v, target_type): result.append(v) else: raise PolyaxonSchemaError( "Cannot convert value `{}` (found in list key: `{}`) " "to `{}`".format(v, key, raise_type)) return result
def _read_from_json(f_path, is_stream=False): if is_stream: try: return json.loads(f_path) except ValueError as e: raise PolyaxonSchemaError(e) try: return json.loads(open(f_path).read()) except ValueError as e: raise PolyaxonSchemaError(e)
def _read_from_json(f_path, is_stream=False): if is_stream: try: return json.loads(f_path) except ValueError as e: raise PolyaxonSchemaError("Json error: %s" % e) from e try: return json.loads(open(f_path).read()) except ValueError as e: raise PolyaxonSchemaError("Received non valid json: `%s`.\n" "Json error %s" % (f_path, e)) from e
def _get_typed_value( key, value, target_type, type_convert, is_optional=False, default=None, options=None, base_types=None, ): """ Return the value corresponding to the key converted to the given type. Args: key: the dict key. target_type: The type we expect the variable or key to be in. type_convert: A lambda expression that converts the key to the desired type. is_optional: To raise an error if key was not found. default: default value if is_optional is True. options: list/tuple if provided, the value must be one of these values. base_types: the base types to check for conversion Returns: The corresponding value of the key converted. """ if value is None or value == NO_VALUE_FOUND: if not is_optional: raise PolyaxonSchemaError( "No value was provided for the non optional key `{}`.".format(key) ) return default base_types = base_types or str if isinstance(value, base_types): try: # _add_key(key, is_secret=is_secret, is_local=is_local) _check_options(key=key, value=value, options=options) return type_convert(value) except (ValueError, ValidationError): raise PolyaxonSchemaError( "Cannot convert value `{}` (key: `{}`) " "to `{}`".format(value, key, target_type) ) if isinstance(value, target_type): # _add_key(key, is_secret=is_secret, is_local=is_local) _check_options(key=key, value=value, options=options) return value raise PolyaxonSchemaError( "Cannot convert value `{}` (key: `{}`) " "to `{}`".format(value, key, target_type) )
def _get_python_file_def(f_path): results = f_path.split(":") if len(results) != 2 or not results[1]: return None, None _f_path = results[0].strip("") _module_name = results[1].strip("") if not os.path.exists(_f_path): raise PolyaxonSchemaError( "Received non existing python file: `{}`".format(f_path)) if not _module_name: raise PolyaxonSchemaError( "Received an invalid python module: `{}`".format(f_path)) return _f_path, _module_name
def is_supported_in_eager_mode(spec: Union[V1Operation, V1CompiledOperation]): if not spec.matrix: if spec.component and spec.component.run: raise PolyaxonSchemaError("This operation with runtime `{}` " "is not supported in eager mode".format( spec.component.run.kind)) else: raise PolyaxonSchemaError( "Received a bad configuration, eager mode not supported") if spec.get_matrix_kind() not in V1MatrixKind.eager_values: raise PolyaxonSchemaError( "This operation is defining a matrix kind `{}` " "which is not supported in eager mode".format( spec.get_matrix_kind()))
def parse_uri_spec(uri_spec): parts = uri_spec.split("@") if len(parts) != 2: raise PolyaxonSchemaError( "Received invalid uri_spec `{}`. " "The uri must be in the format `user:pass@host`".format(uri_spec)) user_pass, host = parts user_pass = user_pass.split(":") if len(user_pass) != 2: raise PolyaxonSchemaError( "Received invalid uri_spec `{}`. `user:host` is not conform." "The uri must be in the format `user:pass@host`".format(uri_spec)) return UriSpec(user=user_pass[0], password=user_pass[1], host=host)
def to_light_dict( self, humanize_values=False, include_attrs=None, exclude_attrs=None, unknown=None, dump=False, ): unknown = unknown or self.UNKNOWN_BEHAVIOUR obj_dict = self.to_dict(humanize_values=humanize_values, unknown=unknown) if all([include_attrs, exclude_attrs]): raise PolyaxonSchemaError( "Only one value `include_attrs` or `exclude_attrs` is allowed." ) if not any([include_attrs, exclude_attrs]): # Use Default setup attrs include_attrs = self.DEFAULT_INCLUDE_ATTRIBUTES exclude_attrs = self.DEFAULT_EXCLUDE_ATTRIBUTES if include_attrs: exclude_attrs = set(obj_dict.keys()) - set(include_attrs) for attr in exclude_attrs: obj_dict.pop(attr, None) if dump: return ujson.dumps(obj_dict) return obj_dict
def parse_gcs_path(gcs_path): """ Parses and validates a google cloud storage url. Returns: tuple(bucket_name, blob). """ parsed_url = urllib.parse.urlparse(gcs_path) if not parsed_url.netloc: raise PolyaxonSchemaError( "Received an invalid GCS url `{}`".format(gcs_path)) if parsed_url.scheme != "gs": raise PolyaxonSchemaError( "Received an invalid url GCS `{}`".format(gcs_path)) blob = parsed_url.path.lstrip("/") return GCSSpec(parsed_url.netloc, blob)
def check_type(self): type_check = self.config_type is None and not isinstance( self.value, (Mapping, six.string_types)) if type_check: raise PolyaxonSchemaError( "Expects Mapping, string, or list of Mapping/string instances, " "received {} instead".format(type(self.value)))
def get_entity_info(entity): if not entity: raise PolyaxonSchemaError( "Received an invalid entity reference: `{}`".format(entity)) parts = entity.replace(".", "/").split("/") if len(parts) > 2: raise PolyaxonSchemaError( "Received an invalid entity reference: `{}`".format(entity)) if len(parts) == 2: owner, entity_name = parts else: owner = None entity_name = entity return owner, entity_name
def convert_to_dict(x): x = json.loads(x) if not isinstance(x, Mapping): raise PolyaxonSchemaError( "Cannot convert value `{}` (key: `{}`) to `dict`".format( x, key)) return x
def is_supported_in_eager_mode(spec: Union[V1Operation, V1CompiledOperation]): if not spec.matrix: if spec.component and spec.component.run: raise PolyaxonSchemaError("This operation with runtime `{}` " "is not supported in eager mode".format( spec.component.run.kind)) else: raise PolyaxonSchemaError( "Received a bad configuration, eager mode not supported") if (not spec.has_random_search_matrix and not spec.has_grid_search_matrix and not spec.has_mapping_matrix): raise PolyaxonSchemaError( "This operation is defining a matrix kind `{}` " "which is not supported in eager mode".format( spec.get_matrix_kind()))
def _get_op_upstream(self, op) -> Set: upstream = set(op.dependencies) if op.dependencies else set([]) if not op.params: return upstream if not isinstance(op.params, Mapping): raise PolyaxonSchemaError( "Op `{}` defines a malformed params `{}`, " "params should be a dictionary of form <name: value>".format( op.name, op.params)) for param in op.params: param_spec = op.params[param].get_spec( name=param, iotype=None, is_flag=None, is_list=None, is_context=None, arg_format=None, ) if param_spec.param.is_ops_ref: upstream.add(param_spec.param.entity_ref) return upstream
def get_eager_matrix_operations( content: str, compiled_operation: V1CompiledOperation, is_cli: bool = False, ) -> List[V1Operation]: is_supported_in_eager_mode(compiled_operation) try: import numpy as np except ImportError as e: if is_cli: Printer.print_error("numpy is required for this operation", sys_exit=True) raise e from polyaxon.polytune.search_managers.grid_search.manager import GridSearchManager from polyaxon.polytune.search_managers.mapping.manager import MappingManager from polyaxon.polytune.search_managers.random_search.manager import ( RandomSearchManager, ) if compiled_operation.has_random_search_matrix: suggestions = RandomSearchManager(compiled_operation.matrix).get_suggestions() elif compiled_operation.has_grid_search_matrix: suggestions = GridSearchManager(compiled_operation.matrix).get_suggestions() elif compiled_operation.has_mapping_matrix: suggestions = MappingManager(compiled_operation.matrix).get_suggestions() else: raise PolyaxonSchemaError( "Received a bad configuration, eager mode not supported, " "I should not be here!" ) return get_ops_from_suggestions( content=content, compiled_operation=compiled_operation, suggestions=suggestions )
def get_dict_of_dicts( key, value, is_list=None, is_optional=False, default=None, options=None # noqa ): """ Get the value corresponding to the key and converts it to `dict`. Add an extra validation that all keys have a dict as values. Args: key: the dict key. value: the value to parse. is_optional: To raise an error if key was not found. default: default value if is_optional is True. options: list/tuple if provided, the value must be one of these values. Returns: `dict or dict`: value corresponding to the key. """ value = get_dict( key=key, value=value, is_optional=is_optional, default=default, options=options ) if not value: return default for k in value: if not isinstance(value[k], Mapping): raise PolyaxonSchemaError( "`{}` must be an object. " "Received a non valid configuration for key `{}`.".format(value[k], key) ) return value
def get_queue_info(queue: str): if not queue: raise PolyaxonSchemaError("Received an invalid queue {}".format(queue)) parts = queue.replace(".", "/").split("/") agent = None queue_name = queue if len(parts) == 2: agent, queue_name = parts elif len(parts) > 2: raise PolyaxonSchemaError( "Please provide a valid queue. " "The queue name should be: queue-name to use the default agent or agent-name/queue." ) return agent, queue_name
def set_op_component(self, op_name): if op_name not in self.dag: raise PolyaxonSchemaError( "Job with name `{}` was not found in Dag, " "make sure to run `process_dag`.".format(op_name)) op_spec = self.dag[op_name] if op_spec.op.component: return if op_name not in self._op_component_mapping: raise PolyaxonSchemaError( "Pipeline op with name `{}` requires a component_ref with name `{}`, " "which is not defined on this pipeline, " "make sure to run `process_components`".format( op_name, op_spec.op.component_ref.name)) component_ref_name = self._op_component_mapping[op_name] op_spec.op.component = self._components_by_names[component_ref_name]
def _read_from_yml(f_path, is_stream=False): try: if is_stream: return yaml.safe_load(f_path) with open(f_path) as f: return yaml.safe_load(f) except (ScannerError, ParserError): raise PolyaxonSchemaError("Received non valid yaml: `{}`".format(f_path))
def parse_auth_spec(auth_spec): user_pass = auth_spec.split(":") if len(user_pass) != 2: raise PolyaxonSchemaError( "Received invalid uri_spec `{}`. `user:host` is not conform." "The uri must be in the format `user:pass`".format(auth_spec)) return AuthSpec(user=user_pass[0], password=user_pass[1])
def get_local_project(is_cli: bool = False): try: return ProjectConfigManager.get_config() except Exception: # noqa if is_cli: Printer.print_error(CACHE_ERROR, sys_exit=True) else: raise PolyaxonSchemaError(CACHE_ERROR)
def _read_from_file(f_path, file_type): _, ext = os.path.splitext(f_path) if ext in (".yml", ".yaml") or file_type in (".yml", ".yaml"): return _read_from_yml(f_path) elif ext == ".json" or file_type == ".json": return _read_from_json(f_path) raise PolyaxonSchemaError( "Expects a file with extension: `.yml`, `.yaml`, or `json`, " "received instead `{}`".format(ext))
def apply_run_contexts(cls, config: V1CompiledOperation, contexts=None): if config.has_pipeline: raise PolyaxonSchemaError( "This method is not allowed on this specification.") params = config.validate_params(is_template=False, check_runs=True) params = {param.name: param for param in params} params = cls._update_params_with_contexts(params, contexts) parsed_data = Parser.parse_run(config.to_dict(), params) return cls.CONFIG.read(parsed_data)
def validate_dag(self, dag=None): dag = dag or self.dag orphan_ops = self.get_orphan_ops(dag=dag) if orphan_ops: raise PolyaxonSchemaError( "Pipeline has a non valid dag, the dag contains an orphan ops: `{}`, " "check if you are referencing this op " "in a parameter or a condition".format(orphan_ops)) self.sort_topologically(dag=dag)
def parse_wasbs_path(wasbs_path): parsed_url = urllib.parse.urlparse(wasbs_path) if parsed_url.scheme != "wasbs": raise PolyaxonSchemaError( "Received an invalid url `{}`".format(wasbs_path)) match = re.match("([^@]+)@([^.]+)\\.blob\\.core\\.windows\\.net", parsed_url.netloc) if match is None: raise PolyaxonSchemaError( "wasbs url must be of the form <container>@<account>.blob.core.windows.net" ) container = match.group(1) storage_account = match.group(2) path = parsed_url.path if path.startswith("/"): path = path[1:] return WasbsSpec(container, storage_account, path)
def _read_from_yml(f_path, is_stream=False): try: if is_stream: return yaml.safe_load(f_path) with open(f_path) as f: return yaml.safe_load(f) except (ScannerError, ParserError) as e: raise PolyaxonSchemaError("Received non valid yaml: `%s`.\n" "Yaml error %s" % (f_path, e)) from e