def get_versioned_entity_info(
entity: str, entity_name: str, default_owner: str
) -> Tuple[str, str, str]:
if not entity:
raise PolyaxonSchemaError(
"Received an invalid {} reference: `{}`".format(entity_name, entity)
)
entity_values = entity.split(":")
if len(entity_values) > 2:
raise PolyaxonSchemaError(
"Received an invalid {} reference: `{}`".format(entity_name, entity)
)
if len(entity_values) == 2:
entity_name, version = entity_values
else:
entity_name, version = entity_values[0], "latest"
version = version or "latest"
parts = entity_name.replace(".", "/").split("/")
owner = default_owner
if not parts or len(parts) > 2:
raise PolyaxonSchemaError(
"Received an invalid {} reference: `{}`".format(entity_name, entity)
)
if len(parts) == 2:
owner, entity_namespace = parts
else:
entity_namespace = entity_name
return owner, entity_namespace, version
def read(self):
if isinstance(self.value, Mapping):
return self.value
# try a python file
if isinstance(self.value, str) and (
self.config_type == ".py" or ".py" in self.value
):
_f_path, _f_module = _get_python_file_def(self.value)
if _f_path and _f_module:
return _read_from_python(_f_path, _f_module)
if os.path.isfile(self.value):
return _read_from_file(self.value, self.config_type)
# try reading a stream of yaml or json
if not self.config_type or self.config_type in (".json", ".yml", ".yaml"):
try:
return _read_from_stream(self.value)
except (ScannerError, ParserError):
raise PolyaxonSchemaError(
"Received an invalid yaml stream: `{}`".format(self.value)
)
if self.config_type == "url":
return _read_from_url(self.value)
if self.config_type == "hub":
if os.environ.get(POLYAXON_KEYS_USE_GIT_REGISTRY, False):
return _read_from_public_hub(self.value)
return _read_from_polyaxon_hub(self.value)
raise PolyaxonSchemaError(
"Received an invalid configuration: `{}`".format(self.value)
)
def read_from(cls, config_values, config_type=None):
"""
Reads an ordered list of configuration values and
deep merge the values in reverse order.
"""
if not config_values:
raise PolyaxonSchemaError(
"Cannot read config_value: `{}`".format(config_values)
)
config_values = to_list(config_values, check_none=True)
config = {}
for config_value in config_values:
config_value = ConfigSpec.get_from(
value=config_value, config_type=config_type
)
config_value.check_type()
config_results = config_value.read()
if config_results and isinstance(config_results, Mapping):
config = deep_update(config, config_results)
elif config_value.check_if_exists:
raise PolyaxonSchemaError(
"Cannot read config_value: `{}`".format(config_value.value)
)
return config
def parse_uri_spec(uri_spec) -> V1UriType:
if isinstance(uri_spec, V1UriType):
return uri_spec
if isinstance(uri_spec, Mapping):
return V1UriType.from_dict(uri_spec)
try:
auth_spec = convert_to_dict(uri_spec, "")
return V1UriType.from_dict(auth_spec)
except (PolyaxonSchemaError, JSONDecodeError):
pass
parts = uri_spec.split("@")
if len(parts) != 2:
raise PolyaxonSchemaError(
"Received invalid uri_spec `{}`. "
"The uri must be in the format `user:pass@host`".format(uri_spec))
user_pass, host = parts
user_pass = user_pass.split(":")
if len(user_pass) != 2:
raise PolyaxonSchemaError(
"Received invalid uri_spec `{}`. `user:host` is not conform."
"The uri must be in the format `user:pass@host`".format(uri_spec))
return V1UriType(user=user_pass[0], password=user_pass[1], host=host)
def _get_typed_list_value(key,
value,
target_type,
type_convert,
is_optional=False,
default=None,
options=None):
"""
Return the value corresponding to the key converted first to list
than each element to the given type.
Args:
key: the dict key.
target_type: The type we expect the variable or key to be in.
type_convert: A lambda expression that converts the key to the desired type.
is_optional: To raise an error if key was not found.
default: default value if is_optional is True.
options: list/tuple if provided, the value must be one of these values.
"""
value = _get_typed_value(
key=key,
value=value,
target_type=list,
type_convert=json.loads,
is_optional=is_optional,
default=default,
options=options,
)
if not value:
return default
raise_type = "dict" if target_type == Mapping else target_type
if not isinstance(value, list):
raise PolyaxonSchemaError("Cannot convert value `{}` (key: `{}`) "
"to `{}`".format(value, key, raise_type))
# If we are here the value must be a list
result = []
for v in value:
if isinstance(v, six.string_types):
try:
result.append(type_convert(v))
except ValueError:
raise PolyaxonSchemaError(
"Cannot convert value `{}` (found in list key: `{}`) "
"to `{}`".format(v, key, raise_type))
elif isinstance(v, target_type):
result.append(v)
else:
raise PolyaxonSchemaError(
"Cannot convert value `{}` (found in list key: `{}`) "
"to `{}`".format(v, key, raise_type))
return result
def _read_from_json(f_path, is_stream=False):
if is_stream:
try:
return json.loads(f_path)
except ValueError as e:
raise PolyaxonSchemaError(e)
try:
return json.loads(open(f_path).read())
except ValueError as e:
raise PolyaxonSchemaError(e)
def _read_from_json(f_path, is_stream=False):
if is_stream:
try:
return json.loads(f_path)
except ValueError as e:
raise PolyaxonSchemaError("Json error: %s" % e) from e
try:
return json.loads(open(f_path).read())
except ValueError as e:
raise PolyaxonSchemaError("Received non valid json: `%s`.\n"
"Json error %s" % (f_path, e)) from e
def _get_typed_value(
key,
value,
target_type,
type_convert,
is_optional=False,
default=None,
options=None,
base_types=None,
):
"""
Return the value corresponding to the key converted to the given type.
Args:
key: the dict key.
target_type: The type we expect the variable or key to be in.
type_convert: A lambda expression that converts the key to the desired type.
is_optional: To raise an error if key was not found.
default: default value if is_optional is True.
options: list/tuple if provided, the value must be one of these values.
base_types: the base types to check for conversion
Returns:
The corresponding value of the key converted.
"""
if value is None or value == NO_VALUE_FOUND:
if not is_optional:
raise PolyaxonSchemaError(
"No value was provided for the non optional key `{}`.".format(key)
)
return default
base_types = base_types or str
if isinstance(value, base_types):
try:
# _add_key(key, is_secret=is_secret, is_local=is_local)
_check_options(key=key, value=value, options=options)
return type_convert(value)
except (ValueError, ValidationError):
raise PolyaxonSchemaError(
"Cannot convert value `{}` (key: `{}`) "
"to `{}`".format(value, key, target_type)
)
if isinstance(value, target_type):
# _add_key(key, is_secret=is_secret, is_local=is_local)
_check_options(key=key, value=value, options=options)
return value
raise PolyaxonSchemaError(
"Cannot convert value `{}` (key: `{}`) "
"to `{}`".format(value, key, target_type)
)
def _get_python_file_def(f_path):
results = f_path.split(":")
if len(results) != 2 or not results[1]:
return None, None
_f_path = results[0].strip("")
_module_name = results[1].strip("")
if not os.path.exists(_f_path):
raise PolyaxonSchemaError(
"Received non existing python file: `{}`".format(f_path))
if not _module_name:
raise PolyaxonSchemaError(
"Received an invalid python module: `{}`".format(f_path))
return _f_path, _module_name
def is_supported_in_eager_mode(spec: Union[V1Operation, V1CompiledOperation]):
if not spec.matrix:
if spec.component and spec.component.run:
raise PolyaxonSchemaError("This operation with runtime `{}` "
"is not supported in eager mode".format(
spec.component.run.kind))
else:
raise PolyaxonSchemaError(
"Received a bad configuration, eager mode not supported")
if spec.get_matrix_kind() not in V1MatrixKind.eager_values:
raise PolyaxonSchemaError(
"This operation is defining a matrix kind `{}` "
"which is not supported in eager mode".format(
spec.get_matrix_kind()))
def parse_uri_spec(uri_spec):
parts = uri_spec.split("@")
if len(parts) != 2:
raise PolyaxonSchemaError(
"Received invalid uri_spec `{}`. "
"The uri must be in the format `user:pass@host`".format(uri_spec))
user_pass, host = parts
user_pass = user_pass.split(":")
if len(user_pass) != 2:
raise PolyaxonSchemaError(
"Received invalid uri_spec `{}`. `user:host` is not conform."
"The uri must be in the format `user:pass@host`".format(uri_spec))
return UriSpec(user=user_pass[0], password=user_pass[1], host=host)
def to_light_dict(
self,
humanize_values=False,
include_attrs=None,
exclude_attrs=None,
unknown=None,
dump=False,
):
unknown = unknown or self.UNKNOWN_BEHAVIOUR
obj_dict = self.to_dict(humanize_values=humanize_values,
unknown=unknown)
if all([include_attrs, exclude_attrs]):
raise PolyaxonSchemaError(
"Only one value `include_attrs` or `exclude_attrs` is allowed."
)
if not any([include_attrs, exclude_attrs]): # Use Default setup attrs
include_attrs = self.DEFAULT_INCLUDE_ATTRIBUTES
exclude_attrs = self.DEFAULT_EXCLUDE_ATTRIBUTES
if include_attrs:
exclude_attrs = set(obj_dict.keys()) - set(include_attrs)
for attr in exclude_attrs:
obj_dict.pop(attr, None)
if dump:
return ujson.dumps(obj_dict)
return obj_dict
def parse_gcs_path(gcs_path):
"""
Parses and validates a google cloud storage url.
Returns:
tuple(bucket_name, blob).
"""
parsed_url = urllib.parse.urlparse(gcs_path)
if not parsed_url.netloc:
raise PolyaxonSchemaError(
"Received an invalid GCS url `{}`".format(gcs_path))
if parsed_url.scheme != "gs":
raise PolyaxonSchemaError(
"Received an invalid url GCS `{}`".format(gcs_path))
blob = parsed_url.path.lstrip("/")
return GCSSpec(parsed_url.netloc, blob)
def check_type(self):
type_check = self.config_type is None and not isinstance(
self.value, (Mapping, six.string_types))
if type_check:
raise PolyaxonSchemaError(
"Expects Mapping, string, or list of Mapping/string instances, "
"received {} instead".format(type(self.value)))
def get_entity_info(entity):
if not entity:
raise PolyaxonSchemaError(
"Received an invalid entity reference: `{}`".format(entity))
parts = entity.replace(".", "/").split("/")
if len(parts) > 2:
raise PolyaxonSchemaError(
"Received an invalid entity reference: `{}`".format(entity))
if len(parts) == 2:
owner, entity_name = parts
else:
owner = None
entity_name = entity
return owner, entity_name
def convert_to_dict(x):
x = json.loads(x)
if not isinstance(x, Mapping):
raise PolyaxonSchemaError(
"Cannot convert value `{}` (key: `{}`) to `dict`".format(
x, key))
return x
def is_supported_in_eager_mode(spec: Union[V1Operation, V1CompiledOperation]):
if not spec.matrix:
if spec.component and spec.component.run:
raise PolyaxonSchemaError("This operation with runtime `{}` "
"is not supported in eager mode".format(
spec.component.run.kind))
else:
raise PolyaxonSchemaError(
"Received a bad configuration, eager mode not supported")
if (not spec.has_random_search_matrix and not spec.has_grid_search_matrix
and not spec.has_mapping_matrix):
raise PolyaxonSchemaError(
"This operation is defining a matrix kind `{}` "
"which is not supported in eager mode".format(
spec.get_matrix_kind()))
def _get_op_upstream(self, op) -> Set:
upstream = set(op.dependencies) if op.dependencies else set([])
if not op.params:
return upstream
if not isinstance(op.params, Mapping):
raise PolyaxonSchemaError(
"Op `{}` defines a malformed params `{}`, "
"params should be a dictionary of form <name: value>".format(
op.name, op.params))
for param in op.params:
param_spec = op.params[param].get_spec(
name=param,
iotype=None,
is_flag=None,
is_list=None,
is_context=None,
arg_format=None,
)
if param_spec.param.is_ops_ref:
upstream.add(param_spec.param.entity_ref)
return upstream
def get_eager_matrix_operations(
content: str, compiled_operation: V1CompiledOperation, is_cli: bool = False,
) -> List[V1Operation]:
is_supported_in_eager_mode(compiled_operation)
try:
import numpy as np
except ImportError as e:
if is_cli:
Printer.print_error("numpy is required for this operation", sys_exit=True)
raise e
from polyaxon.polytune.search_managers.grid_search.manager import GridSearchManager
from polyaxon.polytune.search_managers.mapping.manager import MappingManager
from polyaxon.polytune.search_managers.random_search.manager import (
RandomSearchManager,
)
if compiled_operation.has_random_search_matrix:
suggestions = RandomSearchManager(compiled_operation.matrix).get_suggestions()
elif compiled_operation.has_grid_search_matrix:
suggestions = GridSearchManager(compiled_operation.matrix).get_suggestions()
elif compiled_operation.has_mapping_matrix:
suggestions = MappingManager(compiled_operation.matrix).get_suggestions()
else:
raise PolyaxonSchemaError(
"Received a bad configuration, eager mode not supported, "
"I should not be here!"
)
return get_ops_from_suggestions(
content=content, compiled_operation=compiled_operation, suggestions=suggestions
)
def get_dict_of_dicts(
key, value, is_list=None, is_optional=False, default=None, options=None # noqa
):
"""
Get the value corresponding to the key and converts it to `dict`.
Add an extra validation that all keys have a dict as values.
Args:
key: the dict key.
value: the value to parse.
is_optional: To raise an error if key was not found.
default: default value if is_optional is True.
options: list/tuple if provided, the value must be one of these values.
Returns:
`dict or dict`: value corresponding to the key.
"""
value = get_dict(
key=key, value=value, is_optional=is_optional, default=default, options=options
)
if not value:
return default
for k in value:
if not isinstance(value[k], Mapping):
raise PolyaxonSchemaError(
"`{}` must be an object. "
"Received a non valid configuration for key `{}`.".format(value[k], key)
)
return value
def get_queue_info(queue: str):
if not queue:
raise PolyaxonSchemaError("Received an invalid queue {}".format(queue))
parts = queue.replace(".", "/").split("/")
agent = None
queue_name = queue
if len(parts) == 2:
agent, queue_name = parts
elif len(parts) > 2:
raise PolyaxonSchemaError(
"Please provide a valid queue. "
"The queue name should be: queue-name to use the default agent or agent-name/queue."
)
return agent, queue_name
def set_op_component(self, op_name):
if op_name not in self.dag:
raise PolyaxonSchemaError(
"Job with name `{}` was not found in Dag, "
"make sure to run `process_dag`.".format(op_name))
op_spec = self.dag[op_name]
if op_spec.op.component:
return
if op_name not in self._op_component_mapping:
raise PolyaxonSchemaError(
"Pipeline op with name `{}` requires a component_ref with name `{}`, "
"which is not defined on this pipeline, "
"make sure to run `process_components`".format(
op_name, op_spec.op.component_ref.name))
component_ref_name = self._op_component_mapping[op_name]
op_spec.op.component = self._components_by_names[component_ref_name]
def _read_from_yml(f_path, is_stream=False):
try:
if is_stream:
return yaml.safe_load(f_path)
with open(f_path) as f:
return yaml.safe_load(f)
except (ScannerError, ParserError):
raise PolyaxonSchemaError("Received non valid yaml: `{}`".format(f_path))
def parse_auth_spec(auth_spec):
user_pass = auth_spec.split(":")
if len(user_pass) != 2:
raise PolyaxonSchemaError(
"Received invalid uri_spec `{}`. `user:host` is not conform."
"The uri must be in the format `user:pass`".format(auth_spec))
return AuthSpec(user=user_pass[0], password=user_pass[1])
def get_local_project(is_cli: bool = False):
try:
return ProjectConfigManager.get_config()
except Exception: # noqa
if is_cli:
Printer.print_error(CACHE_ERROR, sys_exit=True)
else:
raise PolyaxonSchemaError(CACHE_ERROR)
def _read_from_file(f_path, file_type):
_, ext = os.path.splitext(f_path)
if ext in (".yml", ".yaml") or file_type in (".yml", ".yaml"):
return _read_from_yml(f_path)
elif ext == ".json" or file_type == ".json":
return _read_from_json(f_path)
raise PolyaxonSchemaError(
"Expects a file with extension: `.yml`, `.yaml`, or `json`, "
"received instead `{}`".format(ext))
def apply_run_contexts(cls, config: V1CompiledOperation, contexts=None):
if config.has_pipeline:
raise PolyaxonSchemaError(
"This method is not allowed on this specification.")
params = config.validate_params(is_template=False, check_runs=True)
params = {param.name: param for param in params}
params = cls._update_params_with_contexts(params, contexts)
parsed_data = Parser.parse_run(config.to_dict(), params)
return cls.CONFIG.read(parsed_data)
def validate_dag(self, dag=None):
dag = dag or self.dag
orphan_ops = self.get_orphan_ops(dag=dag)
if orphan_ops:
raise PolyaxonSchemaError(
"Pipeline has a non valid dag, the dag contains an orphan ops: `{}`, "
"check if you are referencing this op "
"in a parameter or a condition".format(orphan_ops))
self.sort_topologically(dag=dag)
def parse_wasbs_path(wasbs_path):
parsed_url = urllib.parse.urlparse(wasbs_path)
if parsed_url.scheme != "wasbs":
raise PolyaxonSchemaError(
"Received an invalid url `{}`".format(wasbs_path))
match = re.match("([^@]+)@([^.]+)\\.blob\\.core\\.windows\\.net",
parsed_url.netloc)
if match is None:
raise PolyaxonSchemaError(
"wasbs url must be of the form <container>@<account>.blob.core.windows.net"
)
container = match.group(1)
storage_account = match.group(2)
path = parsed_url.path
if path.startswith("/"):
path = path[1:]
return WasbsSpec(container, storage_account, path)
def _read_from_yml(f_path, is_stream=False):
try:
if is_stream:
return yaml.safe_load(f_path)
with open(f_path) as f:
return yaml.safe_load(f)
except (ScannerError, ParserError) as e:
raise PolyaxonSchemaError("Received non valid yaml: `%s`.\n"
"Yaml error %s" % (f_path, e)) from e