def get_op_specification(
self, params=None, profile=None, queue=None, nocache=None
) -> V1Operation:
job_data = {"version": self.config.version, "kind": kinds.OPERATION}
if params:
if not isinstance(params, Mapping):
raise PolyaxonfileError(
"Params: `{}` must be a valid mapping".format(params)
)
job_data["params"] = params
if profile:
job_data["profile"] = profile
if queue:
job_data["queue"] = queue
if nocache is not None:
job_data["cache"] = {"disable": nocache}
if self.config.kind == kinds.OPERATION:
config = get_specification(data=[self.config.to_dict(), job_data])
else:
job_data["component"] = self.config.to_dict()
config = get_specification(data=[job_data])
params = copy.deepcopy(config.params)
# Sanity check if params were passed
run_config = OperationSpecification.compile_operation(config)
run_config.validate_params(params=params, is_template=False)
if run_config.is_dag_run:
CompiledOperationSpecification.apply_context(run_config)
return config
def get_op_specification(
config: Union[V1Component, V1Operation] = None,
hub: str = None,
params: Dict = None,
profile: str = None,
queue: str = None,
nocache: bool = None,
path_context: str = None,
validate_params: bool = True,
) -> V1Operation:
job_data = {
"version": config.version if config else pkg.SCHEMA_VERSION,
"kind": kinds.OPERATION,
}
if params:
if not isinstance(params, Mapping):
raise PolyaxonfileError(
"Params: `{}` must be a valid mapping".format(params)
)
job_data["params"] = params
if profile:
job_data["profile"] = profile
if queue:
# Check only
get_queue_info(queue)
job_data["queue"] = queue
if nocache is not None:
job_data["cache"] = {"disable": nocache}
if config and config.kind == kinds.COMPONENT:
job_data["component"] = config.to_dict()
config = get_specification(data=[job_data])
elif config and config.kind == kinds.OPERATION:
config = get_specification(data=[config.to_dict(), job_data])
elif hub:
job_data["hubRef"] = hub
config = get_specification(data=[job_data])
if hub and config.hub_ref is None:
config.hub_ref = hub
hub = config.hub_ref
public_hub = config.has_public_hub_reference
params = copy.deepcopy(config.params)
# Sanity check if params were passed and we are not dealing with a hub component
if validate_params and not (hub and not public_hub):
run_config = OperationSpecification.compile_operation(config)
run_config.validate_params(params=params, is_template=False)
if run_config.is_dag_run:
run_config.run.set_path_context(path_context)
CompiledOperationSpecification.apply_operation_contexts(run_config)
return config
def collect_references(config: V1Operation, path_context: str = None):
if config.has_component_reference:
return config
elif config.has_hub_reference:
component = ConfigSpec.get_from(config.hub_ref, "hub").read()
elif config.has_url_reference:
component = ConfigSpec.get_from(config.url_ref, "url").read()
elif config.has_path_reference:
path_ref = config.path_ref
if path_context:
path_ref = os.path.join(
os.path.dirname(os.path.abspath(path_context)), path_ref)
component = ConfigSpec.get_from(path_ref).read()
else:
raise PolyaxonfileError("Operation found without component")
component = get_specification(data=component)
if component.kind != kinds.COMPONENT:
if config.has_url_reference:
ref_type = "Url ref"
ref = config.url_ref
else:
ref_type = "Path ref"
ref = config.path_ref
raise PolyaxonfileError(
"the reference ({}) `{}` is of kind `{}`, it should be a `{}`".
format(ref, ref_type, component.kind, kinds.COMPONENT))
config.component = component
if component.is_dag_run:
component.run.collect_components()
return config
def __init__(self, filepaths):
filepaths = to_list(filepaths)
for filepath in filepaths:
if not os.path.isfile(filepath):
raise PolyaxonfileError("`{}` must be a valid file".format(filepath))
self._filenames = [os.path.basename(filepath) for filepath in filepaths]
self.config = get_specification(data=reader.read(filepaths))
def get_op_specification(
config: Union[V1Component, V1Operation] = None,
hub=None,
params=None,
profile=None,
queue=None,
nocache=None,
) -> V1Operation:
job_data = {
"version": config.version if config else pkg.SCHEMA_VERSION,
"kind": kinds.OPERATION
}
if params:
if not isinstance(params, Mapping):
raise PolyaxonfileError(
"Params: `{}` must be a valid mapping".format(params))
job_data["params"] = params
if profile:
job_data["profile"] = profile
if queue:
job_data["queue"] = queue
if nocache is not None:
job_data["cache"] = {"disable": nocache}
if hub:
job_data["hubRef"] = hub
config = get_specification(data=[job_data])
elif config.kind == kinds.OPERATION:
config = get_specification(data=[config.to_dict(), job_data])
else:
job_data["component"] = config.to_dict()
config = get_specification(data=[job_data])
params = copy.deepcopy(config.params)
# Sanity check if params were passed and we are not dealing with a hub component
if not hub:
run_config = OperationSpecification.compile_operation(config)
run_config.validate_params(params=params, is_template=False)
if run_config.is_dag_run:
CompiledOperationSpecification.apply_context(run_config)
return config
def collect_references(config: V1Operation):
if config.has_component_reference or config.has_hub_reference:
return config
elif config.has_url_reference:
component = ConfigSpec.get_from(config.url_ref, "url").read()
elif config.has_path_reference:
component = ConfigSpec.get_from(config.path_ref).read()
else:
raise PolyaxonfileError("Operation found without component")
component = get_specification(data=component)
config.component = component
return config
def check_polyaxonfile(
polyaxonfile: str = None,
python_module: str = None,
url: str = None,
hub: str = None,
params=None,
profile=None,
queue=None,
nocache=None,
log=True,
is_cli: bool = True,
to_op: bool = True,
):
if sum([1 for i in [polyaxonfile, python_module, url, hub] if i]) > 1:
message = (
"You can only use one and only one option: "
"hub, url, module, or path ro polyaxonfile.".format(hub)
)
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
if not any([polyaxonfile, python_module, url, hub]):
polyaxonfile = check_default_path(path=".")
if not any([polyaxonfile, python_module, url, hub]):
polyaxonfile = ""
if hub and not to_op:
message = "Something went wrong, calling hub component `{}` without operation.".format(hub)
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
polyaxonfile = to_list(polyaxonfile, check_none=True)
parsed_params = None
if params:
parsed_params = parse_params(params, is_cli=is_cli)
if not any([os.path.isfile(f) for f in polyaxonfile]) and not any(
[python_module, url, hub]
):
message = (
"Please pass a valid polyaxonfile, a python module, url, or component name"
)
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
try:
plx_file = None
if not hub:
if python_module:
plx_file = ConfigSpec.get_from(python_module, config_type=".py").read()
elif url:
plx_file = ConfigSpec.get_from(url, "url").read()
else:
plx_file = ConfigSpec.read_from(polyaxonfile)
plx_file = get_specification(data=plx_file)
if plx_file.kind == kinds.OPERATION:
plx_file = collect_references(plx_file)
if to_op or hub:
plx_file = get_op_specification(
hub=hub,
config=plx_file,
params=parsed_params,
profile=profile,
queue=queue,
nocache=nocache,
)
if log and not is_cli:
Printer.print_success("Polyaxonfile valid")
return plx_file
except Exception as e:
message = "Polyaxonfile is not valid."
if is_cli:
handle_cli_error(e, message=message, sys_exit=True)
else:
raise PolyaxonfileError(message) from e
def check_polyaxonfile(
polyaxonfile: str = None,
python_module: str = None,
url: str = None,
hub: str = None,
params: Dict = None,
presets: List[str] = None,
queue: str = None,
nocache: bool = None,
cache: bool = None,
verbose: bool = True,
is_cli: bool = True,
to_op: bool = True,
validate_params: bool = True,
eager: bool = False,
git_init: V1Init = None,
ignore_template: bool = False,
):
if sum([1 for i in [python_module, url, hub] if i]) > 1:
message = ("You can only use one and only one option: "
"hub, url, or a python module.".format(hub))
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
if not any([polyaxonfile, python_module, url, hub]):
polyaxonfile = check_default_path(path=".")
if not any([polyaxonfile, python_module, url, hub]):
message = (
"Something went wrong, `check_polyaxonfile` was called without a polyaxonfile, "
"a hub component reference, a url or a python module.")
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
if hub and not to_op:
message = "Something went wrong, calling hub component `{}` without operation.".format(
hub)
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
polyaxonfile = to_list(polyaxonfile, check_none=True)
parsed_params = None
if params:
parsed_params = parse_params(params, is_cli=is_cli)
if not any([os.path.isfile(f)
for f in polyaxonfile]) and not any([python_module, url, hub]):
message = "Please pass a valid polyaxonfile, a python module, a url, or a component name"
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
try:
path_context = None
if python_module:
path_context = python_module
plx_file = (ConfigSpec.get_from(python_module,
config_type=".py").read().to_dict(
include_kind=True,
include_version=True))
elif url:
plx_file = ConfigSpec.get_from(url, "url").read()
elif hub:
plx_file = ConfigSpec.get_from(hub, "hub").read()
else:
path_context = polyaxonfile.pop(0)
plx_file = ConfigSpec.read_from(path_context)
plx_file = get_specification(data=plx_file)
if plx_file.kind == kinds.OPERATION:
plx_file = collect_references(plx_file, path_context)
plx_component = plx_file.component
else:
plx_component = plx_file
if plx_component.is_dag_run:
collect_dag_components(plx_component.run, path_context)
if to_op or hub:
plx_file = get_op_specification(
hub=hub,
config=plx_file,
params=parsed_params,
presets=presets,
queue=queue,
nocache=nocache,
cache=cache,
validate_params=validate_params,
preset_files=polyaxonfile,
git_init=git_init,
)
if verbose and is_cli:
Printer.print_success("Polyaxonfile valid")
if ignore_template:
plx_file.disable_template()
if plx_file.is_template():
template_message = "This polyaxonfile was marked as template by the owner:"
if plx_file.template.description:
template_message += "\ntemplate description: {}".format(
plx_file.template.description)
if plx_file.template.fields:
template_message += "\ntemplate fields that need changes: {}".format(
plx_file.template.fields)
Printer.print_warning(template_message)
if eager:
is_supported_in_eager_mode(spec=plx_file)
return plx_file
except Exception as e:
message = "Polyaxonfile is not valid."
if is_cli:
handle_cli_error(e, message=message, sys_exit=True)
else:
raise PolyaxonfileError(message) from e
def get_op_specification(
config: Union[V1Component, V1Operation] = None,
hub: str = None,
params: Dict = None,
presets: List[str] = None,
queue: str = None,
nocache: bool = None,
cache: bool = None,
validate_params: bool = True,
preset_files: List[str] = None,
git_init: V1Init = None,
) -> V1Operation:
if cache and nocache:
raise PolyaxonfileError("Received both cache and nocache")
job_data = {
"version": config.version if config else pkg.SCHEMA_VERSION,
"kind": kinds.OPERATION,
}
if params:
if not isinstance(params, Mapping):
raise PolyaxonfileError(
"Params: `{}` must be a valid mapping".format(params))
job_data["params"] = params
if presets:
job_data["presets"] = presets
if queue:
# Check only
get_queue_info(queue)
job_data["queue"] = queue
if cache:
job_data["cache"] = {"disable": False}
if nocache:
job_data["cache"] = {"disable": True}
if config and config.kind == kinds.COMPONENT:
job_data["component"] = config.to_dict()
config = get_specification(data=[job_data])
elif config and config.kind == kinds.OPERATION:
config = get_specification(data=[config.to_dict(), job_data])
elif hub:
job_data["hubRef"] = hub
config = get_specification(data=[job_data])
if hub and config.hub_ref is None:
config.hub_ref = hub
# Check if there's presets
for preset_plx_file in preset_files:
preset_plx_file = OperationSpecification.read(preset_plx_file,
is_preset=True)
config = config.patch(preset_plx_file,
strategy=preset_plx_file.patch_strategy)
# Turn git_init to a pre_merge preset
if git_init:
git_preset = V1Operation(run_patch={"init": [git_init.to_dict()]},
is_preset=True)
config = config.patch(git_preset, strategy=V1PatchStrategy.PRE_MERGE)
# Sanity check if params were passed and we are not dealing with a hub component
params = copy.deepcopy(config.params)
if validate_params:
# Avoid in-place patch
run_config = get_specification(config.to_dict())
run_config = OperationSpecification.compile_operation(run_config)
run_config.validate_params(params=params, is_template=False)
if run_config.is_dag_run:
CompiledOperationSpecification.apply_operation_contexts(run_config)
return config
def check_polyaxonfile(
polyaxonfile: str = None,
python_module: str = None,
url: str = None,
hub: str = None,
params: Dict = None,
profile: str = None,
queue: str = None,
nocache: bool = None,
verbose: bool = True,
eager_hub: bool = True,
is_cli: bool = True,
to_op: bool = True,
validate_params: bool = True,
eager: bool = False,
):
if sum([1 for i in [polyaxonfile, python_module, url, hub] if i]) > 1:
message = ("You can only use one and only one option: "
"hub, url, module, or path or polyaxonfile.".format(hub))
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
if not any([polyaxonfile, python_module, url, hub]):
polyaxonfile = check_default_path(path=".")
if not any([polyaxonfile, python_module, url, hub]):
polyaxonfile = ""
if hub and not to_op:
message = "Something went wrong, calling hub component `{}` without operation.".format(
hub)
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
polyaxonfile = to_list(polyaxonfile, check_none=True)
parsed_params = None
if params:
parsed_params = parse_params(params, is_cli=is_cli)
if not any([os.path.isfile(f)
for f in polyaxonfile]) and not any([python_module, url, hub]):
message = (
"Please pass a valid polyaxonfile, a python module, url, or component name"
)
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
try:
plx_file = None
path_context = None
public_hub = hub and "/" not in hub
if not hub or (public_hub and eager_hub):
if python_module:
path_context = python_module
plx_file = (ConfigSpec.get_from(
python_module,
config_type=".py").read().to_dict(include_kind=True,
include_version=True))
elif url:
plx_file = ConfigSpec.get_from(url, "url").read()
elif hub:
plx_file = ConfigSpec.get_from(hub, "hub").read()
else:
path_context = polyaxonfile[0]
plx_file = ConfigSpec.read_from(polyaxonfile)
plx_file = get_specification(data=plx_file)
if plx_file.kind == kinds.OPERATION:
plx_file = collect_references(plx_file, path_context)
if to_op or hub:
plx_file = get_op_specification(
hub=hub,
config=plx_file,
params=parsed_params,
profile=profile,
queue=queue,
nocache=nocache,
path_context=path_context,
validate_params=validate_params,
)
if verbose and is_cli:
Printer.print_success("Polyaxonfile valid")
if eager:
is_supported_in_eager_mode(spec=plx_file)
return plx_file
except Exception as e:
message = "Polyaxonfile is not valid."
if is_cli:
handle_cli_error(e, message=message, sys_exit=True)
else:
raise PolyaxonfileError(message) from e
