def collect_references(config: V1Operation, path_context: str = None):
if config.has_component_reference:
return config
elif config.has_hub_reference:
component = ConfigSpec.get_from(config.hub_ref, "hub").read()
elif config.has_url_reference:
component = ConfigSpec.get_from(config.url_ref, "url").read()
elif config.has_path_reference:
path_ref = config.path_ref
if path_context:
path_ref = os.path.join(
os.path.dirname(os.path.abspath(path_context)), path_ref)
component = ConfigSpec.get_from(path_ref).read()
else:
raise PolyaxonfileError("Operation found without component")
component = get_specification(data=component)
if component.kind != kinds.COMPONENT:
if config.has_url_reference:
ref_type = "Url ref"
ref = config.url_ref
else:
ref_type = "Path ref"
ref = config.path_ref
raise PolyaxonfileError(
"the reference ({}) `{}` is of kind `{}`, it should be a `{}`".
format(ref, ref_type, component.kind, kinds.COMPONENT))
config.component = component
if component.is_dag_run:
component.run.collect_components()
return config
def get_config_from_env(cls, **kwargs) -> ClientConfig:
tmp_path = os.path.join(CONTEXT_TMP_POLYAXON_PATH, cls.CONFIG_FILE_NAME)
user_path = os.path.join(CONTEXT_USER_POLYAXON_PATH, cls.CONFIG_FILE_NAME)
config = ConfigManager.read_configs(
[
ConfigSpec(tmp_path, config_type=".json", check_if_exists=False),
ConfigSpec(user_path, config_type=".json", check_if_exists=False),
os.environ,
]
)
return ClientConfig.from_dict(config.data)
def collect_references(config: V1Operation):
if config.has_component_reference or config.has_hub_reference:
return config
elif config.has_url_reference:
component = ConfigSpec.get_from(config.url_ref, "url").read()
elif config.has_path_reference:
component = ConfigSpec.get_from(config.path_ref).read()
else:
raise PolyaxonfileError("Operation found without component")
component = get_specification(data=component)
config.component = component
return config
def test_reads_config_map(self):
config = ConfigSpec.read_from([{"x": "y"}, {1: 2}, {"x": "override y"}])
assert config == {"x": "override y", 1: 2}
config = ConfigSpec.read_from(
[
{"x": "y"},
{1: 2},
{"x": "override y"},
"tests/fixtures/parsing/yaml_file.yml",
"tests/fixtures/parsing/json_file.json",
]
)
assert config == {"x": 1, "y": 2, 1: 2, "foo": "bar", "type": "json"}
def get_config_from_env(cls) -> AgentConfig:
tmp_path = os.path.join(CONTEXT_TMP_POLYAXON_PATH, ".agent")
user_path = os.path.join(CONTEXT_USER_POLYAXON_PATH, ".agent")
config_paths = [
os.environ,
ConfigSpec(tmp_path, config_type=".json", check_if_exists=False),
ConfigSpec(user_path, config_type=".json", check_if_exists=False),
{
"dummy": "dummy"
},
]
agent_config = ConfigManager.read_configs(config_paths)
return AgentConfig.from_dict(agent_config.data)
def get(ctx, save, filename):
"""Get a component info by component_name, or owner/component_name.
Uses /docs/core/cli/#caching
Examples:
To get a component by name
\b
$ polyaxon hub get component_name
To get a component by owner/name
\b
$ polyaxon hub get owner/component_name
"""
name = ctx.obj.get("name")
if not name:
Printer.print_error("Please provide a valid component name!")
sys.exit(0)
try:
polyaxonfile = ConfigSpec.get_from(name, "hub").read()
except Exception as e:
handle_cli_error(e, message="Could not get component `{}`.".format(name))
sys.exit(1)
specification = get_specification(data=polyaxonfile)
polyaxonfile = yaml.dump(polyaxonfile)
get_component_details(polyaxonfile=polyaxonfile, specification=specification)
if save:
filename = filename or "polyaxonfile.yaml"
with open(filename, "w") as env_file:
env_file.write(polyaxonfile)
def resume(ctx, polyaxonfile, u):
"""Resume run.
Uses [Caching](/references/polyaxon-cli/#caching)
Examples:
\b
```bash
$ polyaxon runs --uid=8aac02e3a62a4f0aaa257c59da5eab80 resume
```
"""
content = None
if polyaxonfile:
content = "{}".format(ConfigSpec.read_from(polyaxonfile))
# Check if we need to upload
if u:
ctx.invoke(upload, sync=False)
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True)
try:
polyaxon_client = RunClient(owner=owner,
project=project_name,
run_uuid=run_uuid)
response = polyaxon_client.resume(override_config=content)
Printer.print_success("Run was resumed with uid {}".format(
response.uuid))
except (ApiException, HTTPError) as e:
handle_cli_error(e,
message="Could not resume run `{}`.".format(run_uuid))
sys.exit(1)
def restart(ctx, copy, polyaxonfile, u):
"""Restart run.
Uses /docs/core/cli/#caching
Examples:
\b
$ polyaxon run --uid=8aac02e3a62a4f0aaa257c59da5eab80 restart
"""
content = None
if polyaxonfile:
content = "{}".format(ConfigSpec.read_from(polyaxonfile))
# Check if we need to upload
if u:
ctx.invoke(upload, sync=False)
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"),
ctx.obj.get("run_uuid"),
is_cli=True,
)
try:
polyaxon_client = RunClient(owner=owner,
project=project_name,
run_uuid=run_uuid)
response = polyaxon_client.restart(override_config=content, copy=copy)
Printer.print_success("Run was {} with uid {}".format(
"copied" if copy else "restarted", response.uuid))
except (ApiException, HTTPError) as e:
handle_cli_error(
e, message="Could not restart run `{}`.".format(run_uuid))
sys.exit(1)
def file(file_context, filepath, copy_path, track):
"""Create auth context."""
from polyaxon.init.file import create_file_lineage
from polyaxon.utils.hashing import hash_value
try:
file_context = V1FileType.from_dict(ConfigSpec.read_from(file_context))
except (PolyaxonSchemaError, ValidationError) as e:
Printer.print_error("received a non valid file context.")
Printer.print_error("Error message: {}.".format(e))
sys.exit(1)
filepath = os.path.join(filepath, file_context.filename)
check_or_create_path(filepath, is_dir=False)
# Clean any previous file on that path
if os.path.exists(filepath):
os.remove(filepath)
with open(filepath, "w") as generated_file:
generated_file.write(file_context.content)
if file_context.chmod:
subprocess.check_call(["chmod", file_context.chmod, filepath])
if copy_path:
filepath = copy_file(filepath, copy_path)
if track:
create_file_lineage(
filepath=filepath,
summary={"hash": hash_value(file_context.content)},
kind=file_context.kind,
)
Printer.print_success("File is initialized, path: `{}`".format(filepath))
def test_reads_yaml_stream(self):
stream = """---
x: y
1: 2
"""
config = ConfigSpec.read_from(stream)
assert config == {"x": "y", 1: 2}
def resume(ctx, polyaxonfile):
"""Resume run.
Uses /docs/core/cli/#caching
Examples:
\b
$ polyaxon ops --uid=8aac02e3a62a4f0aaa257c59da5eab80 resume
"""
content = None
if polyaxonfile:
content = "{}".format(ConfigSpec.read_from(polyaxonfile))
owner, project_name, run_uuid = get_project_run_or_local(
ctx.obj.get("project"), ctx.obj.get("run_uuid"), is_cli=True,
)
try:
polyaxon_client = RunClient(
owner=owner, project=project_name, run_uuid=run_uuid
)
response = polyaxon_client.resume(override_config=content)
Printer.print_success("Run was resumed with uid {}".format(response.uuid))
except (ApiException, HTTPError) as e:
handle_cli_error(e, message="Could not resume run `{}`.".format(run_uuid))
sys.exit(1)
def get_config_from_env(cls) -> AccessTokenConfig:
tmp_path = os.path.join(CONTEXT_TMP_POLYAXON_PATH,
cls.CONFIG_FILE_NAME)
user_path = os.path.join(CONTEXT_USER_POLYAXON_PATH,
cls.CONFIG_FILE_NAME)
auth_config = ConfigManager.read_configs([
os.environ,
ConfigSpec(tmp_path, config_type=".json", check_if_exists=False),
ConfigSpec(user_path, config_type=".json", check_if_exists=False),
ConfigSpec(CONTEXT_MOUNT_AUTH,
config_type=".json",
check_if_exists=False),
{
"dummy": "dummy"
},
])
return AccessTokenConfig.from_dict(auth_config.data)
def generate(polyaxonfile, python_module, build_context, destination,
copy_path, params, track):
"""Generate a dockerfile given the polyaxonfile."""
from polyaxon.init.dockerfile import create_dockerfile_lineage
from polyaxon.utils.hashing import hash_value
if all([polyaxonfile, build_context]):
Printer.print_error(
"Only a polyaxonfile or a build context option is required.")
sys.exit(1)
if build_context:
try:
build_context = [
V1DockerfileType.from_dict(ConfigSpec.read_from(build_context))
]
except (PolyaxonSchemaError, ValidationError) as e:
Printer.print_error("received a non valid build context.")
Printer.print_error("Error message: {}.".format(e))
sys.exit(1)
else:
specification = check_polyaxonfile(
polyaxonfile=polyaxonfile,
python_module=python_module,
params=params,
verbose=False,
)
try:
compiled_operation = OperationSpecification.compile_operation(
specification)
compiled_operation.apply_params(params=specification.config.params)
compiled_operation = CompiledOperationSpecification.apply_operation_contexts(
compiled_operation)
except PolyaxonSchemaError:
Printer.print_error(
"Could not run this polyaxonfile locally, "
"a context is required to resolve it dependencies.")
sys.exit(1)
build_context = compiled_operation.init_dockerfiles
for init_dockerfile in build_context:
generator = DockerFileGenerator(build_context=init_dockerfile,
destination=destination or ".")
generator.create()
Printer.print_success("Dockerfile was generated, path: `{}`".format(
generator.dockerfile_path))
dockerfile_path = generator.dockerfile_path
if copy_path:
dockerfile_path = copy_file(dockerfile_path, copy_path)
if track:
hash_content = hash_value(init_dockerfile.to_dict())
create_dockerfile_lineage(dockerfile_path,
summary={"hash": hash_content})
def test_reads_non_existing_file(self):
# Raises by default
with self.assertRaises(PolyaxonSchemaError):
reader.read("tests/fixtures/parsing/no_file.yml")
with self.assertRaises(PolyaxonSchemaError):
reader.read("tests/fixtures/parsing/no_file.json")
with self.assertRaises(PolyaxonSchemaError):
reader.read(ConfigSpec("tests/fixtures/parsing/no_file"))
with self.assertRaises(PolyaxonSchemaError):
reader.read(ConfigSpec("tests/fixtures/parsing/no_file.yml"))
with self.assertRaises(PolyaxonSchemaError):
reader.read(ConfigSpec("tests/fixtures/parsing/no_file.json"))
# Does not raise if set to ignore
assert (reader.read(
ConfigSpec("tests/fixtures/parsing/no_file",
check_if_exists=False)) == {})
assert (reader.read(
ConfigSpec("tests/fixtures/parsing/no_file.yml",
check_if_exists=False)) == {})
assert (reader.read(
ConfigSpec("tests/fixtures/parsing/no_file.json",
check_if_exists=False)) == {})
def read(cls, values):
if isinstance(values, cls.CONFIG):
return values
values = to_list(values)
data = ConfigSpec.read_from([{"kind": cls._SPEC_KIND}] + values)
try:
config = cls.CONFIG.from_dict(copy.deepcopy(data))
except TypeError as e:
raise ValidationError(
"Received a non valid config `{}`: `{}`".format(
cls._SPEC_KIND, e))
cls.check_data(data)
return config
def read(cls, values):
if isinstance(values, cls.CONFIG):
return values
if not isinstance(values, Mapping) or Sections.KIND not in values:
values = to_list(values)
values = ConfigSpec.read_from([{Sections.KIND: cls._SPEC_KIND}] + values)
cls.check_data(values)
try:
config = cls.CONFIG.from_dict(copy.deepcopy(values))
except TypeError as e:
raise ValidationError(
"Received a non valid config `{}`: `{}`".format(cls._SPEC_KIND, e)
)
return config
def generate(polyaxonfile, python_module, build_context, destination,
copy_path, params):
"""Generate a dockerfile given the polyaxonfile."""
if all([polyaxonfile, build_context]):
Printer.print_error(
"Only a polyaxonfile or a build context option is required.")
sys.exit(1)
if build_context:
try:
build_context = [
V1DockerfileType.from_dict(ConfigSpec.read_from(build_context))
]
except (PolyaxonSchemaError, ValidationError) as e:
Printer.print_error("received a non valid build context.")
Printer.print_error("Error message: {}.".format(e))
sys.exit(1)
else:
specification = check_polyaxonfile(
polyaxonfile=polyaxonfile,
python_module=python_module,
params=params,
log=False,
)
try:
compiled_operation = specification.compile_operation()
compiled_operation.apply_params(params=specification.config.params)
compiled_operation = CompiledOperationSpecification.apply_context(
compiled_operation)
except PolyaxonSchemaError:
Printer.print_error(
"Could not run this polyaxonfile locally, "
"a context is required to resolve it dependencies.")
sys.exit(1)
build_context = compiled_operation.init_dockerfiles
for init_dockerfile in build_context:
generator = DockerFileGenerator(build_context=init_dockerfile,
destination=destination or ".")
generator.create()
Printer.print_success("Dockerfile was generated, path: `{}`".format(
generator.dockerfile_path))
if copy_path:
copy_file(generator.dockerfile_path, copy_path)
def check_polyaxonfile(
polyaxonfile=None,
python_module=None,
params=None,
profile=None,
queue=None,
nocache=None,
log=True,
):
if not any([polyaxonfile, python_module]):
polyaxonfile = PolyaxonFile.check_default_path(path=".")
if not any([polyaxonfile, python_module]):
polyaxonfile = ""
polyaxonfile = to_list(polyaxonfile)
exists = [os.path.isfile(f) for f in polyaxonfile]
parsed_params = None
if params:
parsed_params = parse_params(params)
if not any(exists) and not python_module:
Printer.print_error(
"Polyaxonfile is not present, "
"please run {}".format(constants.INIT_COMMAND)
)
sys.exit(1)
if python_module:
config = ConfigSpec.get_from(python_module)
return config.read()
try:
plx_file = PolyaxonFile(polyaxonfile)
plx_file = plx_file.get_op_specification(
params=parsed_params, profile=profile, queue=queue, nocache=nocache
)
if log:
Printer.print_success("Polyaxonfile valid")
return plx_file
except Exception as e:
handle_cli_error(e, message="Polyaxonfile is not valid.")
sys.exit(1)
def read(config_values):
"""Reads an ordered list of configuration values and deep merge the values in reverse order."""
if not config_values:
raise PolyaxonSchemaError(
"Cannot read config_value: `{}`".format(config_values))
config_values = to_list(config_values)
config = {}
for config_value in config_values:
config_value = ConfigSpec.get_from(value=config_value)
config_value.check_type()
config_results = config_value.read()
if config_results and isinstance(config_results, Mapping):
config = deep_update(config, config_results)
elif config_value.check_if_exists:
raise PolyaxonSchemaError(
"Cannot read config_value: `{}`".format(config_value))
return config
def read_configs(cls, config_values): # pylint:disable=redefined-outer-name
config = ConfigSpec.read_from(config_values) # pylint:disable=redefined-outer-name
return cls(**config) if config else None
def check_polyaxonfile(
polyaxonfile: str = None,
python_module: str = None,
url: str = None,
hub: str = None,
params=None,
profile=None,
queue=None,
nocache=None,
log=True,
is_cli: bool = True,
to_op: bool = True,
):
if sum([1 for i in [polyaxonfile, python_module, url, hub] if i]) > 1:
message = (
"You can only use one and only one option: "
"hub, url, module, or path ro polyaxonfile.".format(hub)
)
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
if not any([polyaxonfile, python_module, url, hub]):
polyaxonfile = check_default_path(path=".")
if not any([polyaxonfile, python_module, url, hub]):
polyaxonfile = ""
if hub and not to_op:
message = "Something went wrong, calling hub component `{}` without operation.".format(hub)
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
polyaxonfile = to_list(polyaxonfile, check_none=True)
parsed_params = None
if params:
parsed_params = parse_params(params, is_cli=is_cli)
if not any([os.path.isfile(f) for f in polyaxonfile]) and not any(
[python_module, url, hub]
):
message = (
"Please pass a valid polyaxonfile, a python module, url, or component name"
)
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
try:
plx_file = None
if not hub:
if python_module:
plx_file = ConfigSpec.get_from(python_module, config_type=".py").read()
elif url:
plx_file = ConfigSpec.get_from(url, "url").read()
else:
plx_file = ConfigSpec.read_from(polyaxonfile)
plx_file = get_specification(data=plx_file)
if plx_file.kind == kinds.OPERATION:
plx_file = collect_references(plx_file)
if to_op or hub:
plx_file = get_op_specification(
hub=hub,
config=plx_file,
params=parsed_params,
profile=profile,
queue=queue,
nocache=nocache,
)
if log and not is_cli:
Printer.print_success("Polyaxonfile valid")
return plx_file
except Exception as e:
message = "Polyaxonfile is not valid."
if is_cli:
handle_cli_error(e, message=message, sys_exit=True)
else:
raise PolyaxonfileError(message) from e
def test_reads_non_valid_yaml_stream(self):
stream = ";sdfsd;sdff"
with self.assertRaises(PolyaxonSchemaError):
ConfigSpec.read_from(stream)
def read(cls, values, unknown=None, config_type=None):
values = ConfigSpec.read_from(values, config_type=config_type)
return cls.from_dict(values, unknown=unknown)
def test_reads_json_stream(self):
stream = """---
{x: y, 1: 2}
"""
config = ConfigSpec.read_from(stream)
assert config is not None
def check_polyaxonfile(
polyaxonfile: str = None,
python_module: str = None,
url: str = None,
hub: str = None,
params: Dict = None,
presets: List[str] = None,
queue: str = None,
nocache: bool = None,
cache: bool = None,
verbose: bool = True,
is_cli: bool = True,
to_op: bool = True,
validate_params: bool = True,
eager: bool = False,
git_init: V1Init = None,
ignore_template: bool = False,
):
if sum([1 for i in [python_module, url, hub] if i]) > 1:
message = ("You can only use one and only one option: "
"hub, url, or a python module.".format(hub))
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
if not any([polyaxonfile, python_module, url, hub]):
polyaxonfile = check_default_path(path=".")
if not any([polyaxonfile, python_module, url, hub]):
message = (
"Something went wrong, `check_polyaxonfile` was called without a polyaxonfile, "
"a hub component reference, a url or a python module.")
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
if hub and not to_op:
message = "Something went wrong, calling hub component `{}` without operation.".format(
hub)
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
polyaxonfile = to_list(polyaxonfile, check_none=True)
parsed_params = None
if params:
parsed_params = parse_params(params, is_cli=is_cli)
if not any([os.path.isfile(f)
for f in polyaxonfile]) and not any([python_module, url, hub]):
message = "Please pass a valid polyaxonfile, a python module, a url, or a component name"
if is_cli:
Printer.print_error(message, sys_exit=True)
else:
raise PolyaxonfileError(message)
try:
path_context = None
if python_module:
path_context = python_module
plx_file = (ConfigSpec.get_from(python_module,
config_type=".py").read().to_dict(
include_kind=True,
include_version=True))
elif url:
plx_file = ConfigSpec.get_from(url, "url").read()
elif hub:
plx_file = ConfigSpec.get_from(hub, "hub").read()
else:
path_context = polyaxonfile.pop(0)
plx_file = ConfigSpec.read_from(path_context)
plx_file = get_specification(data=plx_file)
if plx_file.kind == kinds.OPERATION:
plx_file = collect_references(plx_file, path_context)
plx_component = plx_file.component
else:
plx_component = plx_file
if plx_component.is_dag_run:
collect_dag_components(plx_component.run, path_context)
if to_op or hub:
plx_file = get_op_specification(
hub=hub,
config=plx_file,
params=parsed_params,
presets=presets,
queue=queue,
nocache=nocache,
cache=cache,
validate_params=validate_params,
preset_files=polyaxonfile,
git_init=git_init,
)
if verbose and is_cli:
Printer.print_success("Polyaxonfile valid")
if ignore_template:
plx_file.disable_template()
if plx_file.is_template():
template_message = "This polyaxonfile was marked as template by the owner:"
if plx_file.template.description:
template_message += "\ntemplate description: {}".format(
plx_file.template.description)
if plx_file.template.fields:
template_message += "\ntemplate fields that need changes: {}".format(
plx_file.template.fields)
Printer.print_warning(template_message)
if eager:
is_supported_in_eager_mode(spec=plx_file)
return plx_file
except Exception as e:
message = "Polyaxonfile is not valid."
if is_cli:
handle_cli_error(e, message=message, sys_exit=True)
else:
raise PolyaxonfileError(message) from e
def test_reads_yaml_files(self):
config = ConfigSpec.read_from("tests/fixtures/parsing/yaml_file.yml")
assert config == {"x": 10, "y": 20, "foo": "bar", "type": "yaml"}
def get_specification(data):
if not isinstance(data, Mapping):
data = ConfigSpec.read_from(data)
kind = BaseSpecification.get_kind(data=data)
return SPECIFICATION_BY_KIND[kind].read(data)
def test_reads_json_files(self):
config = ConfigSpec.read_from("tests/fixtures/parsing/json_file.json")
assert config == {"x": 1, "y": 2, "foo": "bar", "type": "json"}
def test_reads_yaml_files_without_extension(self):
config = reader.read(
ConfigSpec("tests/fixtures/parsing/yaml_file", config_type=".yml"))
assert config == {"x": 10, "y": 20, "foo": "bar", "type": "yaml"}
def test_reads_json_files_without_extension(self):
config = reader.read(
ConfigSpec("tests/fixtures/parsing/json_file",
config_type=".json"))
assert config == {"x": 1, "y": 2, "foo": "bar", "type": "json"}