def test_non_yaml_spec(self):
config = ",sdf;ldjks"
with self.assertRaises(PolyaxonSchemaError):
OperationSpecification.read(config)
with self.assertRaises(PolyaxonSchemaError):
ComponentSpecification.read(config)
def test_simple_file_with_run_patch_passes(self):
op_config = OperationSpecification.read(
os.path.abspath("tests/fixtures/plain/simple_job_run_patch.yml"),
)
assert op_config.version == 1.1
assert op_config.tags is None
assert op_config.run_patch["environment"]["annotations"] == {
"tf-version.cloud-tpus.google.com": "2.2"
}
assert len(op_config.component.run.volumes) == 1
assert op_config.component.run.to_dict()["volumes"][0] == {
"name": "foo",
"secret": {"secretName": "mysecret"},
}
assert op_config.component.run.container.image == "python-with-boto3"
assert op_config.component.run.container.command == "python download-s3-bucket"
assert op_config.component.run.container.resources == {
"requests": {"nvidia.com/gpu": 1},
"limits": {"nvidia.com/gpu": 1},
}
assert op_config.component.run.container.volume_mounts == [
{"name": "foo", "mount_path": "~/.aws/credentials", "readOnly": True}
]
run_config = OperationSpecification.compile_operation(op_config)
assert run_config.version == 1.1
assert run_config.tags is None
assert len(run_config.run.volumes) == 1
assert run_config.run.environment.annotations == {
"tf-version.cloud-tpus.google.com": "2.2"
}
assert run_config.run.to_dict()["volumes"][0] == {
"name": "foo",
"secret": {"secretName": "mysecret"},
}
assert run_config.run.container.image == "python-with-boto3"
assert run_config.run.container.command == "python download-s3-bucket"
assert run_config.run.container.resources == {
"requests": {"nvidia.com/gpu": 1},
"limits": {"nvidia.com/gpu": 1},
}
assert run_config.run.container.volume_mounts == [
{"name": "foo", "mount_path": "~/.aws/credentials", "readOnly": True}
]
def test_job_specification_raises_for_missing_container_section(self):
with self.assertRaises(PolyaxonfileError):
OperationSpecification.read(
os.path.abspath(
"tests/fixtures/plain/job_missing_container.yml"))
def get_op_specification(
config: Union[V1Component, V1Operation] = None,
hub: str = None,
params: Dict = None,
presets: List[str] = None,
queue: str = None,
nocache: bool = None,
cache: bool = None,
validate_params: bool = True,
preset_files: List[str] = None,
git_init: V1Init = None,
) -> V1Operation:
if cache and nocache:
raise PolyaxonfileError("Received both cache and nocache")
job_data = {
"version": config.version if config else pkg.SCHEMA_VERSION,
"kind": kinds.OPERATION,
}
if params:
if not isinstance(params, Mapping):
raise PolyaxonfileError(
"Params: `{}` must be a valid mapping".format(params))
job_data["params"] = params
if presets:
job_data["presets"] = presets
if queue:
# Check only
get_queue_info(queue)
job_data["queue"] = queue
if cache:
job_data["cache"] = {"disable": False}
if nocache:
job_data["cache"] = {"disable": True}
if config and config.kind == kinds.COMPONENT:
job_data["component"] = config.to_dict()
config = get_specification(data=[job_data])
elif config and config.kind == kinds.OPERATION:
config = get_specification(data=[config.to_dict(), job_data])
elif hub:
job_data["hubRef"] = hub
config = get_specification(data=[job_data])
if hub and config.hub_ref is None:
config.hub_ref = hub
# Check if there's presets
for preset_plx_file in preset_files:
preset_plx_file = OperationSpecification.read(preset_plx_file,
is_preset=True)
config = config.patch(preset_plx_file,
strategy=preset_plx_file.patch_strategy)
# Turn git_init to a pre_merge preset
if git_init:
git_preset = V1Operation(run_patch={"init": [git_init.to_dict()]},
is_preset=True)
config = config.patch(git_preset, strategy=V1PatchStrategy.PRE_MERGE)
# Sanity check if params were passed and we are not dealing with a hub component
params = copy.deepcopy(config.params)
if validate_params:
# Avoid in-place patch
run_config = get_specification(config.to_dict())
run_config = OperationSpecification.compile_operation(run_config)
run_config.validate_params(params=params, is_template=False)
if run_config.is_dag_run:
CompiledOperationSpecification.apply_operation_contexts(run_config)
return config
