def get_op_specification( self, params=None, profile=None, queue=None, nocache=None ) -> V1Operation: job_data = {"version": self.config.version, "kind": kinds.OPERATION} if params: if not isinstance(params, Mapping): raise PolyaxonfileError( "Params: `{}` must be a valid mapping".format(params) ) job_data["params"] = params if profile: job_data["profile"] = profile if queue: job_data["queue"] = queue if nocache is not None: job_data["cache"] = {"disable": nocache} if self.config.kind == kinds.OPERATION: config = get_specification(data=[self.config.to_dict(), job_data]) else: job_data["component"] = self.config.to_dict() config = get_specification(data=[job_data]) params = copy.deepcopy(config.params) # Sanity check if params were passed run_config = OperationSpecification.compile_operation(config) run_config.validate_params(params=params, is_template=False) if run_config.is_dag_run: CompiledOperationSpecification.apply_context(run_config) return config
def test_tf_passes(self): run_config = CompiledOperationSpecification.read( [ os.path.abspath("tests/fixtures/plain/distributed_tensorflow_file.yml"), {"kind": "compiled_operation"}, ] ) run_config = CompiledOperationSpecification.apply_operation_contexts(run_config) assert run_config.version == 1.1 assert run_config.termination is not None assert run_config.termination.ttl == 12 assert run_config.is_tf_job_run assert run_config.run.worker.replicas == 5 assert run_config.run.worker.environment.affinity is not None assert run_config.run.worker.environment.restart_policy == "OnFailure" assert run_config.run.worker.container.resources == { "requests": {"memory": "300Mi"}, "limits": {"memory": "300Mi"}, } assert run_config.run.ps.replicas == 10 assert run_config.run.ps.environment.affinity is None assert isinstance(run_config.run.ps.environment.tolerations, list) assert run_config.run.ps.environment.restart_policy == "OnFailure" assert run_config.run.ps.container.resources == { "requests": {"cpu": 3, "memory": "256Mi"}, "limits": {"cpu": 3, "memory": "256Mi"}, }
def test_simple_file_passes(self): run_config = CompiledOperationSpecification.read([ os.path.abspath("tests/fixtures/plain/simple_job.yml"), { "kind": "compiled_operation" }, ]) run_config = CompiledOperationSpecification.apply_context(run_config) assert run_config.version == 1.05 assert run_config.tags is None assert len(run_config.run.volumes) == 1 assert run_config.run.to_dict()["volumes"][0] == { "name": "foo", "secret": { "secretName": "mysecret" }, } assert run_config.run.container.image == "python-with-boto3" assert run_config.run.container.command == "python download-s3-bucket" assert run_config.run.container.resources == { "requests": { "nvidia.com/gpu": 1 }, "limits": { "nvidia.com/gpu": 1 }, } assert run_config.run.container.volume_mounts == [{ "name": "foo", "mount_path": "~/.aws/credentials", "readOnly": True }]
def test_job_file_with_environment_passes(self): run_config = CompiledOperationSpecification.read( [ os.path.abspath("tests/fixtures/plain/job_file_with_environment.yml"), {"kind": "compiled_operation"}, ] ) run_config = CompiledOperationSpecification.apply_operation_contexts(run_config) assert run_config.version == 1.1 assert isinstance(run_config.run.environment, V1Environment) assert run_config.run.environment.node_selector == {"polyaxon.com": "core"} assert run_config.run.container.resources == { "requests": {"cpu": 1, "memory": 200}, "limits": {"cpu": 2, "memory": 200}, } assert run_config.run.environment.affinity.node_affinity == { "requiredDuringSchedulingIgnoredDuringExecution": {} } assert run_config.run.environment.tolerations[0].key == "key" assert run_config.run.environment.tolerations[0].operator == "Exists" assert run_config.run.environment.labels == { "label_key1": "val1", "label_key2": "val2", } assert run_config.run.environment.annotations == { "annotation_key1": "val1", "annotation_key2": "val2", } assert run_config.run.environment.service_account_name == "new_sa" assert run_config.run.environment.image_pull_secrets == ["secret1", "secret2"] assert run_config.run.environment.security_context.run_as_user == 1000 assert run_config.run.environment.security_context.run_as_group == 3000 assert isinstance(run_config.plugins, V1Plugins) assert run_config.plugins.log_level == "DEBUG"
def test_specification_with_quotes(self): run_config = CompiledOperationSpecification.read([ os.path.abspath( "tests/fixtures/plain/polyaxonfile_with_quotes.yaml"), { "kind": "compiled_operation" }, ]) run_config = CompiledOperationSpecification.apply_operation_contexts( run_config) expected_run = { "kind": V1RunKind.JOB, "container": { "image": "continuumio/miniconda3", "command": ["python"], "args": ["-c \"print('Tweet tweet')\""], "name": "polyaxon-main", }, } assert run_config.run.to_dict() == expected_run run_config = V1CompiledOperation.read(run_config.to_dict()) run_config = CompiledOperationSpecification.apply_operation_contexts( run_config) assert run_config.run.to_dict() == expected_run
def test_passing_params_overrides_polyaxonfiles(self): run_config = CompiledOperationSpecification.read([ os.path.abspath("tests/fixtures/typing/required_inputs.yml"), { "kind": "compiled_operation" }, ]) with self.assertRaises(ValidationError): CompiledOperationSpecification.apply_context(run_config) assert run_config.inputs[0].value is None assert run_config.inputs[1].value is None run_config.apply_params(params={ "flag": { "value": True }, "loss": { "value": "some-loss" } }) assert run_config.inputs[0].value == "some-loss" assert run_config.inputs[1].value is True run_config = CompiledOperationSpecification.apply_context(run_config) run_config = CompiledOperationSpecification.apply_run_contexts( run_config) assert run_config.version == 1.05 assert run_config.tags == ["foo", "bar"] assert run_config.run.container.image == "my_image" assert run_config.run.container.command == ["/bin/sh", "-c"] assert (run_config.run.container.args == "video_prediction_train --loss=some-loss --flag")
def test_mpi_passes(self): run_config = CompiledOperationSpecification.read( [ os.path.abspath("tests/fixtures/plain/distributed_mpi_file.yml"), {"kind": "compiled_operation"}, ] ) run_config = CompiledOperationSpecification.apply_operation_contexts(run_config) assert run_config.version == 1.1 assert run_config.is_mpi_job_run assert run_config.termination is None assert run_config.run.launcher.replicas == 1 assert run_config.run.launcher.to_dict() == { "replicas": 1, "container": { "name": "polyaxon-main", "image": "mpioperator/tensorflow-benchmarks:latest", "command": ["mpirun", "python", "run.py"], }, } assert run_config.run.launcher.environment is None assert run_config.run.worker.replicas == 2 assert run_config.run.worker.environment.affinity is None assert run_config.run.worker.environment.node_selector is not None assert isinstance(run_config.run.worker.environment.tolerations, list) assert run_config.run.worker.to_dict()["container"] == { "name": "polyaxon-main", "image": "mpioperator/tensorflow-benchmarks:latest", "command": ["mpirun", "python", "run.py"], "resources": {"limits": {"nvidia.com/gpu": 1}}, }
def test_job_file_with_init_passes(self): run_config = CompiledOperationSpecification.read([ os.path.abspath("tests/fixtures/plain/job_file_with_init.yml"), { "kind": "compiled_operation" }, ]) run_config = CompiledOperationSpecification.apply_context(run_config) assert run_config.version == 1.05 assert isinstance(run_config.plugins, V1Plugins) assert run_config.plugins.log_level == "INFO" assert run_config.plugins.auth is True assert run_config.plugins.shm is True assert run_config.plugins.docker is True assert run_config.plugins.collect_artifacts is True assert run_config.plugins.collect_logs is None assert isinstance(run_config.run.environment, V1Environment) assert run_config.run.environment.labels == {"key": "value"} assert isinstance(run_config.run.init, list) assert len(run_config.run.connections) == 2 assert run_config.run.connections == ["data1", "data2"] assert len(run_config.run.volumes) == 2 assert run_config.run.volumes[0].name == "my_ssh_secret" assert run_config.run.volumes[0].secret == {"secretName": "mysecret"} assert run_config.run.volumes[1].name == "config_map" assert run_config.run.volumes[1].config_map == { "configName": "config_map2" }
def test_pytorch_passes(self): run_config = CompiledOperationSpecification.read( [ os.path.abspath("tests/fixtures/plain/distributed_pytorch_file.yml"), {"kind": "compiled_operation"}, ] ) run_config = CompiledOperationSpecification.apply_operation_contexts(run_config) assert run_config.version == 1.1 assert run_config.termination is not None assert run_config.termination.ttl == 12 assert run_config.is_pytorch_job_run assert run_config.run.master.replicas == 5 assert run_config.run.master.environment.to_dict() == { "restartPolicy": "OnFailure", "nodeName": "foo", "serviceAccountName": "sa1", } assert run_config.run.master.container.image == "my_image" assert run_config.run.master.container.resources == { "requests": {"memory": "300Mi"}, "limits": {"memory": "300Mi"}, } assert run_config.run.worker.replicas == 10 assert run_config.run.worker.environment.affinity is None assert run_config.run.worker.environment.restart_policy == "OnFailure" assert isinstance(run_config.run.worker.environment.tolerations, list) assert run_config.run.worker.container.resources == { "requests": {"cpu": 3, "memory": "256Mi"}, "limits": {"cpu": 3, "memory": "256Mi"}, }
def test_pipeline_with_no_ops_raises(self): run_config = V1CompiledOperation.read( [ os.path.abspath("tests/fixtures/pipelines/pipeline_with_no_ops.yml"), {"kind": "compiled_operation"}, ] ) with self.assertRaises(PolyaxonSchemaError): CompiledOperationSpecification.apply_context(run_config)
def test_apply_params_extends_connections_and_init(self): content = { "version": 1.1, "kind": "compiled_operation", "inputs": [ { "name": "docker_image", "type": types.IMAGE }, { "name": "git_repo", "type": types.GIT }, ], "run": { "kind": V1RunKind.JOB, "connections": ["{{ params.docker_image.connection }}"], "container": { "name": "polyaxon-main", "image": "{{ docker_image }}", "command": "train", }, }, } run_config = V1CompiledOperation.read(content) # no params with self.assertRaises(ValidationError): CompiledOperationSpecification.apply_operation_contexts(run_config) params = { "docker_image": { "value": "destination:tag", "connection": "docker-registry", }, "git_repo": { "value": V1GitType(revision="foo"), "connection": "repo-connection", }, } assert run_config.inputs[0].value is None assert run_config.inputs[1].value is None validated_params = run_config.validate_params(params=params) run_config.apply_params(params=params) assert params == {p.name: p.param.to_dict() for p in validated_params} assert run_config.inputs[0].connection == "docker-registry" assert run_config.inputs[1].connection == "repo-connection" run_config = CompiledOperationSpecification.apply_operation_contexts( run_config) run_config = CompiledOperationSpecification.apply_params(run_config) run_config = CompiledOperationSpecification.apply_runtime_contexts( run_config) assert run_config.run.connections == ["docker-registry"] assert run_config.run.container.image == "destination:tag"
def test_cyclic_pipeline_raises(self): run_config = V1CompiledOperation.read( [ os.path.abspath("tests/fixtures/pipelines/cyclic_pipeline.yml"), {"kind": "compiled_operation"}, ] ) assert run_config.is_dag_run is True assert run_config.has_pipeline is True with self.assertRaises(PolyaxonSchemaError): CompiledOperationSpecification.apply_context(run_config)
def test_pipeline_ops_not_corresponding_to_components(self): run_config = V1CompiledOperation.read( [ reader.read( os.path.abspath( "tests/fixtures/pipelines/pipeline_ops_not_corresponding_to_components.yml" ) ), {"kind": "compiled_operation"}, ] ) with self.assertRaises(PolyaxonSchemaError): CompiledOperationSpecification.apply_context(run_config)
def get_op_specification( config: Union[V1Component, V1Operation] = None, hub: str = None, params: Dict = None, profile: str = None, queue: str = None, nocache: bool = None, path_context: str = None, validate_params: bool = True, ) -> V1Operation: job_data = { "version": config.version if config else pkg.SCHEMA_VERSION, "kind": kinds.OPERATION, } if params: if not isinstance(params, Mapping): raise PolyaxonfileError( "Params: `{}` must be a valid mapping".format(params) ) job_data["params"] = params if profile: job_data["profile"] = profile if queue: # Check only get_queue_info(queue) job_data["queue"] = queue if nocache is not None: job_data["cache"] = {"disable": nocache} if config and config.kind == kinds.COMPONENT: job_data["component"] = config.to_dict() config = get_specification(data=[job_data]) elif config and config.kind == kinds.OPERATION: config = get_specification(data=[config.to_dict(), job_data]) elif hub: job_data["hubRef"] = hub config = get_specification(data=[job_data]) if hub and config.hub_ref is None: config.hub_ref = hub hub = config.hub_ref public_hub = config.has_public_hub_reference params = copy.deepcopy(config.params) # Sanity check if params were passed and we are not dealing with a hub component if validate_params and not (hub and not public_hub): run_config = OperationSpecification.compile_operation(config) run_config.validate_params(params=params, is_template=False) if run_config.is_dag_run: run_config.run.set_path_context(path_context) CompiledOperationSpecification.apply_operation_contexts(run_config) return config
def test_apply_context_raises_with_required_inputs(self): content = { "version": 1.1, "kind": "component", "inputs": [ {"name": "lr", "type": types.FLOAT}, {"name": "num_steps", "type": types.INT}, ], "run": { "kind": V1RunKind.JOB, "container": { "name": "polyaxon-main", "image": "test/test:latest", "command": "train", }, }, } component_config = V1Component.read(content) assert component_config.to_dict() == content content = { "version": 1.1, "kind": "compiled_operation", "inputs": [ {"name": "lr", "type": types.FLOAT}, {"name": "num_steps", "type": types.INT}, ], "run": { "kind": V1RunKind.JOB, "container": { "name": "polyaxon-main", "image": "test/test:latest", "command": "train", }, }, } run_config = V1CompiledOperation.read(content) # Raise because required inputs are not met with self.assertRaises(ValidationError): CompiledOperationSpecification.apply_operation_contexts(run_config) # Validation for template should pass validated_params = run_config.validate_params() assert {"lr": None, "num_steps": None} == { p.name: p.param.value for p in validated_params } # Validation for non template should raise with self.assertRaises(ValidationError): run_config.validate_params(is_template=False)
def test_job_file_with_termination_passes(self): run_config = CompiledOperationSpecification.read( [ os.path.abspath("tests/fixtures/plain/job_file_with_termination.yml"), {"kind": "compiled_operation"}, ] ) run_config = CompiledOperationSpecification.apply_operation_contexts(run_config) assert run_config.version == 1.1 assert isinstance(run_config.termination, V1Termination) assert run_config.termination.max_retries == 5 assert run_config.termination.timeout == 500 assert run_config.termination.ttl == 400 assert run_config.run.environment.restart_policy == "Never"
def test_matrix_file_passes_int_float_types(self): run_config = V1CompiledOperation.read( [ os.path.abspath( "tests/fixtures/pipelines/matrix_file_with_int_float_types.yml" ), {"kind": "compiled_operation"}, ] ) run_config = CompiledOperationSpecification.apply_context(run_config) assert run_config.version == 1.05 assert run_config.is_dag_run is True assert run_config.has_pipeline is True assert run_config.schedule is None assert run_config.run.concurrency == 4 assert isinstance(run_config.run, V1Dag) assert run_config.run.early_stopping is None assert run_config.run.kind == V1Dag.IDENTIFIER assert len(run_config.run.operations) == 2 assert len(run_config.run.components) == 1 template_grid = run_config.run.operations[1].parallel assert isinstance(template_grid, V1GridSearch) assert isinstance(template_grid.params["param1"], V1HpChoice) assert isinstance(template_grid.params["param2"], V1HpChoice) assert template_grid.params["param1"].to_dict() == { "kind": "choice", "value": [1, 2], } assert template_grid.params["param2"].to_dict() == { "kind": "choice", "value": [3.3, 4.4], } assert template_grid.concurrency == 2 assert template_grid.early_stopping is None
def test_mapping_early_stopping_file_passes(self): plx_file = check_polyaxonfile( polyaxonfile=os.path.abspath( "tests/fixtures/plain/mapping_job_file_early_stopping.yml"), is_cli=False, to_op=False, ) # Get compiled_operation data config_run = OperationSpecification.compile_operation(plx_file) config_run = CompiledOperationSpecification.apply_context(config_run) assert config_run.version == 1.05 assert isinstance(config_run.parallel, V1Mapping) assert config_run.parallel.values == [ { "lr": 0.001, "loss": "MeanSquaredError" }, { "lr": 0.1, "loss": "AbsoluteDifference" }, ] assert config_run.parallel.concurrency == 2 assert isinstance(config_run.parallel, V1Mapping) assert config_run.parallel.kind == V1Mapping.IDENTIFIER assert len(config_run.parallel.early_stopping) == 1 assert isinstance(config_run.parallel.early_stopping[0], V1MetricEarlyStopping)
def test_matrix_file_passes_int_float_types(self): plx_file = check_polyaxonfile( polyaxonfile=os.path.abspath( "tests/fixtures/plain/matrix_job_file_with_int_float_types.yml" ), is_cli=False, ) # Get compiled_operation data run_config = OperationSpecification.compile_operation(plx_file) run_config = CompiledOperationSpecification.apply_context(run_config) assert run_config.version == 1.05 assert isinstance(run_config.parallel, V1GridSearch) assert isinstance(run_config.parallel.params["param1"], V1HpChoice) assert isinstance(run_config.parallel.params["param2"], V1HpChoice) assert run_config.parallel.params["param1"].to_dict() == { "kind": "choice", "value": [1, 2], } assert run_config.parallel.params["param2"].to_dict() == { "kind": "choice", "value": [3.3, 4.4], } assert run_config.parallel.concurrency == 2 assert isinstance(run_config.parallel, V1GridSearch) assert run_config.parallel.kind == V1GridSearch.IDENTIFIER assert run_config.parallel.early_stopping is None
def test_matrix_early_stopping_file_passes(self): plx_file = check_polyaxonfile( polyaxonfile=os.path.abspath( "tests/fixtures/plain/matrix_job_file_early_stopping.yml"), is_cli=False, to_op=False, ) # Get compiled_operation data run_config = OperationSpecification.compile_operation(plx_file) run_config = CompiledOperationSpecification.apply_context(run_config) assert run_config.version == 1.05 assert isinstance(run_config.parallel, V1RandomSearch) assert isinstance(run_config.parallel.params["lr"], V1HpLinSpace) assert isinstance(run_config.parallel.params["loss"], V1HpChoice) assert run_config.parallel.params["lr"].to_dict() == { "kind": "linspace", "value": { "start": 0.01, "stop": 0.1, "num": 5 }, } assert run_config.parallel.params["loss"].to_dict() == { "kind": "choice", "value": ["MeanSquaredError", "AbsoluteDifference"], } assert run_config.parallel.concurrency == 2 assert run_config.parallel.num_runs == 300 assert isinstance(run_config.parallel, V1RandomSearch) assert run_config.parallel.kind == V1RandomSearch.IDENTIFIER assert len(run_config.parallel.early_stopping) == 1 assert isinstance(run_config.parallel.early_stopping[0], V1MetricEarlyStopping)
def test_matrix_file_with_required_inputs_passes(self): plx_file = check_polyaxonfile( polyaxonfile=os.path.abspath( "tests/fixtures/typing/matrix_job_required_inputs_file.yml"), is_cli=False, ) run_config = OperationSpecification.compile_operation(plx_file) run_config = CompiledOperationSpecification.apply_context(run_config) assert run_config.version == 1.1 assert isinstance(run_config.matrix, V1Hyperband) assert isinstance(run_config.matrix.params["lr"], V1HpLinSpace) assert isinstance(run_config.matrix.params["loss"], V1HpChoice) assert run_config.matrix.params["lr"].to_dict() == { "kind": "linspace", "value": { "start": 0.01, "stop": 0.1, "num": 5 }, } assert run_config.matrix.params["loss"].to_dict() == { "kind": "choice", "value": ["MeanSquaredError", "AbsoluteDifference"], } assert run_config.matrix.concurrency == 2 assert isinstance(run_config.matrix, V1Hyperband) assert run_config.matrix.kind == V1Hyperband.IDENTIFIER assert run_config.matrix.early_stopping is None
def test_dag_pipeline(self): run_config = V1CompiledOperation.read( [ os.path.abspath("tests/fixtures/pipelines/simple_dag_pipeline.yml"), {"kind": "compiled_operation"}, ] ) run_config = CompiledOperationSpecification.apply_context(run_config) assert len(run_config.run.operations) == 5 assert run_config.run.operations[0].name == "job1" assert run_config.run.operations[1].name == "experiment1" assert run_config.run.operations[1].dependencies == ["job1"] assert run_config.run.operations[2].name == "experiment2" assert run_config.run.operations[2].dependencies == ["job1"] assert run_config.run.operations[3].name == "experiment3" assert run_config.run.operations[3].dependencies == ["job1"] assert run_config.run.operations[4].name == "job2" assert run_config.run.operations[4].dependencies == [ "experiment1", "experiment2", "experiment3", ] dag_strategy = run_config.run sorted_dag = dag_strategy.sort_topologically(dag_strategy.dag) assert sorted_dag[0] == ["job1"] assert set(sorted_dag[1]) == {"experiment1", "experiment2", "experiment3"} assert sorted_dag[2] == ["job2"] assert run_config.run.concurrency == 3 assert run_config.schedule is None
def test_parallel_pipeline(self): run_config = V1CompiledOperation.read( [ os.path.abspath( "tests/fixtures/pipelines/simple_parallel_pipeline.yml" ), {"kind": "compiled_operation"}, ] ) run_config = CompiledOperationSpecification.apply_context(run_config) assert len(run_config.run.operations) == 4 assert run_config.run.operations[0].name == "job1" assert run_config.run.operations[0].dependencies is None assert run_config.run.operations[1].name == "job2" assert run_config.run.operations[1].dependencies is None assert run_config.run.operations[2].name == "experiment1" assert run_config.run.operations[2].dependencies is None assert run_config.run.operations[3].name == "experiment2" assert run_config.run.operations[3].dependencies is None dag_strategy = run_config.run assert set(dag_strategy.sort_topologically(dag_strategy.dag)[0]) == { "job1", "job2", "experiment1", "experiment2", } assert run_config.run.concurrency == 2 assert run_config.schedule is None
def test_sequential_pipeline(self): run_config = V1CompiledOperation.read( [ os.path.abspath( "tests/fixtures/pipelines/simple_sequential_pipeline.yml" ), {"kind": "compiled_operation"}, ] ) run_config = CompiledOperationSpecification.apply_context(run_config) assert run_config.run is not None assert len(run_config.run.operations) == 4 assert run_config.run.operations[0].name == "job1" assert run_config.run.operations[1].name == "job2" assert run_config.run.operations[1].dependencies == ["job1"] assert run_config.run.operations[2].name == "experiment1" assert run_config.run.operations[2].dependencies == ["job2"] assert run_config.run.operations[3].name == "experiment2" assert run_config.run.operations[3].dependencies == ["experiment1"] dag_strategy = run_config.run assert dag_strategy.sort_topologically(dag_strategy.dag) == [ ["job1"], ["job2"], ["experiment1"], ["experiment2"], ] assert run_config.schedule is None
def test_matrix_file_passes(self): run_config = V1CompiledOperation.read([ os.path.abspath("tests/fixtures/pipelines/matrix_file.yml"), { "kind": "compiled_operation" }, ]) run_config = CompiledOperationSpecification.apply_operation_contexts( run_config) assert run_config.version == 1.1 assert run_config.is_dag_run is True assert run_config.has_pipeline is True assert run_config.schedule is None assert run_config.run.concurrency == 4 assert isinstance(run_config.run, V1Dag) assert run_config.run.early_stopping is None assert run_config.run.kind == V1Dag.IDENTIFIER assert len(run_config.run.operations) == 2 assert len(run_config.run.components) == 1 template_hyperband = run_config.run.operations[1].matrix assert isinstance(template_hyperband.params["lr"], V1HpLinSpace) assert isinstance(template_hyperband.params["loss"], V1HpChoice) assert template_hyperband.params["lr"].to_dict() == { "kind": "linspace", "value": { "start": 0.01, "stop": 0.1, "num": 5 }, } assert template_hyperband.params["loss"].to_dict() == { "kind": "choice", "value": ["MeanSquaredError", "AbsoluteDifference"], } assert template_hyperband.params["normal_rate"].to_dict() == { "kind": "normal", "value": { "loc": 0, "scale": 0.9 }, } assert template_hyperband.params["dropout"].to_dict() == { "kind": "qloguniform", "value": { "high": 0.8, "low": 0, "q": 0.1 }, } assert template_hyperband.params["activation"].to_dict() == { "kind": "pchoice", "value": [["relu", 0.1], ["sigmoid", 0.8]], } assert template_hyperband.params["model"].to_dict() == { "kind": "choice", "value": ["CDNA", "DNA", "STP"], } assert template_hyperband.concurrency == 2 assert isinstance(template_hyperband, V1Hyperband)
def generate(polyaxonfile, python_module, build_context, destination, copy_path, params, track): """Generate a dockerfile given the polyaxonfile.""" from polyaxon.init.dockerfile import create_dockerfile_lineage from polyaxon.utils.hashing import hash_value if all([polyaxonfile, build_context]): Printer.print_error( "Only a polyaxonfile or a build context option is required.") sys.exit(1) if build_context: try: build_context = [ V1DockerfileType.from_dict(ConfigSpec.read_from(build_context)) ] except (PolyaxonSchemaError, ValidationError) as e: Printer.print_error("received a non valid build context.") Printer.print_error("Error message: {}.".format(e)) sys.exit(1) else: specification = check_polyaxonfile( polyaxonfile=polyaxonfile, python_module=python_module, params=params, log=False, ) try: compiled_operation = specification.compile_operation() compiled_operation.apply_params(params=specification.config.params) compiled_operation = CompiledOperationSpecification.apply_operation_contexts( compiled_operation) except PolyaxonSchemaError: Printer.print_error( "Could not run this polyaxonfile locally, " "a context is required to resolve it dependencies.") sys.exit(1) build_context = compiled_operation.init_dockerfiles for init_dockerfile in build_context: generator = DockerFileGenerator(build_context=init_dockerfile, destination=destination or ".") generator.create() Printer.print_success("Dockerfile was generated, path: `{}`".format( generator.dockerfile_path)) dockerfile_path = generator.dockerfile_path if copy_path: dockerfile_path = copy_file(dockerfile_path, copy_path) if track: hash_content = hash_value(init_dockerfile.to_dict()) create_dockerfile_lineage(dockerfile_path, summary={"hash": hash_content})
def test_no_params_for_required_inputs_outputs_raises(self): # Get compiled_operation data run_config = V1CompiledOperation.read([ os.path.abspath("tests/fixtures/typing/required_inputs.yml"), { "kind": "compiled_operation" }, ]) # Inputs don't have delayed validation by default with self.assertRaises(ValidationError): CompiledOperationSpecification.apply_context(run_config) run_config = V1CompiledOperation.read([ os.path.abspath("tests/fixtures/typing/required_outputs.yml"), { "kind": "compiled_operation" }, ]) # Outputs have delayed validation by default CompiledOperationSpecification.apply_context(run_config)
def get_op_specification( config: Union[V1Component, V1Operation] = None, hub=None, params=None, profile=None, queue=None, nocache=None, ) -> V1Operation: job_data = { "version": config.version if config else pkg.SCHEMA_VERSION, "kind": kinds.OPERATION } if params: if not isinstance(params, Mapping): raise PolyaxonfileError( "Params: `{}` must be a valid mapping".format(params)) job_data["params"] = params if profile: job_data["profile"] = profile if queue: job_data["queue"] = queue if nocache is not None: job_data["cache"] = {"disable": nocache} if hub: job_data["hubRef"] = hub config = get_specification(data=[job_data]) elif config.kind == kinds.OPERATION: config = get_specification(data=[config.to_dict(), job_data]) else: job_data["component"] = config.to_dict() config = get_specification(data=[job_data]) params = copy.deepcopy(config.params) # Sanity check if params were passed and we are not dealing with a hub component if not hub: run_config = OperationSpecification.compile_operation(config) run_config.validate_params(params=params, is_template=False) if run_config.is_dag_run: CompiledOperationSpecification.apply_context(run_config) return config
def test_matrix_file_passes(self): plx_file = check_polyaxonfile( polyaxonfile=os.path.abspath( "tests/fixtures/plain/matrix_job_file.yml"), is_cli=False, ) run_config = OperationSpecification.compile_operation(plx_file) run_config = CompiledOperationSpecification.apply_operation_contexts( run_config) assert run_config.version == 1.1 assert isinstance(run_config.matrix, V1Hyperband) assert isinstance(run_config.matrix.params["lr"], V1HpLinSpace) assert isinstance(run_config.matrix.params["loss"], V1HpChoice) assert run_config.matrix.params["lr"].to_dict() == { "kind": "linspace", "value": { "start": 0.01, "stop": 0.1, "num": 5 }, } assert run_config.matrix.params["loss"].to_dict() == { "kind": "choice", "value": ["MeanSquaredError", "AbsoluteDifference"], } assert run_config.matrix.params["normal_rate"].to_dict() == { "kind": "normal", "value": { "loc": 0, "scale": 0.9 }, } assert run_config.matrix.params["dropout"].to_dict() == { "kind": "qloguniform", "value": { "high": 0.8, "low": 0, "q": 0.1 }, } assert run_config.matrix.params["activation"].to_dict() == { "kind": "pchoice", "value": [["relu", 0.1], ["sigmoid", 0.8]], } assert run_config.matrix.params["model"].to_dict() == { "kind": "choice", "value": ["CDNA", "DNA", "STP"], } assert run_config.matrix.concurrency == 2 assert isinstance(run_config.matrix, V1Hyperband) assert run_config.matrix.kind == V1Hyperband.IDENTIFIER assert run_config.matrix.early_stopping is None
def test_cron_pipeline(self): plx_file = PolyaxonFile( os.path.abspath("tests/fixtures/pipelines/simple_cron_pipeline.yml") ) # Get compiled_operation data run_config = OperationSpecification.compile_operation(plx_file.config) run_config = CompiledOperationSpecification.apply_context(run_config) assert run_config.run is not None assert len(run_config.run.operations) == 1 assert run_config.run.operations[0].name == "cron-task" assert run_config.schedule is not None assert run_config.schedule.kind == "cron" assert run_config.schedule.cron == "0 0 * * *"