def get_ops_from_suggestions( content: str, compiled_operation: V1CompiledOperation, suggestions: List[Dict], ) -> List[V1Operation]: def has_param(k: str): if not compiled_operation.matrix: return None return not compiled_operation.matrix.has_param(k) op_content = V1Operation.read(content) for suggestion in suggestions: params = { k: V1Param(value=Parser.parse_expression(v, {}), context_only=has_param(k)) for (k, v) in suggestion.items() } op_spec = copy.deepcopy(op_content) op_spec.matrix = None op_spec.conditions = None op_spec.schedule = None op_spec.events = None op_spec.dependencies = None op_spec.trigger = None op_spec.skip_on_upstream_skip = None op_spec.cache = compiled_operation.cache op_spec.queue = compiled_operation.queue op_spec.params = params op_spec.component.inputs = compiled_operation.inputs op_spec.component.outputs = compiled_operation.outputs op_spec.component.contexts = compiled_operation.contexts yield op_spec
def get_ops_from_suggestions( content: str, compiled_operation: V1CompiledOperation, suggestions: List[Dict], ) -> List[V1Operation]: ops = [] for suggestion in suggestions: params = { k: V1Param(value=Parser.parse_expression(v, {})) for (k, v) in suggestion.items() } op_spec = V1Operation.read(content) op_spec.matrix = None op_spec.conditions = None op_spec.schedule = None op_spec.events = None op_spec.dependencies = None op_spec.trigger = None op_spec.skip_on_upstream_skip = None op_spec.cache = compiled_operation.cache op_spec.queue = compiled_operation.queue op_spec.params = params op_spec.component.inputs = compiled_operation.inputs op_spec.component.outputs = compiled_operation.outputs op_spec.component.contexts = compiled_operation.contexts ops.append(op_spec) return ops
def test_op_specification(self): config_dict = { "version": 1.1, "kind": "operation", "name": "foo", "description": "a description", "tags": ["value"], "params": {"param1": {"value": "foo"}, "param2": {"value": "bar"}}, "trigger": "all_succeeded", "component": { "name": "build-template", "tags": ["kaniko"], "run": { "kind": V1RunKind.JOB, "container": {"image": "test"}, "init": [ { "container": { "name": "polyaxon-init", "image": "foo", "args": "dev", } } ], "sidecars": [{"name": "foo", "image": "foo", "args": "dev"}], }, }, } op_config = V1Operation.read(values=config_dict) run_config = OperationSpecification.compile_operation(op_config) assert run_config.name == "foo" assert run_config.description == "a description" assert run_config.tags == ["kaniko", "value"] assert [ { "container": { "name": i.container.name, "image": i.container.image, "args": i.container.args, } } for i in run_config.run.init ] == [{"container": {"name": "polyaxon-init", "image": "foo", "args": "dev"}}] env = { "runPatch": { "container": { "resources": { "requests": {"gpu": 1, "tpu": 1}, "limits": {"gpu": 1, "tpu": 1}, } } } } run_config = OperationSpecification.compile_operation(op_config, env) assert ( run_config.run.container.resources == env["runPatch"]["container"]["resources"] )
def clone_run( run: BaseRun, cloning_kind: str, user_id: int = None, name: str = None, description: str = None, content: str = None, readme: str = None, tags: List[int] = None, ) -> BaseRun: op_spec = V1Operation.read(run.raw_content) compiled_operation, instance = operations.init_run( project_id=run.project_id, user_id=user_id or run.user_id, name=name or run.name, description=description or run.description, readme=readme or run.readme, op_spec=op_spec, original_id=run.id, original_uuid=run.uuid.hex, cloning_kind=cloning_kind, tags=tags or run.tags, override=content, override_post=True, ) instance.save() return instance
def resume_run( run: BaseRun, user_id: int = None, name: str = None, description: str = None, content: str = None, readme: str = None, tags: List[str] = None, ) -> BaseRun: op_spec = V1Operation.read(run.raw_content) compiled_operation, instance = operations.init_run( project_id=run.project_id, user_id=user_id or run.user_id, name=name or run.name, description=description or run.description, readme=readme or run.readme, op_spec=op_spec, tags=tags or run.tags, override=content, ) run.user_id = instance.user_id run.name = instance.name run.description = instance.description run.readme = instance.readme run.content = instance.content run.raw_content = instance.raw_content run.tags = instance.tags run.save() new_run_status( run, condition=V1StatusCondition.get_condition(type=V1Statuses.RESUMING, status=True), ) return run
def test_op_specification_with_override_info(self): config_dict = { "version": 1.1, "kind": "operation", "name": "foo", "description": "a description", "tags": ["value"], "params": {"param1": {"value": "foo"}, "param2": {"value": "bar"}}, "trigger": "all_succeeded", "component": { "name": "build-template", "tags": ["kaniko"], "run": { "kind": V1RunKind.JOB, "init": [ { "connection": "foo", "container": { "name": "polyaxon-init", "args": "--branch=dev", }, } ], "container": {"name": "polyaxon-main", "image": "test"}, }, }, } op_config = V1Operation.read(values=config_dict) assert op_config.name == "foo" assert op_config.description == "a description" assert op_config.tags == ["value"] run_config = OperationSpecification.compile_operation(op_config) assert run_config.name == "foo" assert run_config.description == "a description" assert run_config.tags == ["value"] assert [i.to_light_dict() for i in run_config.run.init] == [ { "connection": "foo", "container": {"name": "polyaxon-init", "args": "--branch=dev"}, } ] env = { "run": { "container": { "resources": { "requests": {"gpu": 1, "tpu": 1}, "limits": {"gpu": 1, "tpu": 1}, } } } } run_config = OperationSpecification.compile_operation(op_config, env) assert ( run_config.run.container.resources == env["run"]["container"]["resources"] )
def get_op_from_schedule( content: str, compiled_operation: V1CompiledOperation, ) -> V1Operation: op_spec = V1Operation.read(content) op_spec.conditions = None op_spec.schedule = None op_spec.events = None op_spec.dependencies = None op_spec.trigger = None op_spec.skip_on_upstream_skip = None op_spec.cache = compiled_operation.cache op_spec.queue = compiled_operation.queue op_spec.component.inputs = compiled_operation.inputs op_spec.component.outputs = compiled_operation.outputs op_spec.component.contexts = compiled_operation.contexts return op_spec
def get_ops_from_suggestions( content: str, compiled_operation: V1CompiledOperation, suggestions: List[Dict] ) -> List[V1Operation]: ops = [] for suggestion in suggestions: params = { k: V1Param(value=Parser.parse_expression(v, {})) for (k, v) in suggestion.items() } op_spec = V1Operation.read(content) op_spec.matrix = None # remove matrix op_spec.params = params op_spec.component.inputs = compiled_operation.inputs op_spec.component.outputs = compiled_operation.outputs ops.append(op_spec) return ops
def test_op_specification_with_nocache(self): config_dict = { "version": 1.1, "kind": "operation", "name": "foo", "description": "a description", "tags": ["value"], "cache": {"disable": True, "ttl": 12}, "params": {"param1": {"value": "foo"}, "param2": {"value": "bar"}}, "trigger": "all_succeeded", "component": { "name": "build-template", "tags": ["kaniko"], "run": { "kind": V1RunKind.JOB, "container": {"name": "polyaxon-main", "image": "test"}, "init": [{"connection": "some-connection"}], }, }, } op_config = V1Operation.read(values=config_dict) run_config = OperationSpecification.compile_operation(op_config) assert run_config.name == "foo" assert run_config.description == "a description" assert run_config.tags == ["kaniko", "value"] assert run_config.cache.to_dict() == {"disable": True, "ttl": 12} assert [i.to_light_dict() for i in run_config.run.init] == [ {"connection": "some-connection"} ] env = { "runPatch": { "container": { "resources": { "requests": {"gpu": 1, "tpu": 1}, "limits": {"gpu": 1, "tpu": 1}, } } } } run_config = OperationSpecification.compile_operation(op_config, env) assert ( run_config.run.container.resources == env["runPatch"]["container"]["resources"] )
def clone_run( run: BaseRun, cloning_kind: str, user_id: int = None, name: str = None, description: str = None, content: str = None, readme: str = None, tags: List[int] = None, supported_kinds: Set[str] = None, **kwargs, ) -> BaseRun: op_spec = V1Operation.read(run.raw_content) meta_info = kwargs.pop("meta_info", {}) or {} original_meta_info = run.meta_info or {} original_uuid = run.uuid.hex upload_artifacts = original_meta_info.get(META_UPLOAD_ARTIFACTS) if upload_artifacts: meta_info[META_UPLOAD_ARTIFACTS] = upload_artifacts if cloning_kind == V1CloningKind.COPY and META_COPY_ARTIFACTS not in meta_info: # Handle default copy mode meta_info[META_COPY_ARTIFACTS] = V1ArtifactsType( dirs=[original_uuid]).to_dict() if META_COPY_ARTIFACTS not in meta_info and upload_artifacts: # Handle default copy mode meta_info[META_COPY_ARTIFACTS] = V1ArtifactsType( dirs=["{}/{}".format(original_uuid, upload_artifacts)]).to_dict() compiled_operation, instance = operations.init_run( project_id=run.project_id, user_id=user_id or run.user_id, name=name or run.name, description=description or run.description, readme=readme or run.readme, op_spec=op_spec, original_id=run.id, original_uuid=original_uuid, cloning_kind=cloning_kind, tags=tags or run.tags, override=content, supported_kinds=supported_kinds, meta_info=meta_info, **kwargs, ) instance.save() return instance