def get_ops_from_suggestions(
content: str,
compiled_operation: V1CompiledOperation,
suggestions: List[Dict],
) -> List[V1Operation]:
def has_param(k: str):
if not compiled_operation.matrix:
return None
return not compiled_operation.matrix.has_param(k)
op_content = V1Operation.read(content)
for suggestion in suggestions:
params = {
k: V1Param(value=Parser.parse_expression(v, {}),
context_only=has_param(k))
for (k, v) in suggestion.items()
}
op_spec = copy.deepcopy(op_content)
op_spec.matrix = None
op_spec.conditions = None
op_spec.schedule = None
op_spec.events = None
op_spec.dependencies = None
op_spec.trigger = None
op_spec.skip_on_upstream_skip = None
op_spec.cache = compiled_operation.cache
op_spec.queue = compiled_operation.queue
op_spec.params = params
op_spec.component.inputs = compiled_operation.inputs
op_spec.component.outputs = compiled_operation.outputs
op_spec.component.contexts = compiled_operation.contexts
yield op_spec
def get_ops_from_suggestions(
content: str,
compiled_operation: V1CompiledOperation,
suggestions: List[Dict],
) -> List[V1Operation]:
ops = []
for suggestion in suggestions:
params = {
k: V1Param(value=Parser.parse_expression(v, {}))
for (k, v) in suggestion.items()
}
op_spec = V1Operation.read(content)
op_spec.matrix = None
op_spec.conditions = None
op_spec.schedule = None
op_spec.events = None
op_spec.dependencies = None
op_spec.trigger = None
op_spec.skip_on_upstream_skip = None
op_spec.cache = compiled_operation.cache
op_spec.queue = compiled_operation.queue
op_spec.params = params
op_spec.component.inputs = compiled_operation.inputs
op_spec.component.outputs = compiled_operation.outputs
op_spec.component.contexts = compiled_operation.contexts
ops.append(op_spec)
return ops
def test_op_specification(self):
config_dict = {
"version": 1.1,
"kind": "operation",
"name": "foo",
"description": "a description",
"tags": ["value"],
"params": {"param1": {"value": "foo"}, "param2": {"value": "bar"}},
"trigger": "all_succeeded",
"component": {
"name": "build-template",
"tags": ["kaniko"],
"run": {
"kind": V1RunKind.JOB,
"container": {"image": "test"},
"init": [
{
"container": {
"name": "polyaxon-init",
"image": "foo",
"args": "dev",
}
}
],
"sidecars": [{"name": "foo", "image": "foo", "args": "dev"}],
},
},
}
op_config = V1Operation.read(values=config_dict)
run_config = OperationSpecification.compile_operation(op_config)
assert run_config.name == "foo"
assert run_config.description == "a description"
assert run_config.tags == ["kaniko", "value"]
assert [
{
"container": {
"name": i.container.name,
"image": i.container.image,
"args": i.container.args,
}
}
for i in run_config.run.init
] == [{"container": {"name": "polyaxon-init", "image": "foo", "args": "dev"}}]
env = {
"runPatch": {
"container": {
"resources": {
"requests": {"gpu": 1, "tpu": 1},
"limits": {"gpu": 1, "tpu": 1},
}
}
}
}
run_config = OperationSpecification.compile_operation(op_config, env)
assert (
run_config.run.container.resources
== env["runPatch"]["container"]["resources"]
)
def clone_run(
run: BaseRun,
cloning_kind: str,
user_id: int = None,
name: str = None,
description: str = None,
content: str = None,
readme: str = None,
tags: List[int] = None,
) -> BaseRun:
op_spec = V1Operation.read(run.raw_content)
compiled_operation, instance = operations.init_run(
project_id=run.project_id,
user_id=user_id or run.user_id,
name=name or run.name,
description=description or run.description,
readme=readme or run.readme,
op_spec=op_spec,
original_id=run.id,
original_uuid=run.uuid.hex,
cloning_kind=cloning_kind,
tags=tags or run.tags,
override=content,
override_post=True,
)
instance.save()
return instance
def resume_run(
run: BaseRun,
user_id: int = None,
name: str = None,
description: str = None,
content: str = None,
readme: str = None,
tags: List[str] = None,
) -> BaseRun:
op_spec = V1Operation.read(run.raw_content)
compiled_operation, instance = operations.init_run(
project_id=run.project_id,
user_id=user_id or run.user_id,
name=name or run.name,
description=description or run.description,
readme=readme or run.readme,
op_spec=op_spec,
tags=tags or run.tags,
override=content,
)
run.user_id = instance.user_id
run.name = instance.name
run.description = instance.description
run.readme = instance.readme
run.content = instance.content
run.raw_content = instance.raw_content
run.tags = instance.tags
run.save()
new_run_status(
run,
condition=V1StatusCondition.get_condition(type=V1Statuses.RESUMING,
status=True),
)
return run
def test_op_specification_with_override_info(self):
config_dict = {
"version": 1.1,
"kind": "operation",
"name": "foo",
"description": "a description",
"tags": ["value"],
"params": {"param1": {"value": "foo"}, "param2": {"value": "bar"}},
"trigger": "all_succeeded",
"component": {
"name": "build-template",
"tags": ["kaniko"],
"run": {
"kind": V1RunKind.JOB,
"init": [
{
"connection": "foo",
"container": {
"name": "polyaxon-init",
"args": "--branch=dev",
},
}
],
"container": {"name": "polyaxon-main", "image": "test"},
},
},
}
op_config = V1Operation.read(values=config_dict)
assert op_config.name == "foo"
assert op_config.description == "a description"
assert op_config.tags == ["value"]
run_config = OperationSpecification.compile_operation(op_config)
assert run_config.name == "foo"
assert run_config.description == "a description"
assert run_config.tags == ["value"]
assert [i.to_light_dict() for i in run_config.run.init] == [
{
"connection": "foo",
"container": {"name": "polyaxon-init", "args": "--branch=dev"},
}
]
env = {
"run": {
"container": {
"resources": {
"requests": {"gpu": 1, "tpu": 1},
"limits": {"gpu": 1, "tpu": 1},
}
}
}
}
run_config = OperationSpecification.compile_operation(op_config, env)
assert (
run_config.run.container.resources == env["run"]["container"]["resources"]
)
def get_op_from_schedule(
content: str,
compiled_operation: V1CompiledOperation,
) -> V1Operation:
op_spec = V1Operation.read(content)
op_spec.conditions = None
op_spec.schedule = None
op_spec.events = None
op_spec.dependencies = None
op_spec.trigger = None
op_spec.skip_on_upstream_skip = None
op_spec.cache = compiled_operation.cache
op_spec.queue = compiled_operation.queue
op_spec.component.inputs = compiled_operation.inputs
op_spec.component.outputs = compiled_operation.outputs
op_spec.component.contexts = compiled_operation.contexts
return op_spec
def get_ops_from_suggestions(
content: str, compiled_operation: V1CompiledOperation, suggestions: List[Dict]
) -> List[V1Operation]:
ops = []
for suggestion in suggestions:
params = {
k: V1Param(value=Parser.parse_expression(v, {}))
for (k, v) in suggestion.items()
}
op_spec = V1Operation.read(content)
op_spec.matrix = None # remove matrix
op_spec.params = params
op_spec.component.inputs = compiled_operation.inputs
op_spec.component.outputs = compiled_operation.outputs
ops.append(op_spec)
return ops
def test_op_specification_with_nocache(self):
config_dict = {
"version": 1.1,
"kind": "operation",
"name": "foo",
"description": "a description",
"tags": ["value"],
"cache": {"disable": True, "ttl": 12},
"params": {"param1": {"value": "foo"}, "param2": {"value": "bar"}},
"trigger": "all_succeeded",
"component": {
"name": "build-template",
"tags": ["kaniko"],
"run": {
"kind": V1RunKind.JOB,
"container": {"name": "polyaxon-main", "image": "test"},
"init": [{"connection": "some-connection"}],
},
},
}
op_config = V1Operation.read(values=config_dict)
run_config = OperationSpecification.compile_operation(op_config)
assert run_config.name == "foo"
assert run_config.description == "a description"
assert run_config.tags == ["kaniko", "value"]
assert run_config.cache.to_dict() == {"disable": True, "ttl": 12}
assert [i.to_light_dict() for i in run_config.run.init] == [
{"connection": "some-connection"}
]
env = {
"runPatch": {
"container": {
"resources": {
"requests": {"gpu": 1, "tpu": 1},
"limits": {"gpu": 1, "tpu": 1},
}
}
}
}
run_config = OperationSpecification.compile_operation(op_config, env)
assert (
run_config.run.container.resources
== env["runPatch"]["container"]["resources"]
)
def clone_run(
run: BaseRun,
cloning_kind: str,
user_id: int = None,
name: str = None,
description: str = None,
content: str = None,
readme: str = None,
tags: List[int] = None,
supported_kinds: Set[str] = None,
**kwargs,
) -> BaseRun:
op_spec = V1Operation.read(run.raw_content)
meta_info = kwargs.pop("meta_info", {}) or {}
original_meta_info = run.meta_info or {}
original_uuid = run.uuid.hex
upload_artifacts = original_meta_info.get(META_UPLOAD_ARTIFACTS)
if upload_artifacts:
meta_info[META_UPLOAD_ARTIFACTS] = upload_artifacts
if cloning_kind == V1CloningKind.COPY and META_COPY_ARTIFACTS not in meta_info:
# Handle default copy mode
meta_info[META_COPY_ARTIFACTS] = V1ArtifactsType(
dirs=[original_uuid]).to_dict()
if META_COPY_ARTIFACTS not in meta_info and upload_artifacts:
# Handle default copy mode
meta_info[META_COPY_ARTIFACTS] = V1ArtifactsType(
dirs=["{}/{}".format(original_uuid, upload_artifacts)]).to_dict()
compiled_operation, instance = operations.init_run(
project_id=run.project_id,
user_id=user_id or run.user_id,
name=name or run.name,
description=description or run.description,
readme=readme or run.readme,
op_spec=op_spec,
original_id=run.id,
original_uuid=original_uuid,
cloning_kind=cloning_kind,
tags=tags or run.tags,
override=content,
supported_kinds=supported_kinds,
meta_info=meta_info,
**kwargs,
)
instance.save()
return instance
