def resume_run(
run: BaseRun,
user_id: int = None,
name: str = None,
description: str = None,
content: str = None,
readme: str = None,
tags: List[str] = None,
) -> BaseRun:
op_spec = V1Operation.read(run.raw_content)
compiled_operation, instance = operations.init_run(
project_id=run.project_id,
user_id=user_id or run.user_id,
name=name or run.name,
description=description or run.description,
readme=readme or run.readme,
op_spec=op_spec,
tags=tags or run.tags,
override=content,
)
run.user_id = instance.user_id
run.name = instance.name
run.description = instance.description
run.readme = instance.readme
run.content = instance.content
run.raw_content = instance.raw_content
run.tags = instance.tags
run.save()
new_run_status(
run,
condition=V1StatusCondition.get_condition(type=V1Statuses.RESUMING,
status=True),
)
return run
def get_ops_from_suggestions(
content: str,
compiled_operation: V1CompiledOperation,
suggestions: List[Dict],
) -> List[V1Operation]:
ops = []
for suggestion in suggestions:
params = {
k: V1Param(value=Parser.parse_expression(v, {}))
for (k, v) in suggestion.items()
}
op_spec = V1Operation.read(content)
op_spec.matrix = None
op_spec.conditions = None
op_spec.schedule = None
op_spec.events = None
op_spec.dependencies = None
op_spec.trigger = None
op_spec.skip_on_upstream_skip = None
op_spec.cache = compiled_operation.cache
op_spec.queue = compiled_operation.queue
op_spec.params = params
op_spec.component.inputs = compiled_operation.inputs
op_spec.component.outputs = compiled_operation.outputs
op_spec.component.contexts = compiled_operation.contexts
ops.append(op_spec)
return ops
def test_patch_pre_merge_full_values_with_empty_preset(self):
operation = self.get_full_operation()
tmp_operation = self.get_full_operation()
result = tmp_operation.patch(V1Operation(is_preset=True),
strategy=V1PatchStrategy.PRE_MERGE)
assert result.to_dict() == operation.to_dict()
tmp_operation = self.get_full_operation()
preset = self.get_empty_preset()
result = tmp_operation.patch(preset,
strategy=V1PatchStrategy.PRE_MERGE)
result_dict = result.to_dict()
# Since there's no component to validate the runPatch section it stays the same
assert result_dict == operation.to_dict()
operation = self.get_full_operation_with_component()
tmp_operation = self.get_full_operation_with_component()
preset = self.get_empty_preset()
result = tmp_operation.patch(preset,
strategy=V1PatchStrategy.PRE_MERGE)
result_dict = result.to_dict()
assert result_dict["runPatch"]["container"].pop(
"name") == MAIN_JOB_CONTAINER
# Run patch was validated and merged
assert result_dict == operation.to_dict()
def clone_run(
run: BaseRun,
cloning_kind: str,
user_id: int = None,
name: str = None,
description: str = None,
content: str = None,
readme: str = None,
tags: List[int] = None,
) -> BaseRun:
op_spec = V1Operation.read(run.raw_content)
compiled_operation, instance = operations.init_run(
project_id=run.project_id,
user_id=user_id or run.user_id,
name=name or run.name,
description=description or run.description,
readme=readme or run.readme,
op_spec=op_spec,
original_id=run.id,
original_uuid=run.uuid.hex,
cloning_kind=cloning_kind,
tags=tags or run.tags,
override=content,
override_post=True,
)
instance.save()
return instance
def collect_references(config: V1Operation, path_context: str = None):
if config.has_component_reference:
return config
elif config.has_hub_reference:
component = ConfigSpec.get_from(config.hub_ref, "hub").read()
elif config.has_url_reference:
component = ConfigSpec.get_from(config.url_ref, "url").read()
elif config.has_path_reference:
path_ref = config.path_ref
if path_context:
path_ref = os.path.join(
os.path.dirname(os.path.abspath(path_context)), path_ref)
component = ConfigSpec.get_from(path_ref).read()
else:
raise PolyaxonfileError("Operation found without component")
component = get_specification(data=component)
if component.kind != kinds.COMPONENT:
if config.has_url_reference:
ref_type = "Url ref"
ref = config.url_ref
else:
ref_type = "Path ref"
ref = config.path_ref
raise PolyaxonfileError(
"the reference ({}) `{}` is of kind `{}`, it should be a `{}`".
format(ref, ref_type, component.kind, kinds.COMPONENT))
config.component = component
if component.is_dag_run:
component.run.collect_components()
return config
def test_patch_replace_full_values_with_empty_preset(self):
operation = self.get_full_operation()
tmp_operation = self.get_full_operation()
result = tmp_operation.patch(
V1Operation(is_preset=True), strategy=V1PatchStrategy.REPLACE
)
assert result.to_dict() == operation.to_dict()
tmp_operation = self.get_full_operation()
preset = self.get_empty_preset()
result = tmp_operation.patch(preset, strategy=V1PatchStrategy.REPLACE)
result_dict = result.to_dict()
assert result_dict.pop("hubRef") == operation.hub_ref
assert result_dict.pop("name") == operation.name
assert result_dict.pop("trigger") == operation.trigger
assert result_dict.pop("conditions") == operation.conditions
assert result_dict.pop("skipOnUpstreamSkip") == operation.skip_on_upstream_skip
assert result_dict.pop("schedule") == operation.schedule.to_dict()
assert result_dict.pop("conditions", None) is None
assert result_dict.pop("matrix") == operation.matrix.to_dict()
assert result_dict.pop("cache") == operation.cache.to_dict()
assert result_dict.pop("plugins") == operation.plugins.to_dict()
assert result_dict.pop("termination") == operation.termination.to_dict()
expected = preset.to_dict()
expected.pop("isPreset")
expected.pop("cache")
expected.pop("plugins")
expected.pop("termination")
assert result_dict == expected
def get_ops_from_suggestions(
content: str,
compiled_operation: V1CompiledOperation,
suggestions: List[Dict],
) -> List[V1Operation]:
def has_param(k: str):
if not compiled_operation.matrix:
return None
return not compiled_operation.matrix.has_param(k)
op_content = V1Operation.read(content)
for suggestion in suggestions:
params = {
k: V1Param(value=Parser.parse_expression(v, {}),
context_only=has_param(k))
for (k, v) in suggestion.items()
}
op_spec = copy.deepcopy(op_content)
op_spec.matrix = None
op_spec.conditions = None
op_spec.schedule = None
op_spec.events = None
op_spec.dependencies = None
op_spec.trigger = None
op_spec.skip_on_upstream_skip = None
op_spec.cache = compiled_operation.cache
op_spec.queue = compiled_operation.queue
op_spec.params = params
op_spec.component.inputs = compiled_operation.inputs
op_spec.component.outputs = compiled_operation.outputs
op_spec.component.contexts = compiled_operation.contexts
yield op_spec
def test_op_specification(self):
config_dict = {
"version": 1.1,
"kind": "operation",
"name": "foo",
"description": "a description",
"tags": ["value"],
"params": {"param1": {"value": "foo"}, "param2": {"value": "bar"}},
"trigger": "all_succeeded",
"component": {
"name": "build-template",
"tags": ["kaniko"],
"run": {
"kind": V1RunKind.JOB,
"container": {"image": "test"},
"init": [
{
"container": {
"name": "polyaxon-init",
"image": "foo",
"args": "dev",
}
}
],
"sidecars": [{"name": "foo", "image": "foo", "args": "dev"}],
},
},
}
op_config = V1Operation.read(values=config_dict)
run_config = OperationSpecification.compile_operation(op_config)
assert run_config.name == "foo"
assert run_config.description == "a description"
assert run_config.tags == ["kaniko", "value"]
assert [
{
"container": {
"name": i.container.name,
"image": i.container.image,
"args": i.container.args,
}
}
for i in run_config.run.init
] == [{"container": {"name": "polyaxon-init", "image": "foo", "args": "dev"}}]
env = {
"runPatch": {
"container": {
"resources": {
"requests": {"gpu": 1, "tpu": 1},
"limits": {"gpu": 1, "tpu": 1},
}
}
}
}
run_config = OperationSpecification.compile_operation(op_config, env)
assert (
run_config.run.container.resources
== env["runPatch"]["container"]["resources"]
)
def test_op_specification_with_override_info(self):
config_dict = {
"version": 1.1,
"kind": "operation",
"name": "foo",
"description": "a description",
"tags": ["value"],
"params": {"param1": {"value": "foo"}, "param2": {"value": "bar"}},
"trigger": "all_succeeded",
"component": {
"name": "build-template",
"tags": ["kaniko"],
"run": {
"kind": V1RunKind.JOB,
"init": [
{
"connection": "foo",
"container": {
"name": "polyaxon-init",
"args": "--branch=dev",
},
}
],
"container": {"name": "polyaxon-main", "image": "test"},
},
},
}
op_config = V1Operation.read(values=config_dict)
assert op_config.name == "foo"
assert op_config.description == "a description"
assert op_config.tags == ["value"]
run_config = OperationSpecification.compile_operation(op_config)
assert run_config.name == "foo"
assert run_config.description == "a description"
assert run_config.tags == ["value"]
assert [i.to_light_dict() for i in run_config.run.init] == [
{
"connection": "foo",
"container": {"name": "polyaxon-init", "args": "--branch=dev"},
}
]
env = {
"run": {
"container": {
"resources": {
"requests": {"gpu": 1, "tpu": 1},
"limits": {"gpu": 1, "tpu": 1},
}
}
}
}
run_config = OperationSpecification.compile_operation(op_config, env)
assert (
run_config.run.container.resources == env["run"]["container"]["resources"]
)
def create(
self,
name: str = None,
description: str = None,
tags: Union[str, Sequence[str]] = None,
content: Union[str, Dict, V1Operation] = None,
):
"""Creates a new run based on the data passed.
N.B. Create methods are only useful if you want to create a run programmatically,
if you run a component/operation from the CLI/UI an instance will be created automatically.
This is a generic create function, you can check other methods for creating runs:
* from yaml: `create_from_polyaxonfile`
* from url: `create_from_url`
* from hub: `create_from_hub`
> Note that if you don't pass `content`, the creation will pass,
and the run will be marked as non-managed.
[Run API](/docs/api/#operation/CreateRun)
Args:
name: str, optional, it will override the name in the operation if provided.
description: str, optional,
it will override the description in the operation if provided.
tags: str or List[str], optional, list of tags,
it will override the tags in the operation if provided.
content: str or Dict or V1Operation, optional.
Returns:
V1Run, run instance from the response.
"""
is_managed = True
if not content:
is_managed = False
elif not isinstance(content, (str, Mapping, V1Operation)):
raise PolyaxonClientException(
"Received an invalid content: {}".format(content)
)
if content:
if isinstance(content, Mapping):
content = V1Operation.from_dict(content)
content = (
content if isinstance(content, str) else content.to_dict(dump=True)
)
data = polyaxon_sdk.V1OperationBody(
name=name,
description=description,
tags=tags,
content=content,
is_managed=is_managed,
)
self._create(data=data, async_req=False)
self._post_create()
return self.run_data
def generate_run_data(cls,
config: V1Operation,
override=None,
override_post=True):
op_config = config.to_light_dict()
name = None
if config.component:
name = config.component.get_name()
component_config = op_config.pop("component", {})
if name:
component_config["name"] = name
values = [
{
"version": config.version
},
component_config,
{
"kind": kinds.COMPILED_OPERATION
},
]
op_override = {}
for field in [
cls.NAME,
cls.DESCRIPTION,
cls.TAGS,
cls.PROFILE,
cls.QUEUE,
cls.CACHE,
cls.PLUGINS,
cls.TERMINATION,
cls.PARALLEL,
cls.SCHEDULE,
cls.DEPENDENCIES,
cls.TRIGGER,
cls.CONDITIONS,
cls.SKIP_ON_UPSTREAM_SKIP,
cls.HUB_REF,
]:
override_field = op_config.get(field)
if override_field:
op_override[field] = override_field
# Patch run
run_patch = op_config.get(cls.RUN_PATCH)
if run_patch:
op_override[cls.RUN] = run_patch
if override_post:
if op_override:
values.append(op_override)
if override:
values.append(override)
else:
if override:
values.append(override)
if op_override:
values.append(op_override)
return values
def get_tuner(
name: str,
container: V1Container,
matrix: V1Matrix,
search: V1ParamSearch,
iteration: int,
bracket_iteration: int = None,
) -> V1Operation:
params = {
"matrix": V1Param(value=matrix.to_dict()),
"search": V1Param(value=search.to_dict()),
"iteration": V1Param(value=iteration),
}
inputs = [
V1IO(name="matrix", iotype=types.DICT, is_list=False,
is_optional=True),
V1IO(name="search", iotype=types.DICT, is_list=False,
is_optional=True),
V1IO(name="iteration",
iotype=types.INT,
is_list=False,
is_optional=True),
]
if bracket_iteration is not None:
params["bracket_iteration"] = V1Param(value=bracket_iteration)
inputs.append(
V1IO(
name="bracket_iteration",
iotype=types.INT,
is_list=False,
is_optional=True,
))
return V1Operation(
params=params,
component=V1Component(
name=name,
plugins=V1Plugins(
auth=True,
collect_logs=True,
collect_artifacts=True,
collect_resources=False,
sync_statuses=False,
),
inputs=inputs,
outputs=[
V1IO(
name="suggestions",
iotype=types.DICT,
is_list=True,
is_optional=False,
),
],
run=V1Tuner(container=container, ),
),
)
def test_patch_isnull_full_values_with_empty_preset(self):
operation = self.get_full_operation()
tmp_operation = self.get_full_operation()
result = tmp_operation.patch(V1Operation(is_preset=True),
strategy=V1PatchStrategy.ISNULL)
assert result.to_dict() == operation.to_dict()
tmp_operation = self.get_full_operation()
preset = self.get_empty_preset()
result = tmp_operation.patch(preset, strategy=V1PatchStrategy.ISNULL)
assert result.to_dict() == operation.to_dict()
def compile_operation(cls,
config: V1Operation,
override: Dict = None) -> V1CompiledOperation:
if override:
preset = OperationSpecification.read(override, is_preset=True)
config = config.patch(preset, preset.patch_strategy)
# Patch run
component = config.component # type: V1Component
if config.run_patch:
component.run = component.run.patch(
validate_run_patch(config.run_patch, component.run.kind),
strategy=config.patch_strategy,
)
# Gather contexts io
config_params = config.params or {}
contexts = [
V1IO(name=p) for p in config_params
if config_params[p].context_only
]
patch_compiled = V1CompiledOperation(
name=config.name,
description=config.description,
contexts=contexts,
tags=config.tags,
presets=config.presets,
queue=config.queue,
cache=config.cache,
hooks=config.hooks,
actions=config.actions,
events=config.events,
plugins=config.plugins,
termination=config.termination,
matrix=config.matrix,
schedule=config.schedule,
dependencies=config.dependencies,
trigger=config.trigger,
conditions=config.conditions,
skip_on_upstream_skip=config.skip_on_upstream_skip,
)
values = [
{
cls.VERSION: config.version
},
component.to_dict(),
{
cls.KIND: kinds.COMPILED_OPERATION
},
]
compiled = V1CompiledOperation.read(
values) # type: V1CompiledOperation
return compiled.patch(patch_compiled, strategy=config.patch_strategy)
def collect_references(config: V1Operation):
if config.has_component_reference or config.has_hub_reference:
return config
elif config.has_url_reference:
component = ConfigSpec.get_from(config.url_ref, "url").read()
elif config.has_path_reference:
component = ConfigSpec.get_from(config.path_ref).read()
else:
raise PolyaxonfileError("Operation found without component")
component = get_specification(data=component)
config.component = component
return config
def create(
self,
name: str = None,
description: str = None,
tags: Union[str, Sequence[str]] = None,
content: Union[str, Dict, V1Operation] = None,
):
"""Creates a new run based on the data passed.
This is a generic create function, you can check other methods for creating runs:
* from yaml
* from hub
* from url
Note that if you don't pass data, the creation will pass,
and the run will be marked as non-managed.
[Run API](/docs/api/#operation/CreateRun)
Args:
name: str, optional, name
note it will override the name in the operation if available.
description: str, optional, description
note it will override the description in the operation if available.
tags: str or List[str], optional, list of tags,
note it will override the tags in the operation if available.
content: str or Dict or V1Operation, optional.
Returns:
V1Run, run instance from the response.
"""
is_managed = True
if not content:
is_managed = False
elif not isinstance(content, (str, Mapping, V1Operation)):
raise PolyaxonClientException(
"Received an invalid content: {}".format(content))
if content:
if isinstance(content, Mapping):
content = V1Operation.from_dict(content)
content = (content if isinstance(content, str) else
content.to_dict(dump=True))
data = polyaxon_sdk.V1OperationBody(
name=name,
description=description,
tags=tags,
content=content,
is_managed=is_managed,
)
self._create(data=data, async_req=False)
self._post_create()
def get_tuner(
name: str,
container: V1Container,
matrix: V1Matrix,
configs: List[Dict],
metrics: List[float],
iteration: int,
) -> V1Operation:
return V1Operation(
params={
"configs": V1Param(value=configs),
"metrics": V1Param(value=metrics),
"matrix": V1Param(value=matrix),
"iteration": V1Param(value=iteration),
},
termination=V1Termination(max_retries=3),
component=V1Component(
name=name,
plugins=V1Plugins(
auth=True,
collect_logs=False,
collect_artifacts=False,
collect_resources=False,
sync_statuses=True,
),
inputs=[
V1IO(name="configs",
iotype=types.DICT,
is_list=True,
is_optional=False),
V1IO(name="metrics",
iotype=types.FLOAT,
is_list=True,
is_optional=False),
V1IO(name="iteration",
iotype=types.INT,
is_list=True,
is_optional=True),
],
outputs=[
V1IO(
name="suggestions",
iotype=types.DICT,
is_list=True,
is_optional=False,
),
],
run=V1Tuner(container=container, ),
),
)
def test_patch_isnull_empty_values_with_full_preset(self):
operation = self.get_empty_operation()
tmp_operation = self.get_empty_operation()
result = tmp_operation.patch(V1Operation(is_preset=True),
strategy=V1PatchStrategy.ISNULL)
assert result.to_dict() == operation.to_dict()
tmp_operation = self.get_empty_operation()
preset = self.get_empty_preset()
result = tmp_operation.patch(preset, strategy=V1PatchStrategy.ISNULL)
result_dict = result.to_dict()
assert result_dict.pop("hubRef") == operation.hub_ref
expected = preset.to_dict()
expected.pop("isPreset")
assert result_dict == expected
def clone_run(
run: BaseRun,
cloning_kind: str,
user_id: int = None,
name: str = None,
description: str = None,
content: str = None,
readme: str = None,
tags: List[int] = None,
supported_kinds: Set[str] = None,
**kwargs,
) -> BaseRun:
op_spec = V1Operation.read(run.raw_content)
meta_info = kwargs.pop("meta_info", {}) or {}
original_meta_info = run.meta_info or {}
original_uuid = run.uuid.hex
upload_artifacts = original_meta_info.get(META_UPLOAD_ARTIFACTS)
if upload_artifacts:
meta_info[META_UPLOAD_ARTIFACTS] = upload_artifacts
if cloning_kind == V1CloningKind.COPY and META_COPY_ARTIFACTS not in meta_info:
# Handle default copy mode
meta_info[META_COPY_ARTIFACTS] = V1ArtifactsType(
dirs=[original_uuid]).to_dict()
if META_COPY_ARTIFACTS not in meta_info and upload_artifacts:
# Handle default copy mode
meta_info[META_COPY_ARTIFACTS] = V1ArtifactsType(
dirs=["{}/{}".format(original_uuid, upload_artifacts)]).to_dict()
compiled_operation, instance = operations.init_run(
project_id=run.project_id,
user_id=user_id or run.user_id,
name=name or run.name,
description=description or run.description,
readme=readme or run.readme,
op_spec=op_spec,
original_id=run.id,
original_uuid=original_uuid,
cloning_kind=cloning_kind,
tags=tags or run.tags,
override=content,
supported_kinds=supported_kinds,
meta_info=meta_info,
**kwargs,
)
instance.save()
return instance
def get_op_from_schedule(
content: str,
compiled_operation: V1CompiledOperation,
) -> V1Operation:
op_spec = V1Operation.read(content)
op_spec.conditions = None
op_spec.schedule = None
op_spec.events = None
op_spec.dependencies = None
op_spec.trigger = None
op_spec.skip_on_upstream_skip = None
op_spec.cache = compiled_operation.cache
op_spec.queue = compiled_operation.queue
op_spec.component.inputs = compiled_operation.inputs
op_spec.component.outputs = compiled_operation.outputs
op_spec.component.contexts = compiled_operation.contexts
return op_spec
def test_op_specification_with_nocache(self):
config_dict = {
"version": 1.1,
"kind": "operation",
"name": "foo",
"description": "a description",
"tags": ["value"],
"cache": {"disable": True, "ttl": 12},
"params": {"param1": {"value": "foo"}, "param2": {"value": "bar"}},
"trigger": "all_succeeded",
"component": {
"name": "build-template",
"tags": ["kaniko"],
"run": {
"kind": V1RunKind.JOB,
"container": {"name": "polyaxon-main", "image": "test"},
"init": [{"connection": "some-connection"}],
},
},
}
op_config = V1Operation.read(values=config_dict)
run_config = OperationSpecification.compile_operation(op_config)
assert run_config.name == "foo"
assert run_config.description == "a description"
assert run_config.tags == ["kaniko", "value"]
assert run_config.cache.to_dict() == {"disable": True, "ttl": 12}
assert [i.to_light_dict() for i in run_config.run.init] == [
{"connection": "some-connection"}
]
env = {
"runPatch": {
"container": {
"resources": {
"requests": {"gpu": 1, "tpu": 1},
"limits": {"gpu": 1, "tpu": 1},
}
}
}
}
run_config = OperationSpecification.compile_operation(op_config, env)
assert (
run_config.run.container.resources
== env["runPatch"]["container"]["resources"]
)
def get_ops_from_suggestions(
content: str, compiled_operation: V1CompiledOperation, suggestions: List[Dict]
) -> List[V1Operation]:
ops = []
for suggestion in suggestions:
params = {
k: V1Param(value=Parser.parse_expression(v, {}))
for (k, v) in suggestion.items()
}
op_spec = V1Operation.read(content)
op_spec.matrix = None # remove matrix
op_spec.params = params
op_spec.component.inputs = compiled_operation.inputs
op_spec.component.outputs = compiled_operation.outputs
ops.append(op_spec)
return ops
def get_notifier_operation(
connection: str,
backend: str,
owner: str,
project: str,
run_uuid: str,
run_name: str,
condition: Union[str, Dict],
) -> V1Operation:
return V1Operation(
params={
"backend": V1Param(value=backend),
"owner": V1Param(value=owner),
"project": V1Param(value=project),
"uuid": V1Param(value=run_uuid),
"name": V1Param(value=run_name),
"condition": V1Param(value=condition),
},
termination=V1Termination(max_retries=3),
component=V1Component(
name="notifier",
plugins=V1Plugins(
auth=False,
collect_logs=False,
collect_artifacts=False,
collect_resources=False,
auto_resume=False,
sync_statuses=False,
external_host=True,
),
inputs=[
V1IO(name="backend", type=types.STR, is_optional=False),
V1IO(name="owner", type=types.STR, is_optional=False),
V1IO(name="project", type=types.STR, is_optional=False),
V1IO(name="uuid", type=types.STR, is_optional=False),
V1IO(name="name", type=types.STR, is_optional=True),
V1IO(name="condition", type=types.DICT, is_optional=True),
V1IO(name="connection", type=types.STR, is_optional=True),
],
run=V1NotifierJob(
connections=[connection],
container=get_default_notification_container(),
),
),
)
def get_notifier_operation(
connection: str,
kind: str,
owner: str,
project: str,
run_uuid: str,
run_name: str,
condition: str,
) -> V1Operation:
return V1Operation(
params={
"kind": V1Param(value=kind),
"owner": V1Param(value=owner),
"project": V1Param(value=project),
"run_uuid": V1Param(value=run_uuid),
"run_name": V1Param(value=run_name),
"condition": V1Param(value=condition),
},
termination=V1Termination(max_retries=3),
component=V1Component(
name="slack-notification",
plugins=V1Plugins(
auth=False,
collect_logs=False,
collect_artifacts=False,
collect_resources=False,
sync_statuses=False,
),
inputs=[
V1IO(name="kind", iotype=types.STR, is_optional=False),
V1IO(name="owner", iotype=types.STR, is_optional=False),
V1IO(name="project", iotype=types.STR, is_optional=False),
V1IO(name="run_uuid", iotype=types.STR, is_optional=False),
V1IO(name="run_name", iotype=types.STR, is_optional=True),
V1IO(name="condition", iotype=types.STR, is_optional=True),
V1IO(name="connection", iotype=types.STR, is_optional=True),
],
run=V1Notifier(
connections=[connection],
container=get_default_notification_container(),
),
),
)
def test_patch_post_merge_full_values_with_empty_preset(self):
operation = self.get_full_operation()
tmp_operation = self.get_full_operation()
result = tmp_operation.patch(V1Operation(is_preset=True),
strategy=V1PatchStrategy.POST_MERGE)
assert result.to_dict() == operation.to_dict()
tmp_operation = self.get_full_operation()
preset = self.get_empty_preset()
result = tmp_operation.patch(preset,
strategy=V1PatchStrategy.POST_MERGE)
result_dict = result.to_dict()
assert result_dict["description"] == ""
result_dict["description"] = self.DEFAULT_STR_VALUE
assert result_dict["queue"] == ""
result_dict["queue"] = "{}/{}".format(self.DEFAULT_STR_VALUE,
self.DEFAULT_STR_VALUE)
result_dict["presets"] = [self.DEFAULT_STR_VALUE]
# Since there's no component to validate the runPatch section it stays the same
assert result_dict == operation.to_dict()
operation = self.get_full_operation_with_component()
tmp_operation = self.get_full_operation_with_component()
preset = self.get_empty_preset()
result = tmp_operation.patch(preset,
strategy=V1PatchStrategy.POST_MERGE)
result_dict = result.to_dict()
assert result_dict["description"] == ""
result_dict["description"] = self.DEFAULT_STR_VALUE
assert result_dict["queue"] == ""
result_dict["queue"] = "{}/{}".format(self.DEFAULT_STR_VALUE,
self.DEFAULT_STR_VALUE)
# Run patch was validated and merged
assert result_dict["runPatch"]["environment"][
"serviceAccountName"] == ""
result_dict["runPatch"]["environment"][
"serviceAccountName"] = operation.run_patch["environment"][
"serviceAccountName"]
assert result_dict["runPatch"]["container"].pop(
"name") == MAIN_JOB_CONTAINER
assert result_dict == operation.to_dict()
def get_cleaner_operation(connection: V1ConnectionType, run_uuid: str,
run_kind: str) -> V1Operation:
return V1Operation(
termination=V1Termination(max_retries=1),
component=V1Component(
name="cleaner",
plugins=V1Plugins(
auth=False,
collect_logs=False,
collect_artifacts=False,
collect_resources=False,
auto_resume=False,
sync_statuses=False,
),
run=V1CleanerJob(
connections=[connection.name],
container=get_default_cleaner_container(
connection, run_uuid, run_kind),
),
),
)
def get_batch_cleaner_operation(
connection: V1ConnectionType,
paths: List[str],
) -> V1Operation:
return V1Operation(
termination=V1Termination(max_retries=1),
component=V1Component(
name="cleaner",
plugins=V1Plugins(
auth=False,
collect_logs=False,
collect_artifacts=False,
collect_resources=False,
auto_resume=False,
sync_statuses=False,
),
run=V1CleanerJob(
connections=[connection.name],
container=get_batch_cleaner_container(connection, paths),
),
),
)
def set_spec(spec: V1Operation, **kwargs) -> Tuple[V1Operation, Dict]:
kwargs["raw_content"] = spec.to_dict(dump=True)
return spec, kwargs
def init_run(
self,
project_id: int,
user_id: int,
op_spec: V1Operation = None,
compiled_operation: V1CompiledOperation = None,
name: str = None,
description: str = None,
tags: str = None,
override: Union[str, Dict] = None,
override_post: bool = True,
params: Dict = None,
readme: str = None,
original_id: int = None,
cloning_kind: str = None,
**kwargs,
) -> Tuple[V1CompiledOperation, BaseRun]:
content = None
raw_content = None
if op_spec:
op_spec = self.set_spec(op_spec)
raw_content = op_spec.to_dict(dump=True)
if op_spec:
if not compiled_operation or override:
compiled_operation = OperationSpecification.compile_operation(
op_spec, override=override, override_post=override_post)
params = op_spec.params
params = params or {}
inputs = {p: pv.value for p, pv in params.items() if pv.is_literal}
params = {p: pv.to_dict() for p, pv in params.items()}
kind = None
meta_info = {}
if compiled_operation:
content = compiled_operation.to_dict(dump=True)
name = name or compiled_operation.name
description = description or compiled_operation.description
tags = tags or compiled_operation.tags
kind, meta_kind = self.get_kind(compiled_operation)
kind, meta_info = self.get_meta_info(compiled_operation, kind,
meta_kind)
instance = get_run_model()(
project_id=project_id,
user_id=user_id,
name=name,
description=description,
tags=tags,
readme=readme,
raw_content=raw_content,
content=content,
params=params,
inputs=inputs,
kind=kind,
meta_info=meta_info,
original_id=original_id,
cloning_kind=cloning_kind,
status_conditions=[
V1StatusCondition.get_condition(
type=V1Statuses.CREATED,
status="True",
reason="PolyaxonRunCreated",
message="Run is created",
).to_dict()
],
**self.sanitize_kwargs(**kwargs),
)
return compiled_operation, instance
def compile_operation(cls,
config: V1Operation,
override: Dict = None) -> V1CompiledOperation:
if override:
preset = OperationSpecification.read(override, is_preset=True)
config = config.patch(preset, preset.patch_strategy)
# Patch run
component = config.component # type: V1Component
if config.run_patch:
component.run = component.run.patch(
validate_run_patch(config.run_patch, component.run.kind),
strategy=config.patch_strategy,
)
contexts = []
def get_context_io(c_name: str, c_io: V1Param, is_list=None):
if not c_io.context_only:
return
contexts.append(
V1IO(
name=c_name,
to_init=c_io.to_init,
connection=c_io.connection,
is_list=is_list,
))
# Collect contexts io form params
for p in config.params or {}:
get_context_io(c_name=p, c_io=config.params[p])
# Collect contexts io form joins
for j in config.joins or []:
for p in j.params or {}:
get_context_io(c_name=p, c_io=j.params[p], is_list=True)
patch_compiled = V1CompiledOperation(
name=config.name,
description=config.description,
contexts=contexts,
tags=config.tags,
presets=config.presets,
queue=config.queue,
cache=config.cache,
hooks=config.hooks,
events=config.events,
plugins=config.plugins,
termination=config.termination,
matrix=config.matrix,
joins=config.joins,
schedule=config.schedule,
dependencies=config.dependencies,
trigger=config.trigger,
conditions=config.conditions,
skip_on_upstream_skip=config.skip_on_upstream_skip,
)
values = [
{
cls.VERSION: config.version
},
component.to_dict(),
{
cls.KIND: kinds.COMPILED_OPERATION
},
]
compiled = V1CompiledOperation.read(
values) # type: V1CompiledOperation
return compiled.patch(patch_compiled, strategy=config.patch_strategy)