def patch_normal_merge(current_value,
value,
strategy: V1PatchStrategy = None):
strategy = strategy or V1PatchStrategy.POST_MERGE
if isinstance(current_value, Mapping):
if strategy == V1PatchStrategy.POST_MERGE:
return deep_update(current_value, value)
elif strategy == V1PatchStrategy.PRE_MERGE:
return deep_update(value, current_value)
elif isinstance(current_value, list):
if strategy == V1PatchStrategy.POST_MERGE:
return current_value + [
i for i in value if i not in current_value
]
elif strategy == V1PatchStrategy.PRE_MERGE:
return value + [i for i in current_value if i not in value]
elif isinstance(current_value, BaseConfig):
return current_value.patch(value, strategy=strategy)
elif hasattr(current_value, "to_dict"):
if strategy == V1PatchStrategy.POST_MERGE:
return deep_update(current_value.to_dict(), value.to_dict())
elif strategy == V1PatchStrategy.PRE_MERGE:
return deep_update(value.to_dict(), current_value.to_dict())
else:
if strategy == V1PatchStrategy.POST_MERGE:
return value
elif strategy == V1PatchStrategy.PRE_MERGE:
return current_value
def read_from(cls, config_values, config_type=None):
"""
Reads an ordered list of configuration values and
deep merge the values in reverse order.
"""
if not config_values:
raise PolyaxonSchemaError(
"Cannot read config_value: `{}`".format(config_values)
)
config_values = to_list(config_values, check_none=True)
config = {}
for config_value in config_values:
config_value = ConfigSpec.get_from(
value=config_value, config_type=config_type
)
config_value.check_type()
config_results = config_value.read()
if config_results and isinstance(config_results, Mapping):
config = deep_update(config, config_results)
elif config_value.check_if_exists:
raise PolyaxonSchemaError(
"Cannot read config_value: `{}`".format(config_value.value)
)
return config
def parse(cls, config, param_spec: Dict[str, ParamSpec]): # pylint:disable=too-many-branches
param_spec = param_spec or {}
parsed_params = {
param: param_spec[param].display_value
for param in param_spec
}
parsed_data = {
Sections.VERSION: config.version,
Sections.KIND: config.kind
}
if config.name:
parsed_data[Sections.NAME] = config.name
if config.description:
parsed_data[Sections.DESCRIPTION] = config.description
if config.tags:
parsed_data[Sections.TAGS] = config.tags
inputs = getattr(config, Sections.INPUTS)
if inputs:
parsed_data[Sections.INPUTS] = [io.to_dict() for io in inputs]
outputs = getattr(config, Sections.OUTPUTS)
if outputs:
parsed_data[Sections.OUTPUTS] = [
cls.parse_expression(io.to_dict(), parsed_params)
for io in outputs
]
# Check workflow
matrix_section = cls._get_section(config, Sections.MATRIX)
if matrix_section:
parsed_data[Sections.MATRIX] = cls.parse_expression(
matrix_section, parsed_params)
matrix_params = copy.copy(parsed_data[Sections.MATRIX])
if matrix_params:
parsed_params = deep_update(matrix_params, parsed_params)
for section in Sections.PARSING_SECTIONS:
config_section = cls._get_section(config, section)
if config_section:
parsed_data[section] = cls.parse_expression(
config_section, parsed_params)
for section in Sections.OP_PARSING_SECTIONS:
config_section = cls._get_section(config, section)
if config_section:
parsed_data[section] = cls.parse_expression(
config_section, parsed_params)
config_section = cls._get_section(config, Sections.RUN)
if config_section:
parsed_data[Sections.RUN] = config_section
return parsed_data
def parse(cls, spec, config, params): # pylint:disable=too-many-branches
params = params or {}
parsed_params = {
param: params[param].display_value
for param in params
}
parsed_data = {spec.VERSION: config.version, spec.KIND: config.kind}
if config.name:
parsed_data[spec.NAME] = config.name
if config.description:
parsed_data[spec.DESCRIPTION] = config.description
if config.tags:
parsed_data[spec.TAGS] = config.tags
inputs = getattr(config, spec.INPUTS)
if inputs:
parsed_data[spec.INPUTS] = [io.to_dict() for io in inputs]
outputs = getattr(config, spec.OUTPUTS)
if outputs:
parsed_data[spec.OUTPUTS] = [
cls.parse_expression(spec, io.to_dict(), parsed_params)
for io in outputs
]
# Check workflow
parallel_section = cls._get_section(config, spec.PARALLEL)
if parallel_section:
parsed_data[spec.PARALLEL] = cls.parse_expression(
spec, parallel_section, parsed_params)
parallel_params = copy.copy(parsed_data[spec.PARALLEL])
if parallel_params:
parsed_params = deep_update(parallel_params, parsed_params)
for section in spec.PARSING_SECTIONS:
config_section = cls._get_section(config, section)
if config_section:
parsed_data[section] = cls.parse_expression(
spec, config_section, parsed_params)
for section in spec.OP_PARSING_SECTIONS:
config_section = cls._get_section(config, section)
if config_section:
parsed_data[section] = cls.parse_expression(
spec, config_section, parsed_params)
config_section = cls._get_section(config, spec.RUN)
if config_section:
parsed_data[spec.RUN] = config_section
return parsed_data
def test_patch_pre_merge_full_values_with_full_preset(self):
operation = self.get_full_operation()
tmp_operation = self.get_full_operation()
preset = self.get_full_preset()
preset_dict = preset.to_dict()
expected = operation.to_dict()
result = tmp_operation.patch(preset, strategy=V1PatchStrategy.PRE_MERGE)
result_dict = result.to_dict()
expected["tags"] = preset_dict["tags"] + operation.tags
expected["presets"] = preset_dict["presets"] + operation.presets
expected["hooks"] = preset_dict["hooks"] + [
i.to_dict() for i in operation.hooks
]
expected["dependencies"] = preset_dict["dependencies"] + operation.dependencies
expected["events"] = preset_dict["events"] + [
i.to_dict() for i in operation.events
]
expected["joins"] = preset_dict["joins"] + [
i.to_dict() for i in operation.joins
]
expected["matrix"]["values"] = (
preset_dict["matrix"]["values"] + operation.matrix.values
)
assert result_dict == expected
operation = self.get_full_operation_with_component()
tmp_operation = self.get_full_operation_with_component()
preset = self.get_full_preset()
preset_dict = preset.to_dict()
expected = operation.to_dict()
result = tmp_operation.patch(preset, strategy=V1PatchStrategy.PRE_MERGE)
result_dict = result.to_dict()
expected["tags"] = preset_dict["tags"] + operation.tags
expected["presets"] = preset_dict["presets"] + operation.presets
expected["hooks"] = preset_dict["hooks"] + [
i.to_dict() for i in operation.hooks
]
expected["dependencies"] = preset_dict["dependencies"] + operation.dependencies
expected["events"] = preset_dict["events"] + [
i.to_dict() for i in operation.events
]
expected["joins"] = preset_dict["joins"] + [
i.to_dict() for i in operation.joins
]
expected["matrix"]["values"] = (
preset_dict["matrix"]["values"] + operation.matrix.values
)
# Run patch was validated and merged
assert result_dict["runPatch"]["container"].pop("name") == MAIN_JOB_CONTAINER
assert result_dict["runPatch"]["container"].pop("resources") == deep_update(
preset_dict["runPatch"]["container"]["resources"],
expected["runPatch"]["container"]["resources"],
)
result_dict["runPatch"]["container"]["resources"] = expected["runPatch"][
"container"
]["resources"]
assert (
result_dict["runPatch"]["connections"]
== preset_dict["runPatch"]["connections"]
+ expected["runPatch"]["connections"]
)
result_dict["runPatch"]["connections"] = expected["runPatch"]["connections"]
assert (
result_dict["runPatch"]["init"]
== preset_dict["runPatch"]["init"] + expected["runPatch"]["init"]
)
result_dict["runPatch"]["init"] = expected["runPatch"]["init"]
assert (
result_dict["runPatch"]["environment"]["imagePullSecrets"]
== preset_dict["runPatch"]["environment"]["imagePullSecrets"]
+ expected["runPatch"]["environment"]["imagePullSecrets"]
)
result_dict["runPatch"]["environment"]["imagePullSecrets"] = expected[
"runPatch"
]["environment"]["imagePullSecrets"]
assert result_dict["runPatch"]["environment"]["nodeSelector"] == {
**preset_dict["runPatch"]["environment"]["nodeSelector"],
**expected["runPatch"]["environment"]["nodeSelector"],
}
result_dict["runPatch"]["environment"]["nodeSelector"] = expected["runPatch"][
"environment"
]["nodeSelector"]
assert result_dict == expected