def test_deep_merge_conflict_no_overwrite(self):
dict1 = {
'step-runner-config': {
'step-foo': {
'implementer': 'foo1',
'config': {
'test0': 'foo',
'test1': 'foo'
}
}
}
}
dict2 = {
'step-runner-config': {
'step-foo': {
'implementer': 'foo1',
'config': {
'test1': 'bar',
'test2': 'bar'
}
}
}
}
with self.assertRaisesRegex(
ValueError,
r"Conflict at step-runner-config.step-foo.config.test1"):
deep_merge(dict1, dict2)
def merge_sub_step_env_config(self, new_sub_step_env_config):
"""Merge new sub step environment configuration into the existing
sub step environment configuration.
Parameters
----------
new_sub_step_env_config : dict
New sub step environment configuration to merge into
the existing sub step environment configuration.
Raises
------
ValueError
If new sub step environment configuration has duplicative leaf keys to
existing sub step environment configuration.
"""
if new_sub_step_env_config is not None:
try:
self.__sub_step_env_config = deep_merge(
self.__sub_step_env_config,
copy.deepcopy(new_sub_step_env_config)
)
except ValueError as error:
raise ValueError(
"Error merging new sub step environment configuration" +
" into existing sub step environment configuration" +
f" for sub step ({self.sub_step_name}) of step ({self.step_name}): {error}"
) from error
def __get_all_step_results_dict(self):
"""Get a dictionary of all of the recorded StepResults.
Returns
-------
results: dict
results of all steps from list
"""
all_results = {}
for step_result in self.workflow_list:
all_results = deep_merge(dest=all_results,
source=step_result.get_step_result_dict(),
overwrite_duplicate_keys=True)
step_runner_results = {'step-runner-results': all_results}
return step_runner_results
def test_deep_merge_conflict_overwrite_duplicate_keys(self):
dict1 = {
'step-runner-config': {
'step-foo': {
'implementer': 'foo1',
'config': {
'test0': 'foo',
'test1': 'foo'
}
}
}
}
dict2 = {
'step-runner-config': {
'step-foo': {
'implementer': 'foo1',
'config': {
'test1': 'bar',
'test2': 'bar'
}
}
}
}
result = deep_merge(dest=dict1,
source=dict2,
overwrite_duplicate_keys=True)
# assert expected merge result
self.assertEqual(
result, {
'step-runner-config': {
'step-foo': {
'implementer': 'foo1',
'config': {
'test0': 'foo',
'test1': 'bar',
'test2': 'bar'
}
}
}
})
def test_deep_merge_no_conflict(self):
dict1 = {
'step-runner-config': {
'step-foo': {
'implementer': 'foo1',
'config': {
'test0': 'foo',
'test1': 'foo'
}
}
}
}
dict2 = {
'step-runner-config': {
'step-foo': {
'implementer': 'foo1',
'config': {
'test2': 'bar'
}
}
}
}
result = deep_merge(dict1, dict2)
# assert that it is in an inplace deep merge of the first param
self.assertEqual(result, dict1)
# assert expected merge result
self.assertEqual(
result, {
'step-runner-config': {
'step-foo': {
'implementer': 'foo1',
'config': {
'test0': 'foo',
'test1': 'foo',
'test2': 'bar'
}
}
}
})
def __add_config_dict(self, config_dict, source_file_path=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements
"""Add a configuration dictionary to the list of configuration dictionaries.
Parameters
----------
config_dict : dict
A dictionary to validate as a configuration and to add to this Config.
source_file_path : str, optional
File path to the file from which the given config_dict came from.
Raises
------
AssertionError
If the given config_dict is not a valid configuration dictionary.
If attempt to update an existing sub step and new and existing sub step implementers
do not match.
If sub step does not define a step implementer.
ValueError
If duplicative leaf keys when merging global defaults
If duplicative leaf keys when merging global env defaults
If step config is not of type dict or list
If new sub step configuration has duplicative leaf keys to
existing sub step configuration.
If new sub step environment configuration has duplicative leaf keys to
existing sub step environment configuration.
"""
assert Config.CONFIG_KEY in config_dict, \
"Failed to add invalid config. " + \
f"Missing expected top level key ({Config.CONFIG_KEY}): " + \
f"{config_dict}"
# if file path given use that as the source when creating ConfigValue objects
# else use a copy of the given configuration dictionary
if source_file_path is not None:
parent_source = source_file_path
else:
parent_source = copy.deepcopy(config_dict)
# convert all the leaves of the configuration dictionary under
# the Config.CONFIG_KEY to ConfigValue objects
config_values = ConfigValue.convert_leaves_to_config_values(
values=copy.deepcopy(config_dict[Config.CONFIG_KEY]),
parent_source=parent_source,
path_parts=[Config.CONFIG_KEY]
)
for key, value in config_values.items():
# if global default key
# else if global env defaults key
# else assume step config
if key == Config.CONFIG_KEY_GLOBAL_DEFAULTS:
try:
self.__global_defaults = deep_merge(
copy.deepcopy(self.__global_defaults),
copy.deepcopy(value)
)
except ValueError as error:
raise ValueError(
f"Error merging global defaults: {error}"
) from error
elif key == Config.CONFIG_KEY_GLOBAL_ENVIRONMENT_DEFAULTS:
for env, env_config in value.items():
if env not in self.__global_environment_defaults:
self.__global_environment_defaults[env] = {
Config.CONFIG_KEY_ENVIRONMENT_NAME: env
}
try:
self.__global_environment_defaults[env] = deep_merge(
copy.deepcopy(self.__global_environment_defaults[env]),
copy.deepcopy(env_config)
)
except ValueError as error:
raise ValueError(
f"Error merging global environment ({env}) defaults: {error}"
) from error
elif key == Config.CONFIG_KEY_DECRYPTORS:
config_decryptor_definitions = ConfigValue.convert_leaves_to_values(value)
Config.parse_and_register_decryptors_definitions(config_decryptor_definitions)
else:
step_name = key
step_config = value
# if step_config is dict then assume step with single sub step
if isinstance(step_config, dict):
sub_steps = [step_config]
elif isinstance(step_config, list):
sub_steps = step_config
else:
raise ValueError(
f"Expected step ({step_name}) to have have step config ({step_config})" +
f" of type dict or list but got: {type(step_config)}"
)
for sub_step in sub_steps:
assert Config.CONFIG_KEY_STEP_IMPLEMENTER in sub_step, \
f"Step ({step_name}) defines a single sub step with values " + \
f"({sub_step}) but is missing value for key: " + \
f"{Config.CONFIG_KEY_STEP_IMPLEMENTER}"
sub_step_implementer_name = \
sub_step[Config.CONFIG_KEY_STEP_IMPLEMENTER].value
# if sub step name given
# else if no sub step name given use step implementer as sub step name
if Config.CONFIG_KEY_SUB_STEP_NAME in sub_step:
sub_step_name = sub_step[Config.CONFIG_KEY_SUB_STEP_NAME].value
else:
sub_step_name = sub_step_implementer_name
# determine sub step config
if Config.CONFIG_KEY_SUB_STEP_CONFIG in sub_step:
sub_step_config_dict = copy.deepcopy(
sub_step[Config.CONFIG_KEY_SUB_STEP_CONFIG])
else:
sub_step_config_dict = {}
# determine sub step environment config
if Config.CONFIG_KEY_SUB_STEP_ENVIRONMENT_CONFIG in sub_step:
sub_step_env_config = copy.deepcopy(
sub_step[Config.CONFIG_KEY_SUB_STEP_ENVIRONMENT_CONFIG])
else:
sub_step_env_config = {}
# determine if continue sub steps on this sub step failure
sub_step_contine_sub_steps_on_failure = False
if Config.CONFIG_KEY_CONTINUE_SUB_STEPS_ON_FAILURE in sub_step:
sub_step_contine_sub_steps_on_failure = sub_step[
Config.CONFIG_KEY_CONTINUE_SUB_STEPS_ON_FAILURE
]
if isinstance(sub_step_contine_sub_steps_on_failure.value, bool):
sub_step_contine_sub_steps_on_failure = \
sub_step_contine_sub_steps_on_failure.value
else:
sub_step_contine_sub_steps_on_failure = bool(
util.strtobool(sub_step_contine_sub_steps_on_failure.value)
)
self.add_or_update_step_config(
step_name=step_name,
sub_step_name=sub_step_name,
sub_step_implementer_name=sub_step_implementer_name,
sub_step_config_dict=sub_step_config_dict,
sub_step_env_config=sub_step_env_config,
sub_step_contine_sub_steps_on_failure=sub_step_contine_sub_steps_on_failure
)
