def test_realistic():
from_dict = {
'context': {
'unittest': {
'resources': {
'db_resource': {'config': {'user': '******', 'password': '******'}}
}
}
}
}
onto_dict = {'context': {'unittest': {'resources': {'another': {'config': 'not_sensitive'}}}}}
result_dict = {
'context': {
'unittest': {
'resources': {
'db_resource': {'config': {'user': '******', 'password': '******'}},
'another': {'config': 'not_sensitive'},
}
}
}
}
assert dict_merge(from_dict, onto_dict) == result_dict
def monthly_trip_ingest_schedule(date):
return dict_merge(
{
'resources': trip_etl_environment['resources'],
'solids': {
'trip_etl': trip_etl_environment['solids']['trip_etl']
},
},
{
'solids': {
'trip_etl': {
'solids': {
'download_baybike_zipfile_from_url': {
'inputs': {
'file_name': {
'value':
'{}-fordgobike-tripdata.csv.zip'.format(
date.date().strftime('%Y%m'))
}
}
}
}
}
}
},
)
def daily_weather_ingest_schedule(date):
unix_seconds_since_epoch = int(
(date - datetime(year=1970, month=1, day=1)).total_seconds())
return dict_merge(
{
'resources': weather_etl_environment['resources'],
'solids': {
'weather_etl': weather_etl_environment['solids']['weather_etl']
},
},
{
'solids': {
'weather_etl': {
'solids': {
'download_weather_report_from_weather_api': {
'inputs': {
'epoch_date': {
'value': unix_seconds_since_epoch
}
}
},
},
}
}
},
)
def _evaluate_composite_solid_config(context):
'''Evaluates config for a composite solid and returns CompositeSolidEvaluationResult
'''
# Support config mapping override functions
if not is_solid_container_config(context.config_type):
return EvaluateValueResult.empty()
handle = context.config_type.handle
# If we've already seen this handle, skip -- we've already run the block of code below
if not handle or handle in context.seen_handles:
return EvaluateValueResult.empty()
solid_def = context.pipeline.get_solid(
context.config_type.handle).definition
solid_def_name = context.pipeline.get_solid(handle).definition.name
has_mapping = isinstance(
solid_def, CompositeSolidDefinition) and solid_def.has_config_mapping
# If there's no config mapping function provided for this composite solid, bail
if not has_mapping:
return EvaluateValueResult.empty()
# We first validate the provided environment config as normal against the composite solid config
# schema. This will perform a full traversal rooted at the SolidContainerConfigDict and thread
# errors up to the root
config_context = context.new_context_with_handle(handle)
evaluate_value_result = _evaluate_config(config_context)
if not evaluate_value_result.success:
return evaluate_value_result
try:
mapped_config_value = solid_def.config_mapping.config_fn(
ConfigMappingContext(run_config=context.run_config),
# ensure we don't mutate the source environment dict
frozendict(evaluate_value_result.value.get('config') or {}),
)
except Exception: # pylint: disable=W0703
return EvaluateValueResult.for_error(
create_bad_user_config_fn_error(
context,
solid_def.config_mapping.config_fn.__name__,
str(handle),
solid_def_name,
traceback.format_exc(),
))
if not mapped_config_value:
return EvaluateValueResult.empty()
# Perform basic validation on the mapped config value; remaining validation will happen via the
# evaluate_config call below
if not isinstance(mapped_config_value, dict):
return EvaluateValueResult.for_error(
create_bad_mapping_error(
context,
solid_def.config_mapping.config_fn.__name__,
solid_def_name,
str(handle),
mapped_config_value,
))
if 'solids' in context.config_value:
return EvaluateValueResult.for_error(
create_bad_mapping_solids_key_error(context, solid_def_name,
str(handle)))
# We've validated the composite solid config; now validate the mapping fn overrides against the
# config schema subtree for child solids
evaluate_value_result = _evaluate_config(
context.for_mapped_composite_config(handle, mapped_config_value))
if evaluate_value_result.errors:
prefix = (
'Config override mapping function defined by solid {handle_name} from '
'definition {solid_def_name} {path_msg} caused error: '.format(
path_msg=get_friendly_path_msg(context.stack),
handle_name=str(handle),
solid_def_name=solid_def_name,
))
errors = [
e._replace(message=prefix + e.message)
for e in evaluate_value_result.errors
]
return EvaluateValueResult.for_errors(errors)
return EvaluateValueResult.for_value(
dict_merge(context.config_value,
{'solids': evaluate_value_result.value}))
def test_nested_merge():
from_dict = {'key': {'nested_one': 1}}
onto_dict = {'key': {'nested_two': 2}}
assert dict_merge(from_dict, onto_dict) == {'key': {'nested_one': 1, 'nested_two': 2}}
def test_simple_merge():
assert dict_merge({}, {}) == {}
assert dict_merge({1: 2}, {}) == {1: 2}
assert dict_merge({}, {1: 2}) == {1: 2}
def test_smash():
from_dict = {'value': 'smasher'}
onto_dict = {'value': 'got_smashed'}
assert dict_merge(from_dict, onto_dict)['value'] == 'smasher'