def test_realistic(): from_dict = { 'context': { 'unittest': { 'resources': { 'db_resource': {'config': {'user': '******', 'password': '******'}} } } } } onto_dict = {'context': {'unittest': {'resources': {'another': {'config': 'not_sensitive'}}}}} result_dict = { 'context': { 'unittest': { 'resources': { 'db_resource': {'config': {'user': '******', 'password': '******'}}, 'another': {'config': 'not_sensitive'}, } } } } assert dict_merge(from_dict, onto_dict) == result_dict
def monthly_trip_ingest_schedule(date): return dict_merge( { 'resources': trip_etl_environment['resources'], 'solids': { 'trip_etl': trip_etl_environment['solids']['trip_etl'] }, }, { 'solids': { 'trip_etl': { 'solids': { 'download_baybike_zipfile_from_url': { 'inputs': { 'file_name': { 'value': '{}-fordgobike-tripdata.csv.zip'.format( date.date().strftime('%Y%m')) } } } } } } }, )
def daily_weather_ingest_schedule(date): unix_seconds_since_epoch = int( (date - datetime(year=1970, month=1, day=1)).total_seconds()) return dict_merge( { 'resources': weather_etl_environment['resources'], 'solids': { 'weather_etl': weather_etl_environment['solids']['weather_etl'] }, }, { 'solids': { 'weather_etl': { 'solids': { 'download_weather_report_from_weather_api': { 'inputs': { 'epoch_date': { 'value': unix_seconds_since_epoch } } }, }, } } }, )
def _evaluate_composite_solid_config(context): '''Evaluates config for a composite solid and returns CompositeSolidEvaluationResult ''' # Support config mapping override functions if not is_solid_container_config(context.config_type): return EvaluateValueResult.empty() handle = context.config_type.handle # If we've already seen this handle, skip -- we've already run the block of code below if not handle or handle in context.seen_handles: return EvaluateValueResult.empty() solid_def = context.pipeline.get_solid( context.config_type.handle).definition solid_def_name = context.pipeline.get_solid(handle).definition.name has_mapping = isinstance( solid_def, CompositeSolidDefinition) and solid_def.has_config_mapping # If there's no config mapping function provided for this composite solid, bail if not has_mapping: return EvaluateValueResult.empty() # We first validate the provided environment config as normal against the composite solid config # schema. This will perform a full traversal rooted at the SolidContainerConfigDict and thread # errors up to the root config_context = context.new_context_with_handle(handle) evaluate_value_result = _evaluate_config(config_context) if not evaluate_value_result.success: return evaluate_value_result try: mapped_config_value = solid_def.config_mapping.config_fn( ConfigMappingContext(run_config=context.run_config), # ensure we don't mutate the source environment dict frozendict(evaluate_value_result.value.get('config') or {}), ) except Exception: # pylint: disable=W0703 return EvaluateValueResult.for_error( create_bad_user_config_fn_error( context, solid_def.config_mapping.config_fn.__name__, str(handle), solid_def_name, traceback.format_exc(), )) if not mapped_config_value: return EvaluateValueResult.empty() # Perform basic validation on the mapped config value; remaining validation will happen via the # evaluate_config call below if not isinstance(mapped_config_value, dict): return EvaluateValueResult.for_error( create_bad_mapping_error( context, solid_def.config_mapping.config_fn.__name__, solid_def_name, str(handle), mapped_config_value, )) if 'solids' in context.config_value: return EvaluateValueResult.for_error( create_bad_mapping_solids_key_error(context, solid_def_name, str(handle))) # We've validated the composite solid config; now validate the mapping fn overrides against the # config schema subtree for child solids evaluate_value_result = _evaluate_config( context.for_mapped_composite_config(handle, mapped_config_value)) if evaluate_value_result.errors: prefix = ( 'Config override mapping function defined by solid {handle_name} from ' 'definition {solid_def_name} {path_msg} caused error: '.format( path_msg=get_friendly_path_msg(context.stack), handle_name=str(handle), solid_def_name=solid_def_name, )) errors = [ e._replace(message=prefix + e.message) for e in evaluate_value_result.errors ] return EvaluateValueResult.for_errors(errors) return EvaluateValueResult.for_value( dict_merge(context.config_value, {'solids': evaluate_value_result.value}))
def test_nested_merge(): from_dict = {'key': {'nested_one': 1}} onto_dict = {'key': {'nested_two': 2}} assert dict_merge(from_dict, onto_dict) == {'key': {'nested_one': 1, 'nested_two': 2}}
def test_simple_merge(): assert dict_merge({}, {}) == {} assert dict_merge({1: 2}, {}) == {1: 2} assert dict_merge({}, {1: 2}) == {1: 2}
def test_smash(): from_dict = {'value': 'smasher'} onto_dict = {'value': 'got_smashed'} assert dict_merge(from_dict, onto_dict)['value'] == 'smasher'