def _get_mapped_solids_dict(composite_def, current_stack, current_solid_config): # the spec of the config mapping function is that it takes the dictionary at: # solid_name: # config: {dict_passed_to_user} # and it returns the dictionary rooted at solids # solid_name: # solids: {return_value_of_config_fn} # We must call the config mapping function and then validate it against # the child schema. with user_code_error_boundary(DagsterConfigMappingFunctionError, _get_error_lambda(current_stack)): mapped_solids_config = composite_def.config_mapping.config_fn( current_solid_config.get('config', {})) # Dynamically construct the type that the output of the config mapping function will # be evaluated against type_to_evaluate_against = define_solid_dictionary_cls( composite_def.solids, composite_def.dependency_structure, current_stack.handle) # process against that new type evr = process_config(type_to_evaluate_against, mapped_solids_config) if not evr.success: raise_composite_descent_config_error(current_stack, mapped_solids_config, evr) return evr.value
def test_solid_dictionary_some_no_config(): pipeline_def = PipelineDefinition(solids=[ SolidDefinition( name='int_config_solid', config_field=Field(Int), inputs=[], outputs=[], transform_fn=lambda *args: None, ), SolidDefinition(name='no_config_solid', inputs=[], outputs=[], transform_fn=lambda *args: None), ]) solid_dict_type = define_solid_dictionary_cls( 'foobar', create_creation_data(pipeline_def)).inst() value = construct_solid_dictionary( throwing_evaluate_config_value(solid_dict_type, {'int_config_solid': { 'config': 1 }})) assert set(['int_config_solid']) == set(value.keys()) assert value == {'int_config_solid': SolidConfig(1)}
def test_solid_dictionary_some_no_config(): pipeline_def = PipelineDefinition( solid_defs=[ SolidDefinition( name='int_config_solid', config_field=Field(Int), input_defs=[], output_defs=[], compute_fn=lambda *args: None, ), SolidDefinition( name='no_config_solid', input_defs=[], output_defs=[], compute_fn=lambda *args: None ), ] ) solid_dict_type = define_solid_dictionary_cls( 'foobar', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name ).inst() value = construct_solid_dictionary( throwing_evaluate_config_value(solid_dict_type, {'int_config_solid': {'config': 1}}) ) assert set(['int_config_solid']) == set(value.keys()) assert value == {'int_config_solid': SolidConfig(1)}
def test_solid_dictionary_type(): pipeline_def = define_test_solids_config_pipeline() solid_dict_type = define_solid_dictionary_cls( 'foobar', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name).inst() value = construct_solid_dictionary( throwing_evaluate_config_value( solid_dict_type, { 'int_config_solid': { 'config': 1 }, 'string_config_solid': { 'config': 'bar' } }, )) assert set(['int_config_solid', 'string_config_solid']) == set(value.keys()) assert value == { 'int_config_solid': SolidConfig(1), 'string_config_solid': SolidConfig('bar') } assert solid_dict_type.type_attributes.is_system_config for specific_solid_config_field in solid_dict_type.fields.values(): specific_solid_config_type = specific_solid_config_field.config_type assert specific_solid_config_type.type_attributes.is_system_config user_config_field = specific_solid_config_field.config_type.fields[ 'config'] assert user_config_field.config_type.type_attributes.is_system_config is False
def test_solid_dictionary_some_no_config(): @solid(name='int_config_solid', config_field=Field(Int), input_defs=[], output_defs=[]) def int_config_solid(_): return None @solid(name='no_config_solid', input_defs=[], output_defs=[]) def no_config_solid(_): return None @pipeline def pipeline_def(): int_config_solid() no_config_solid() solid_dict_type = define_solid_dictionary_cls( 'foobar', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name).inst() value = construct_solid_dictionary( throwing_evaluate_config_value(solid_dict_type, {'int_config_solid': { 'config': 1 }})) assert set(['int_config_solid']) == set(value.keys()) assert value == {'int_config_solid': SolidConfig(1)}
def test_solid_config_error(): solid_dict_type = define_solid_dictionary_cls( 'slkdfjkjdsf', create_creation_data(define_test_solids_config_pipeline()) ).inst() int_solid_config_type = solid_dict_type.fields['int_config_solid'].config_type with pytest.raises(DagsterEvaluateConfigValueError, match='Field "notconfig" is not defined'): throwing_evaluate_config_value(int_solid_config_type, {'notconfig': 1}) with pytest.raises(DagsterEvaluateConfigValueError): throwing_evaluate_config_value(int_solid_config_type, 1)
def test_solid_config_error(): pipeline_def = define_test_solids_config_pipeline() solid_dict_type = define_solid_dictionary_cls( 'slkdfjkjdsf', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name ).inst() int_solid_config_type = solid_dict_type.fields['int_config_solid'].config_type with pytest.raises(DagsterEvaluateConfigValueError, match='Field "notconfig" is not defined'): throwing_evaluate_config_value(int_solid_config_type, {'notconfig': 1}) with pytest.raises(DagsterEvaluateConfigValueError): throwing_evaluate_config_value(int_solid_config_type, 1)
def test_solid_config_error(): pipeline_def = define_test_solids_config_pipeline() solid_dict_type = define_solid_dictionary_cls( pipeline_def.solids, pipeline_def.dependency_structure, ) int_solid_config_type = solid_dict_type.fields['int_config_solid'].config_type with pytest.raises(DagsterEvaluateConfigValueError, match='Undefined field "notconfig"'): throwing_validate_config_value(int_solid_config_type, {'notconfig': 1}) with pytest.raises(DagsterEvaluateConfigValueError): throwing_validate_config_value(int_solid_config_type, 1)
def test_solid_config_error(): pipeline_def = define_test_solids_config_pipeline() solid_dict_type = define_solid_dictionary_cls( pipeline_def.solids, pipeline_def.dependency_structure, ) int_solid_config_type = solid_dict_type.fields['int_config_solid'].config_type res = process_config(int_solid_config_type, {'notconfig': 1}) assert not res.success assert re.match('Undefined field "notconfig"', res.errors[0].message) res = process_config(int_solid_config_type, 1) assert not res.success
def _get_mapped_solids_dict(composite, graph_def, current_stack, current_solid_config, resource_defs): # the spec of the config mapping function is that it takes the dictionary at: # solid_name: # config: {dict_passed_to_user} # and it returns the dictionary rooted at solids # solid_name: # solids: {return_value_of_config_fn} # We must call the config mapping function and then validate it against # the child schema. # apply @configured config mapping to the composite's incoming config before we get to the # composite's own config mapping process config_mapped_solid_config = graph_def.apply_config_mapping( current_solid_config) if not config_mapped_solid_config.success: raise DagsterInvalidConfigError( "Error in config for composite solid {}".format(composite.name), config_mapped_solid_config.errors, config_mapped_solid_config, ) with user_code_error_boundary(DagsterConfigMappingFunctionError, _get_error_lambda(current_stack)): mapped_solids_config = graph_def.config_mapping.config_fn( config_mapped_solid_config.value.get("config", {})) # Dynamically construct the type that the output of the config mapping function will # be evaluated against type_to_evaluate_against = define_solid_dictionary_cls( solids=graph_def.solids, ignored_solids=None, dependency_structure=graph_def.dependency_structure, parent_handle=current_stack.handle, resource_defs=resource_defs, ) # process against that new type evr = process_config(type_to_evaluate_against, mapped_solids_config) if not evr.success: raise_composite_descent_config_error(current_stack, mapped_solids_config, evr) return evr.value
def test_solid_config_error(): pipeline_def = define_test_solids_config_pipeline() solid_dict_type = define_solid_dictionary_cls( solids=pipeline_def.solids, ignored_solids=None, dependency_structure=pipeline_def.dependency_structure, parent_handle=None, resource_defs={}, ) int_solid_config_type = solid_dict_type.fields["int_config_solid"].config_type res = process_config(int_solid_config_type, {"notconfig": 1}) assert not res.success assert re.match('Received unexpected config entry "notconfig"', res.errors[0].message) res = process_config(int_solid_config_type, 1) assert not res.success
def test_solid_dictionary_type(): pipeline_def = define_test_solids_config_pipeline() solid_dict_type = define_solid_dictionary_cls( pipeline_def.solids, pipeline_def.dependency_structure) env_obj = EnvironmentConfig.build( pipeline_def, { 'solids': { 'int_config_solid': { 'config': 1 }, 'string_config_solid': { 'config': 'bar' } }, }, ) value = env_obj.solids assert set(['int_config_solid', 'string_config_solid']) == set(value.keys()) assert value == { 'int_config_solid': SolidConfig(1), 'string_config_solid': SolidConfig('bar') } assert solid_dict_type.type_attributes.is_system_config for specific_solid_config_field in solid_dict_type.fields.values(): specific_solid_config_type = specific_solid_config_field.config_type assert specific_solid_config_type.type_attributes.is_system_config user_config_field = specific_solid_config_field.config_type.fields[ 'config'] assert user_config_field.config_type.type_attributes.is_system_config is False