def test_solid_dictionary_type(): pipeline_def = define_test_solids_config_pipeline() env_obj = EnvironmentConfig.build( pipeline_def, { "solids": { "int_config_solid": { "config": 1 }, "string_config_solid": { "config": "bar" } }, }, ) value = env_obj.solids assert set(["int_config_solid", "string_config_solid"]) == set(value.keys()) assert value == { "int_config_solid": SolidConfig(1), "string_config_solid": SolidConfig("bar") }
def test_solid_dictionary_type(): pipeline_def = define_test_solids_config_pipeline() env_obj = ResolvedRunConfig.build( pipeline_def, { "solids": { "int_config_solid": { "config": 1 }, "string_config_solid": { "config": "bar" } }, }, ) value = env_obj.solids assert set(["int_config_solid", "string_config_solid"]) == set(value.keys()) assert value == { "int_config_solid": SolidConfig.from_dict({"config": 1}), "string_config_solid": SolidConfig.from_dict({"config": "bar"}), }
def test_solid_dictionary_type(): pipeline_def = define_test_solids_config_pipeline() env_obj = EnvironmentConfig.build( pipeline_def, { 'solids': { 'int_config_solid': { 'config': 1 }, 'string_config_solid': { 'config': 'bar' } }, }, ) value = env_obj.solids assert set(['int_config_solid', 'string_config_solid']) == set(value.keys()) assert value == { 'int_config_solid': SolidConfig(1), 'string_config_solid': SolidConfig('bar') }
def test_solid_dictionary_some_no_config(): @solid(name='int_config_solid', config_schema=Int, input_defs=[], output_defs=[]) def int_config_solid(_): return None @solid(name='no_config_solid', input_defs=[], output_defs=[]) def no_config_solid(_): return None @pipeline def pipeline_def(): int_config_solid() no_config_solid() env = EnvironmentConfig.build( pipeline_def, {'solids': { 'int_config_solid': { 'config': 1 } }}) assert {'int_config_solid', 'no_config_solid'} == set(env.solids.keys()) assert env.solids == { 'int_config_solid': SolidConfig(1), 'no_config_solid': SolidConfig() }
def test_solid_dictionary_some_no_config(): @solid(name="int_config_solid", config_schema=Int, input_defs=[], output_defs=[]) def int_config_solid(_): return None @solid(name="no_config_solid", input_defs=[], output_defs=[]) def no_config_solid(_): return None @pipeline def pipeline_def(): int_config_solid() no_config_solid() env = ResolvedRunConfig.build( pipeline_def, {"solids": { "int_config_solid": { "config": 1 } }}) assert {"int_config_solid", "no_config_solid"} == set(env.solids.keys()) assert env.solids == { "int_config_solid": SolidConfig.from_dict({"config": 1}), "no_config_solid": SolidConfig.from_dict({}), }
def test_solid_dictionary_type(): pipeline_def = define_test_solids_config_pipeline() solid_dict_type = define_solid_dictionary_cls( 'foobar', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name).inst() value = construct_solid_dictionary( throwing_evaluate_config_value( solid_dict_type, { 'int_config_solid': { 'config': 1 }, 'string_config_solid': { 'config': 'bar' } }, )) assert set(['int_config_solid', 'string_config_solid']) == set(value.keys()) assert value == { 'int_config_solid': SolidConfig(1), 'string_config_solid': SolidConfig('bar') } assert solid_dict_type.type_attributes.is_system_config for specific_solid_config_field in solid_dict_type.fields.values(): specific_solid_config_type = specific_solid_config_field.config_type assert specific_solid_config_type.type_attributes.is_system_config user_config_field = specific_solid_config_field.config_type.fields[ 'config'] assert user_config_field.config_type.type_attributes.is_system_config is False
def _composite_descent(parent_stack, solids_config_dict, resource_defs): """ The core implementation of composite_descent. This yields a stream of SolidConfigEntry. This is used by composite_descent to construct a dictionary. It descends over the entire solid hierarchy, constructing an entry for every handle. If it encounters a composite solid instance with a config mapping, it will invoke that config mapping fn, producing the config that is necessary to configure the child solids. This process unrolls recursively as you descend down the tree. """ for solid in parent_stack.current_container.solids: current_stack = parent_stack.descend(solid) current_handle = current_stack.handle current_solid_config = solids_config_dict.get(solid.name, {}) # the base case if isinstance(solid.definition, SolidDefinition): config_mapped_solid_config = solid.definition.apply_config_mapping( {"config": current_solid_config.get("config")}) if not config_mapped_solid_config.success: raise DagsterInvalidConfigError( "Error in config for solid {}".format(solid.name), config_mapped_solid_config.errors, config_mapped_solid_config, ) complete_config_object = merge_dicts( current_solid_config, config_mapped_solid_config.value) yield SolidConfigEntry( current_handle, SolidConfig.from_dict(complete_config_object)) continue graph_def = check.inst(solid.definition, GraphDefinition) yield SolidConfigEntry( current_handle, SolidConfig.from_dict({ "inputs": current_solid_config.get("inputs"), "outputs": current_solid_config.get("outputs"), }), ) # If there is a config mapping, invoke it and get the descendent solids # config that way. Else just grabs the solids entry of the current config solids_dict = (_get_mapped_solids_dict( solid, graph_def, current_stack, current_solid_config, resource_defs) if graph_def.config_mapping else current_solid_config.get("solids", {})) yield from _composite_descent(current_stack, solids_dict, resource_defs)
def _composite_descent(parent_stack, solids_config_dict): ''' The core implementation of composite_descent. This yields a stream of SolidConfigEntry. This is used by composite_descent to construct a dictionary. It descends over the entire solid hierarchy, constructing an entry for every handle. If it encounters a composite solid instance with a config mapping, it will invoke that config mapping fn, producing the config that is necessary to configure the child solids. This process unrolls recursively as you descend down the tree. ''' for solid in parent_stack.current_container.solids: current_stack = parent_stack.descend(solid) current_handle = current_stack.handle current_solid_config = solids_config_dict.get(solid.name, {}) # the base case if not isinstance(solid.definition, CompositeSolidDefinition): transformed_current_solid_config = solid.definition.apply_config_mapping( current_solid_config) if not transformed_current_solid_config.success: raise DagsterInvalidConfigError( 'Error in config for solid {}'.format(solid.name), transformed_current_solid_config.errors, transformed_current_solid_config, ) yield SolidConfigEntry( current_handle, SolidConfig.from_dict(transformed_current_solid_config.value)) continue composite_def = check.inst(solid.definition, CompositeSolidDefinition) yield SolidConfigEntry( current_handle, SolidConfig.from_dict({ 'inputs': current_solid_config.get('inputs'), 'outputs': current_solid_config.get('outputs'), }), ) # If there is a config mapping, invoke it and get the descendent solids # config that way. Else just grabs the solids entry of the current config solids_dict = (_get_mapped_solids_dict(composite_def, current_stack, current_solid_config) if composite_def.config_mapping else current_solid_config.get('solids', {})) for sce in _composite_descent(current_stack, solids_dict): yield sce
def test_whole_environment(): pipeline_def = PipelineDefinition( name="some_pipeline", mode_defs=[ ModeDefinition( name="test_mode", resource_defs={ "test_resource": ResourceDefinition(resource_fn=lambda _: None, config_schema=Any) }, ) ], solid_defs=[ SolidDefinition( name="int_config_solid", config_schema=Int, input_defs=[], output_defs=[], compute_fn=lambda *args: None, ), SolidDefinition(name="no_config_solid", input_defs=[], output_defs=[], compute_fn=lambda *args: None), ], ) env = EnvironmentConfig.build( pipeline_def, { "resources": { "test_resource": { "config": 1 } }, "solids": { "int_config_solid": { "config": 123 } }, }, ) assert isinstance(env, EnvironmentConfig) assert env.solids == { "int_config_solid": SolidConfig.from_dict({"config": 123}), "no_config_solid": SolidConfig.from_dict({}), } assert env.resources == { "test_resource": { "config": 1 }, "object_manager": {} }
def test_whole_environment(): pipeline_def = PipelineDefinition( name='some_pipeline', mode_defs=[ ModeDefinition( name='test_mode', resource_defs={ 'test_resource': ResourceDefinition(resource_fn=lambda: None, config_schema=Any) }, ) ], solid_defs=[ SolidDefinition( name='int_config_solid', config_schema=Int, input_defs=[], output_defs=[], compute_fn=lambda *args: None, ), SolidDefinition(name='no_config_solid', input_defs=[], output_defs=[], compute_fn=lambda *args: None), ], ) env = EnvironmentConfig.build( pipeline_def, { 'resources': { 'test_resource': { 'config': 1 } }, 'solids': { 'int_config_solid': { 'config': 123 } }, }, ) assert isinstance(env, EnvironmentConfig) assert env.solids == { 'int_config_solid': SolidConfig(123), 'no_config_solid': SolidConfig() } assert env.resources == {'test_resource': {'config': 1}}
def test_solid_dictionary_some_no_config(): pipeline_def = PipelineDefinition( solid_defs=[ SolidDefinition( name='int_config_solid', config_field=Field(Int), input_defs=[], output_defs=[], compute_fn=lambda *args: None, ), SolidDefinition( name='no_config_solid', input_defs=[], output_defs=[], compute_fn=lambda *args: None ), ] ) solid_dict_type = define_solid_dictionary_cls( 'foobar', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name ).inst() value = construct_solid_dictionary( throwing_evaluate_config_value(solid_dict_type, {'int_config_solid': {'config': 1}}) ) assert set(['int_config_solid']) == set(value.keys()) assert value == {'int_config_solid': SolidConfig(1)}
def test_solid_dictionary_some_no_config(): @solid(name='int_config_solid', config_field=Field(Int), input_defs=[], output_defs=[]) def int_config_solid(_): return None @solid(name='no_config_solid', input_defs=[], output_defs=[]) def no_config_solid(_): return None @pipeline def pipeline_def(): int_config_solid() no_config_solid() solid_dict_type = define_solid_dictionary_cls( 'foobar', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name).inst() value = construct_solid_dictionary( throwing_evaluate_config_value(solid_dict_type, {'int_config_solid': { 'config': 1 }})) assert set(['int_config_solid']) == set(value.keys()) assert value == {'int_config_solid': SolidConfig(1)}
def test_solid_dictionary_some_no_config(): pipeline_def = PipelineDefinition(solids=[ SolidDefinition( name='int_config_solid', config_field=Field(Int), inputs=[], outputs=[], transform_fn=lambda *args: None, ), SolidDefinition(name='no_config_solid', inputs=[], outputs=[], transform_fn=lambda *args: None), ]) solid_dict_type = define_solid_dictionary_cls( 'foobar', create_creation_data(pipeline_def)).inst() value = construct_solid_dictionary( throwing_evaluate_config_value(solid_dict_type, {'int_config_solid': { 'config': 1 }})) assert set(['int_config_solid']) == set(value.keys()) assert value == {'int_config_solid': SolidConfig(1)}
def test_whole_environment(): pipeline_def = PipelineDefinition( name='some_pipeline', mode_definitions=[ ModeDefinition( name='test_mode', resources={ 'test_resource': ResourceDefinition(resource_fn=lambda: None, config_field=Field(Any)) }, ) ], solid_defs=[ SolidDefinition( name='int_config_solid', config_field=Field(Int), inputs=[], outputs=[], compute_fn=lambda *args: None, ), SolidDefinition(name='no_config_solid', inputs=[], outputs=[], compute_fn=lambda *args: None), ], ) environment_type = create_environment_type(pipeline_def) assert (environment_type.fields['resources'].config_type.name == 'SomePipeline.Mode.TestMode.Resources') solids_type = environment_type.fields['solids'].config_type assert solids_type.name == 'SomePipeline.SolidsConfigDictionary' assert (solids_type.fields['int_config_solid'].config_type.name == 'SomePipeline.SolidConfig.IntConfigSolid') assert (environment_type.fields['expectations'].config_type.name == 'SomePipeline.ExpectationsConfig') env = EnvironmentConfig.from_dict( throwing_evaluate_config_value( environment_type, { 'resources': { 'test_resource': { 'config': 1 } }, 'solids': { 'int_config_solid': { 'config': 123 } }, }, )) assert isinstance(env, EnvironmentConfig) assert env.solids == {'int_config_solid': SolidConfig(123)} assert env.expectations == ExpectationsConfig(evaluate=True) assert env.resources == {'test_resource': {'config': 1}}
def construct_solid_dictionary(solid_dict_value): return { key: SolidConfig( config=value.get('config'), inputs=value.get('inputs', {}), outputs=value.get('outputs', []), ) for key, value in solid_dict_value.items() }
def test_whole_environment(): pipeline_def = PipelineDefinition( name='some_pipeline', context_definitions={ 'test': PipelineContextDefinition( config_field=Field(Any), context_fn=lambda *args: ExecutionContext()) }, solids=[ SolidDefinition( name='int_config_solid', config_field=Field(Int), inputs=[], outputs=[], transform_fn=lambda *args: None, ), SolidDefinition(name='no_config_solid', inputs=[], outputs=[], transform_fn=lambda *args: None), ], ) environment_type = pipeline_def.environment_type assert environment_type.fields[ 'context'].config_type.name == 'SomePipeline.ContextConfig' solids_type = environment_type.fields['solids'].config_type assert solids_type.name == 'SomePipeline.SolidsConfigDictionary' assert (solids_type.fields['int_config_solid'].config_type.name == 'SomePipeline.SolidConfig.IntConfigSolid') assert (environment_type.fields['expectations'].config_type.name == 'SomePipeline.ExpectationsConfig') env = construct_environment_config( throwing_evaluate_config_value( pipeline_def.environment_type, { 'context': { 'test': { 'config': 1 } }, 'solids': { 'int_config_solid': { 'config': 123 } } }, )) assert isinstance(env, EnvironmentConfig) assert env.context == ContextConfig('test', 1, persistence={'file': {}}) assert env.solids == {'int_config_solid': SolidConfig(123)} assert env.expectations == ExpectationsConfig(evaluate=True)
def test_solid_dictionary_type(): pipeline_def = define_test_solids_config_pipeline() solid_dict_type = define_solid_dictionary_cls( pipeline_def.solids, pipeline_def.dependency_structure) env_obj = EnvironmentConfig.build( pipeline_def, { 'solids': { 'int_config_solid': { 'config': 1 }, 'string_config_solid': { 'config': 'bar' } }, }, ) value = env_obj.solids assert set(['int_config_solid', 'string_config_solid']) == set(value.keys()) assert value == { 'int_config_solid': SolidConfig(1), 'string_config_solid': SolidConfig('bar') } assert solid_dict_type.type_attributes.is_system_config for specific_solid_config_field in solid_dict_type.fields.values(): specific_solid_config_type = specific_solid_config_field.config_type assert specific_solid_config_type.type_attributes.is_system_config user_config_field = specific_solid_config_field.config_type.fields[ 'config'] assert user_config_field.config_type.type_attributes.is_system_config is False
def test_whole_environment(): pipeline_def = PipelineDefinition( name='some_pipeline', mode_defs=[ ModeDefinition( name='test_mode', resource_defs={ 'test_resource': ResourceDefinition( resource_fn=lambda: None, config_field=Field(Any) ) }, ) ], solid_defs=[ SolidDefinition( name='int_config_solid', config_field=Field(Int), input_defs=[], output_defs=[], compute_fn=lambda *args: None, ), SolidDefinition( name='no_config_solid', input_defs=[], output_defs=[], compute_fn=lambda *args: None ), ], ) environment_type = create_environment_type(pipeline_def) env = EnvironmentConfig.from_config_value( throwing_evaluate_config_value( environment_type, { 'resources': {'test_resource': {'config': 1}}, 'solids': {'int_config_solid': {'config': 123}}, }, ), { 'resources': {'test_resource': {'config': 1}}, 'solids': {'int_config_solid': {'config': 123}}, }, ) assert isinstance(env, EnvironmentConfig) assert env.solids == {'int_config_solid': SolidConfig(123)} assert env.resources == {'test_resource': {'config': 1}}