def test_optional_solid_with_optional_scalar_config(): def _assert_config_none(context, value): assert context.solid_config is value pipeline_def = PipelineDefinition( name='some_pipeline', solid_defs=[ SolidDefinition( name='int_config_solid', config_field=Field(Int, is_optional=True), input_defs=[], output_defs=[], compute_fn=lambda context, _inputs: _assert_config_none(context, 234), ) ], ) env_type = create_environment_type(pipeline_def) assert env_type.fields['solids'].is_optional is True solids_type = env_type.fields['solids'].config_type assert solids_type.fields['int_config_solid'].is_optional is True solids_default_obj = construct_solid_dictionary(throwing_evaluate_config_value(solids_type, {})) assert solids_default_obj['int_config_solid'].config is None env_obj = EnvironmentConfig.from_dict(throwing_evaluate_config_value(env_type, {})) assert env_obj.solids['int_config_solid'].config is None
def test_optional_solid_with_optional_scalar_config(): def _assert_config_none(info, value): assert info.config is value pipeline_def = PipelineDefinition( name='some_pipeline', solids=[ SolidDefinition( name='int_config_solid', config_field=Field(Int, is_optional=True), inputs=[], outputs=[], transform_fn=lambda info, _inputs: _assert_config_none(info, 234), ) ], ) env_type = pipeline_def.environment_type assert env_type.fields['solids'].is_optional is True solids_type = env_type.fields['solids'].config_type assert solids_type.fields['int_config_solid'].is_optional is True solids_default_obj = construct_solid_dictionary(throwing_evaluate_config_value(solids_type, {})) assert solids_default_obj['int_config_solid'].config is None env_obj = construct_environment_config( throwing_evaluate_config_value(pipeline_def.environment_type, {}) ) assert env_obj.solids['int_config_solid'].config is None
def test_select_context(): context_defs = { 'int_context': PipelineContextDefinition( config_field=Field(Int), context_fn=lambda *args: ExecutionContext() ), 'string_context': PipelineContextDefinition( config_field=Field(String), context_fn=lambda *args: ExecutionContext() ), } context_config_type = define_context_context_cls('something', context_defs).inst() assert construct_context_config( throwing_evaluate_config_value(context_config_type, {'int_context': {'config': 1}}) ) == ContextConfig(name='int_context', config=1) assert construct_context_config( throwing_evaluate_config_value(context_config_type, {'string_context': {'config': 'bar'}}) ) == ContextConfig(name='string_context', config='bar') # mismatched field type mismatch with pytest.raises(DagsterEvaluateConfigValueError): assert throwing_evaluate_config_value( context_config_type, {'int_context': {'config': 'bar'}} ) # mismatched field type mismatch with pytest.raises(DagsterEvaluateConfigValueError): assert throwing_evaluate_config_value( context_config_type, {'string_context': {'config': 1}} )
def test_default_expectations(): expect_config_type = define_expectations_config_cls('some_name').inst() assert (ExpectationsConfig( **throwing_evaluate_config_value(expect_config_type, {})).evaluate is True) assert (ExpectationsConfig( **throwing_evaluate_config_value(expect_config_type, None)).evaluate is True)
def test_required_solid_with_required_subfield(): pipeline_def = PipelineDefinition( name='some_pipeline', solid_defs=[ SolidDefinition( name='int_config_solid', config_field=Field(Dict({'required_field': Field(String)})), input_defs=[], output_defs=[], compute_fn=lambda *_args: None, ) ], ) env_type = create_environment_type(pipeline_def) assert env_type.fields['solids'].is_optional is False assert env_type.fields['solids'].config_type solids_type = env_type.fields['solids'].config_type assert solids_type.fields['int_config_solid'].is_optional is False int_config_solid_type = solids_type.fields['int_config_solid'].config_type assert int_config_solid_type.fields['config'].is_optional is False assert env_type.fields['execution'].is_optional assert env_type.fields['expectations'].is_optional env_obj = EnvironmentConfig.from_config_value( throwing_evaluate_config_value( env_type, { 'solids': { 'int_config_solid': { 'config': { 'required_field': 'foobar' } } } }), { 'solids': { 'int_config_solid': { 'config': { 'required_field': 'foobar' } } } }, ) assert env_obj.solids['int_config_solid'].config[ 'required_field'] == 'foobar' with pytest.raises(DagsterEvaluateConfigValueError): throwing_evaluate_config_value(env_type, {'solids': {}}) with pytest.raises(DagsterEvaluateConfigValueError): throwing_evaluate_config_value(env_type, {})
def test_expectations_config(): expectations_config_type = define_expectations_config_cls('ksjdfkd').inst() expectations = ExpectationsConfig(**throwing_evaluate_config_value( expectations_config_type, {'evaluate': True})) assert isinstance(expectations, ExpectationsConfig) assert expectations.evaluate is True assert ExpectationsConfig(**throwing_evaluate_config_value( expectations_config_type, {'evaluate': False})) == ExpectationsConfig( evaluate=False)
def test_solid_config_error(): solid_dict_type = define_solid_dictionary_cls( 'slkdfjkjdsf', define_test_solids_config_pipeline() ).inst() int_solid_config_type = solid_dict_type.fields['int_config_solid'].config_type with pytest.raises(DagsterEvaluateConfigValueError, match='Field "notconfig" is not defined'): throwing_evaluate_config_value(int_solid_config_type, {'notconfig': 1}) with pytest.raises(DagsterEvaluateConfigValueError): throwing_evaluate_config_value(int_solid_config_type, 1)
def test_solid_config_error(): pipeline_def = define_test_solids_config_pipeline() solid_dict_type = define_solid_dictionary_cls( 'slkdfjkjdsf', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name ) int_solid_config_type = solid_dict_type.fields['int_config_solid'].config_type with pytest.raises(DagsterEvaluateConfigValueError, match='Field "notconfig" is not defined'): throwing_evaluate_config_value(int_solid_config_type, {'notconfig': 1}) with pytest.raises(DagsterEvaluateConfigValueError): throwing_evaluate_config_value(int_solid_config_type, 1)
def test_solid_dictionary_some_no_config(): pipeline_def = PipelineDefinition(solids=[ SolidDefinition( name='int_config_solid', config_field=Field(Int), inputs=[], outputs=[], transform_fn=lambda *args: None, ), SolidDefinition(name='no_config_solid', inputs=[], outputs=[], transform_fn=lambda *args: None), ]) solid_dict_type = define_solid_dictionary_cls( 'foobar', create_creation_data(pipeline_def)).inst() value = construct_solid_dictionary( throwing_evaluate_config_value(solid_dict_type, {'int_config_solid': { 'config': 1 }})) assert set(['int_config_solid']) == set(value.keys()) assert value == {'int_config_solid': SolidConfig(1)}
def test_solid_dictionary_some_no_config(): @solid(name='int_config_solid', config_field=Field(Int), input_defs=[], output_defs=[]) def int_config_solid(_): return None @solid(name='no_config_solid', input_defs=[], output_defs=[]) def no_config_solid(_): return None @pipeline def pipeline_def(): int_config_solid() no_config_solid() solid_dict_type = define_solid_dictionary_cls( 'foobar', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name).inst() value = construct_solid_dictionary( throwing_evaluate_config_value(solid_dict_type, {'int_config_solid': { 'config': 1 }})) assert set(['int_config_solid']) == set(value.keys()) assert value == {'int_config_solid': SolidConfig(1)}
def test_solid_dictionary_type(): pipeline_def = define_test_solids_config_pipeline() solid_dict_type = define_solid_dictionary_cls( 'foobar', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name).inst() value = construct_solid_dictionary( throwing_evaluate_config_value( solid_dict_type, { 'int_config_solid': { 'config': 1 }, 'string_config_solid': { 'config': 'bar' } }, )) assert set(['int_config_solid', 'string_config_solid']) == set(value.keys()) assert value == { 'int_config_solid': SolidConfig(1), 'string_config_solid': SolidConfig('bar') } assert solid_dict_type.type_attributes.is_system_config for specific_solid_config_field in solid_dict_type.fields.values(): specific_solid_config_type = specific_solid_config_field.config_type assert specific_solid_config_type.type_attributes.is_system_config user_config_field = specific_solid_config_field.config_type.fields[ 'config'] assert user_config_field.config_type.type_attributes.is_system_config is False
def test_solid_dictionary_some_no_config(): pipeline_def = PipelineDefinition( solid_defs=[ SolidDefinition( name='int_config_solid', config_field=Field(Int), input_defs=[], output_defs=[], compute_fn=lambda *args: None, ), SolidDefinition( name='no_config_solid', input_defs=[], output_defs=[], compute_fn=lambda *args: None ), ] ) solid_dict_type = define_solid_dictionary_cls( 'foobar', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name ).inst() value = construct_solid_dictionary( throwing_evaluate_config_value(solid_dict_type, {'int_config_solid': {'config': 1}}) ) assert set(['int_config_solid']) == set(value.keys()) assert value == {'int_config_solid': SolidConfig(1)}
def test_whole_environment(): pipeline_def = PipelineDefinition( name='some_pipeline', mode_definitions=[ ModeDefinition( name='test_mode', resources={ 'test_resource': ResourceDefinition(resource_fn=lambda: None, config_field=Field(Any)) }, ) ], solid_defs=[ SolidDefinition( name='int_config_solid', config_field=Field(Int), inputs=[], outputs=[], compute_fn=lambda *args: None, ), SolidDefinition(name='no_config_solid', inputs=[], outputs=[], compute_fn=lambda *args: None), ], ) environment_type = create_environment_type(pipeline_def) assert (environment_type.fields['resources'].config_type.name == 'SomePipeline.Mode.TestMode.Resources') solids_type = environment_type.fields['solids'].config_type assert solids_type.name == 'SomePipeline.SolidsConfigDictionary' assert (solids_type.fields['int_config_solid'].config_type.name == 'SomePipeline.SolidConfig.IntConfigSolid') assert (environment_type.fields['expectations'].config_type.name == 'SomePipeline.ExpectationsConfig') env = EnvironmentConfig.from_dict( throwing_evaluate_config_value( environment_type, { 'resources': { 'test_resource': { 'config': 1 } }, 'solids': { 'int_config_solid': { 'config': 123 } }, }, )) assert isinstance(env, EnvironmentConfig) assert env.solids == {'int_config_solid': SolidConfig(123)} assert env.expectations == ExpectationsConfig(evaluate=True) assert env.resources == {'test_resource': {'config': 1}}
def test_provided_default_config(): pipeline_def = PipelineDefinition( context_definitions={ 'some_context': PipelineContextDefinition( config_field=Field( Dict({'with_default_int': Field(Int, is_optional=True, default_value=23434)}) ), context_fn=lambda *args: None, ) }, solids=[ SolidDefinition( name='some_solid', inputs=[], outputs=[], transform_fn=lambda *args: None ) ], ) env_type = pipeline_def.environment_type some_context_field = env_type.fields['context'].config_type.fields['some_context'] assert some_context_field.is_optional some_context_config_field = some_context_field.config_type.fields['config'] assert some_context_config_field.is_optional assert some_context_config_field.default_value == {'with_default_int': 23434} assert some_context_field.default_value == { 'config': {'with_default_int': 23434}, 'resources': {}, } value = construct_environment_config( throwing_evaluate_config_value(pipeline_def.environment_type, {}) ) assert value.context.name == 'some_context' assert env_type.type_attributes.is_system_config
def test_solid_config(): solid_config_type = define_solid_config_cls('kdjfkd', Field(Int), None, None).inst() solid_inst = throwing_evaluate_config_value(solid_config_type, {'config': 1}) assert solid_inst['config'] == 1 assert solid_config_type.inst().type_attributes.is_system_config
def test_resource_config_any(): resource_type = define_resource_cls( 'Parent', 'Foo', ResourceDefinition(lambda: None, Field(Any)) ).inst() assert resource_type.type_attributes.is_system_config assert throwing_evaluate_config_value(resource_type, {'config': 1}) == {'config': 1}
def test_files_default_config(): pipeline_def = PipelineDefinition(name='pipeline', solid_defs=[]) env_type = create_environment_type(pipeline_def) assert 'storage' in env_type.fields config_value = throwing_evaluate_config_value(env_type, {}) assert 'storage' not in config_value
def test_storage_in_memory_config(): pipeline_def = PipelineDefinition(name='pipeline', solid_defs=[]) env_type = create_environment_type(pipeline_def) assert 'storage' in env_type.fields config_value = throwing_evaluate_config_value(env_type, {'storage': {'in_memory': {}}}) assert config_value['storage'] == {'in_memory': {}}
def test_optional_and_required_context(): pipeline_def = PipelineDefinition( name='some_pipeline', solid_defs=[], mode_defs=[ ModeDefinition( name='mixed', resource_defs={ 'optional_resource': ResourceDefinition( lambda: None, config_field=Field( dagster_type=Dict( fields={'optional_field': Field(String, is_optional=True)} ) ), ), 'required_resource': ResourceDefinition( lambda: None, config_field=Field( dagster_type=Dict(fields={'required_field': Field(String)}) ), ), }, ) ], ) env_type = create_environment_type(pipeline_def) assert env_type.fields['solids'].is_optional assert env_type.fields['execution'].is_optional assert env_type.fields['expectations'].is_optional assert nested_field(env_type, 'resources').is_required assert nested_field(env_type, 'resources', 'optional_resource').is_optional assert nested_field(env_type, 'resources', 'optional_resource', 'config').is_optional assert nested_field( env_type, 'resources', 'optional_resource', 'config', 'optional_field' ).is_optional assert nested_field(env_type, 'resources', 'required_resource').is_required assert nested_field(env_type, 'resources', 'required_resource', 'config').is_required assert nested_field( env_type, 'resources', 'required_resource', 'config', 'required_field' ).is_required env_obj = EnvironmentConfig.from_dict( throwing_evaluate_config_value( env_type, {'resources': {'required_resource': {'config': {'required_field': 'foo'}}}} ) ) assert env_obj.resources == { 'optional_resource': {'config': {}}, 'required_resource': {'config': {'required_field': 'foo'}}, }
def test_whole_environment(): pipeline_def = PipelineDefinition( name='some_pipeline', context_definitions={ 'test': PipelineContextDefinition( config_field=Field(Any), context_fn=lambda *args: ExecutionContext()) }, solids=[ SolidDefinition( name='int_config_solid', config_field=Field(Int), inputs=[], outputs=[], transform_fn=lambda *args: None, ), SolidDefinition(name='no_config_solid', inputs=[], outputs=[], transform_fn=lambda *args: None), ], ) environment_type = pipeline_def.environment_type assert environment_type.fields[ 'context'].config_type.name == 'SomePipeline.ContextConfig' solids_type = environment_type.fields['solids'].config_type assert solids_type.name == 'SomePipeline.SolidsConfigDictionary' assert (solids_type.fields['int_config_solid'].config_type.name == 'SomePipeline.SolidConfig.IntConfigSolid') assert (environment_type.fields['expectations'].config_type.name == 'SomePipeline.ExpectationsConfig') env = construct_environment_config( throwing_evaluate_config_value( pipeline_def.environment_type, { 'context': { 'test': { 'config': 1 } }, 'solids': { 'int_config_solid': { 'config': 123 } } }, )) assert isinstance(env, EnvironmentConfig) assert env.context == ContextConfig('test', 1, persistence={'file': {}}) assert env.solids == {'int_config_solid': SolidConfig(123)} assert env.expectations == ExpectationsConfig(evaluate=True)
def test_provided_default_on_resources_config(): pipeline_def = PipelineDefinition( mode_definitions=[ ModeDefinition( name='some_mode', resources={ 'some_resource': ResourceDefinition( resource_fn=lambda: None, config_field=Field( Dict({ 'with_default_int': Field(Int, is_optional=True, default_value=23434) })), ) }, ) ], solid_defs=[ SolidDefinition(name='some_solid', inputs=[], outputs=[], compute_fn=lambda *args: None) ], ) env_type = create_environment_type(pipeline_def) assert env_type.type_attributes.is_system_config some_resource_field = env_type.fields['resources'].config_type.fields[ 'some_resource'] assert some_resource_field.is_optional some_resource_config_field = some_resource_field.config_type.fields[ 'config'] assert some_resource_config_field.is_optional assert some_resource_config_field.default_value == { 'with_default_int': 23434 } assert some_resource_field.default_value == { 'config': { 'with_default_int': 23434 } } value = EnvironmentConfig.from_dict( throwing_evaluate_config_value(env_type, {})) assert value.resources == { 'some_resource': { 'config': { 'with_default_int': 23434 } } }
def test_default_environment(): @solid(name='some_solid', input_defs=[], output_defs=[]) def some_solid(_): return None @pipeline def pipeline_def(): some_solid() EnvironmentConfig.from_config_value( throwing_evaluate_config_value(create_environment_type(pipeline_def), {}), {})
def test_default_environment(): pipeline_def = PipelineDefinition(solids=[ SolidDefinition(name='some_solid', inputs=[], outputs=[], transform_fn=lambda *args: None) ]) env_obj = construct_environment_config( throwing_evaluate_config_value(pipeline_def.environment_type, {})) assert env_obj.expectations.evaluate is True
def test_default_environment(): pipeline_def = PipelineDefinition( solids=[ SolidDefinition(name='some_solid', inputs=[], outputs=[], compute_fn=lambda *args: None) ] ) env_obj = EnvironmentConfig.from_dict( throwing_evaluate_config_value(create_environment_type(pipeline_def), {}) ) assert env_obj.expectations.evaluate is True
def test_default_environment(): @solid(name='some_solid', input_defs=[], output_defs=[]) def some_solid(_): return None @pipeline def pipeline_def(): some_solid() env_obj = EnvironmentConfig.from_config_value( throwing_evaluate_config_value(create_environment_type(pipeline_def), {}), {}) assert env_obj.expectations.evaluate is True
def test_optional_and_required_context(): pipeline_def = PipelineDefinition( name='some_pipeline', solids=[], context_definitions={ 'optional_field_context': PipelineContextDefinition( context_fn=lambda *args: None, config_field=Field(dagster_type=Dict( fields={'optional_field': Field(String, is_optional=True) })), ), 'required_field_context': PipelineContextDefinition( context_fn=lambda *args: None, config_field=Field(dagster_type=Dict( fields={'required_field': Field(String)})), ), }, ) env_type = pipeline_def.environment_type assert env_type.fields['solids'].is_optional assert env_type.fields['context'].is_optional is False context_type = env_type.fields['context'].config_type assert context_type.fields['optional_field_context'].is_optional assert context_type.fields['required_field_context'].is_optional assert env_type.fields['execution'].is_optional assert env_type.fields['expectations'].is_optional env_obj = construct_environment_config( throwing_evaluate_config_value( pipeline_def.environment_type, { 'context': { 'optional_field_context': { 'config': { 'optional_field': 'foobar' } } } }, )) assert env_obj.context.name == 'optional_field_context' assert env_obj.context.config == {'optional_field': 'foobar'}
def test_context_config_any(): context_defs = { 'test': PipelineContextDefinition( config_field=Field(Any), context_fn=lambda *args: ExecutionContext() ) } context_config_type = define_context_context_cls('something', context_defs).inst() assert context_config_type.type_attributes.is_system_config output = construct_context_config( throwing_evaluate_config_value(context_config_type, {'test': {'config': 1}}) ) assert output.name == 'test' assert output.config == 1
def test_whole_environment(): pipeline_def = PipelineDefinition( name='some_pipeline', mode_defs=[ ModeDefinition( name='test_mode', resource_defs={ 'test_resource': ResourceDefinition( resource_fn=lambda: None, config_field=Field(Any) ) }, ) ], solid_defs=[ SolidDefinition( name='int_config_solid', config_field=Field(Int), input_defs=[], output_defs=[], compute_fn=lambda *args: None, ), SolidDefinition( name='no_config_solid', input_defs=[], output_defs=[], compute_fn=lambda *args: None ), ], ) environment_type = create_environment_type(pipeline_def) env = EnvironmentConfig.from_config_value( throwing_evaluate_config_value( environment_type, { 'resources': {'test_resource': {'config': 1}}, 'solids': {'int_config_solid': {'config': 123}}, }, ), { 'resources': {'test_resource': {'config': 1}}, 'solids': {'int_config_solid': {'config': 123}}, }, ) assert isinstance(env, EnvironmentConfig) assert env.solids == {'int_config_solid': SolidConfig(123)} assert env.resources == {'test_resource': {'config': 1}}
def test_context_config(): context_defs = { 'test': PipelineContextDefinition( config_field=Field(dagster_type=Dict({'some_str': Field(String)})), context_fn=lambda *args: ExecutionContext(), ) } context_config_type = define_context_context_cls('something', context_defs).inst() output = construct_context_config( throwing_evaluate_config_value( context_config_type, {'test': {'config': {'some_str': 'something'}}} ) ) assert isinstance(output, ContextConfig) assert output.name == 'test' assert output.config == {'some_str': 'something'}
def test_default_context_config(): pipeline_def = PipelineDefinition(solids=[ SolidDefinition(name='some_solid', inputs=[], outputs=[], transform_fn=lambda *args: None) ]) context_config_type = define_context_context_cls( pipeline_def.name, pipeline_def.context_definitions).inst() assert 'default' in context_config_type.fields assert context_config_type.fields['default'].is_optional default_context_config_type = context_config_type.fields[ 'default'].config_type assert 'config' in default_context_config_type.fields context_dict = throwing_evaluate_config_value(context_config_type, {}) assert 'default' in context_dict