def test_optional_solid_with_optional_scalar_config():
def _assert_config_none(context, value):
assert context.solid_config is value
pipeline_def = PipelineDefinition(
name='some_pipeline',
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int, is_optional=True),
input_defs=[],
output_defs=[],
compute_fn=lambda context, _inputs: _assert_config_none(context, 234),
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields['solids'].is_optional is True
solids_type = env_type.fields['solids'].config_type
assert solids_type.fields['int_config_solid'].is_optional is True
solids_default_obj = construct_solid_dictionary(throwing_evaluate_config_value(solids_type, {}))
assert solids_default_obj['int_config_solid'].config is None
env_obj = EnvironmentConfig.from_dict(throwing_evaluate_config_value(env_type, {}))
assert env_obj.solids['int_config_solid'].config is None
def test_optional_solid_with_optional_scalar_config():
def _assert_config_none(info, value):
assert info.config is value
pipeline_def = PipelineDefinition(
name='some_pipeline',
solids=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int, is_optional=True),
inputs=[],
outputs=[],
transform_fn=lambda info, _inputs: _assert_config_none(info, 234),
)
],
)
env_type = pipeline_def.environment_type
assert env_type.fields['solids'].is_optional is True
solids_type = env_type.fields['solids'].config_type
assert solids_type.fields['int_config_solid'].is_optional is True
solids_default_obj = construct_solid_dictionary(throwing_evaluate_config_value(solids_type, {}))
assert solids_default_obj['int_config_solid'].config is None
env_obj = construct_environment_config(
throwing_evaluate_config_value(pipeline_def.environment_type, {})
)
assert env_obj.solids['int_config_solid'].config is None
def test_select_context():
context_defs = {
'int_context': PipelineContextDefinition(
config_field=Field(Int), context_fn=lambda *args: ExecutionContext()
),
'string_context': PipelineContextDefinition(
config_field=Field(String), context_fn=lambda *args: ExecutionContext()
),
}
context_config_type = define_context_context_cls('something', context_defs).inst()
assert construct_context_config(
throwing_evaluate_config_value(context_config_type, {'int_context': {'config': 1}})
) == ContextConfig(name='int_context', config=1)
assert construct_context_config(
throwing_evaluate_config_value(context_config_type, {'string_context': {'config': 'bar'}})
) == ContextConfig(name='string_context', config='bar')
# mismatched field type mismatch
with pytest.raises(DagsterEvaluateConfigValueError):
assert throwing_evaluate_config_value(
context_config_type, {'int_context': {'config': 'bar'}}
)
# mismatched field type mismatch
with pytest.raises(DagsterEvaluateConfigValueError):
assert throwing_evaluate_config_value(
context_config_type, {'string_context': {'config': 1}}
)
def test_default_expectations():
expect_config_type = define_expectations_config_cls('some_name').inst()
assert (ExpectationsConfig(
**throwing_evaluate_config_value(expect_config_type, {})).evaluate is
True)
assert (ExpectationsConfig(
**throwing_evaluate_config_value(expect_config_type, None)).evaluate is
True)
def test_required_solid_with_required_subfield():
pipeline_def = PipelineDefinition(
name='some_pipeline',
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Dict({'required_field': Field(String)})),
input_defs=[],
output_defs=[],
compute_fn=lambda *_args: None,
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields['solids'].is_optional is False
assert env_type.fields['solids'].config_type
solids_type = env_type.fields['solids'].config_type
assert solids_type.fields['int_config_solid'].is_optional is False
int_config_solid_type = solids_type.fields['int_config_solid'].config_type
assert int_config_solid_type.fields['config'].is_optional is False
assert env_type.fields['execution'].is_optional
assert env_type.fields['expectations'].is_optional
env_obj = EnvironmentConfig.from_config_value(
throwing_evaluate_config_value(
env_type, {
'solids': {
'int_config_solid': {
'config': {
'required_field': 'foobar'
}
}
}
}),
{
'solids': {
'int_config_solid': {
'config': {
'required_field': 'foobar'
}
}
}
},
)
assert env_obj.solids['int_config_solid'].config[
'required_field'] == 'foobar'
with pytest.raises(DagsterEvaluateConfigValueError):
throwing_evaluate_config_value(env_type, {'solids': {}})
with pytest.raises(DagsterEvaluateConfigValueError):
throwing_evaluate_config_value(env_type, {})
def test_expectations_config():
expectations_config_type = define_expectations_config_cls('ksjdfkd').inst()
expectations = ExpectationsConfig(**throwing_evaluate_config_value(
expectations_config_type, {'evaluate': True}))
assert isinstance(expectations, ExpectationsConfig)
assert expectations.evaluate is True
assert ExpectationsConfig(**throwing_evaluate_config_value(
expectations_config_type, {'evaluate': False})) == ExpectationsConfig(
evaluate=False)
def test_solid_config_error():
solid_dict_type = define_solid_dictionary_cls(
'slkdfjkjdsf', define_test_solids_config_pipeline()
).inst()
int_solid_config_type = solid_dict_type.fields['int_config_solid'].config_type
with pytest.raises(DagsterEvaluateConfigValueError, match='Field "notconfig" is not defined'):
throwing_evaluate_config_value(int_solid_config_type, {'notconfig': 1})
with pytest.raises(DagsterEvaluateConfigValueError):
throwing_evaluate_config_value(int_solid_config_type, 1)
def test_solid_config_error():
pipeline_def = define_test_solids_config_pipeline()
solid_dict_type = define_solid_dictionary_cls(
'slkdfjkjdsf', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name
)
int_solid_config_type = solid_dict_type.fields['int_config_solid'].config_type
with pytest.raises(DagsterEvaluateConfigValueError, match='Field "notconfig" is not defined'):
throwing_evaluate_config_value(int_solid_config_type, {'notconfig': 1})
with pytest.raises(DagsterEvaluateConfigValueError):
throwing_evaluate_config_value(int_solid_config_type, 1)
def test_solid_dictionary_some_no_config():
pipeline_def = PipelineDefinition(solids=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
inputs=[],
outputs=[],
transform_fn=lambda *args: None,
),
SolidDefinition(name='no_config_solid',
inputs=[],
outputs=[],
transform_fn=lambda *args: None),
])
solid_dict_type = define_solid_dictionary_cls(
'foobar', create_creation_data(pipeline_def)).inst()
value = construct_solid_dictionary(
throwing_evaluate_config_value(solid_dict_type,
{'int_config_solid': {
'config': 1
}}))
assert set(['int_config_solid']) == set(value.keys())
assert value == {'int_config_solid': SolidConfig(1)}
def test_solid_dictionary_some_no_config():
@solid(name='int_config_solid',
config_field=Field(Int),
input_defs=[],
output_defs=[])
def int_config_solid(_):
return None
@solid(name='no_config_solid', input_defs=[], output_defs=[])
def no_config_solid(_):
return None
@pipeline
def pipeline_def():
int_config_solid()
no_config_solid()
solid_dict_type = define_solid_dictionary_cls(
'foobar', pipeline_def.solids, pipeline_def.dependency_structure,
pipeline_def.name).inst()
value = construct_solid_dictionary(
throwing_evaluate_config_value(solid_dict_type,
{'int_config_solid': {
'config': 1
}}))
assert set(['int_config_solid']) == set(value.keys())
assert value == {'int_config_solid': SolidConfig(1)}
def test_solid_dictionary_type():
pipeline_def = define_test_solids_config_pipeline()
solid_dict_type = define_solid_dictionary_cls(
'foobar', pipeline_def.solids, pipeline_def.dependency_structure,
pipeline_def.name).inst()
value = construct_solid_dictionary(
throwing_evaluate_config_value(
solid_dict_type,
{
'int_config_solid': {
'config': 1
},
'string_config_solid': {
'config': 'bar'
}
},
))
assert set(['int_config_solid',
'string_config_solid']) == set(value.keys())
assert value == {
'int_config_solid': SolidConfig(1),
'string_config_solid': SolidConfig('bar')
}
assert solid_dict_type.type_attributes.is_system_config
for specific_solid_config_field in solid_dict_type.fields.values():
specific_solid_config_type = specific_solid_config_field.config_type
assert specific_solid_config_type.type_attributes.is_system_config
user_config_field = specific_solid_config_field.config_type.fields[
'config']
assert user_config_field.config_type.type_attributes.is_system_config is False
def test_solid_dictionary_some_no_config():
pipeline_def = PipelineDefinition(
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(
name='no_config_solid', input_defs=[], output_defs=[], compute_fn=lambda *args: None
),
]
)
solid_dict_type = define_solid_dictionary_cls(
'foobar', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name
).inst()
value = construct_solid_dictionary(
throwing_evaluate_config_value(solid_dict_type, {'int_config_solid': {'config': 1}})
)
assert set(['int_config_solid']) == set(value.keys())
assert value == {'int_config_solid': SolidConfig(1)}
def test_whole_environment():
pipeline_def = PipelineDefinition(
name='some_pipeline',
mode_definitions=[
ModeDefinition(
name='test_mode',
resources={
'test_resource':
ResourceDefinition(resource_fn=lambda: None,
config_field=Field(Any))
},
)
],
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
inputs=[],
outputs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(name='no_config_solid',
inputs=[],
outputs=[],
compute_fn=lambda *args: None),
],
)
environment_type = create_environment_type(pipeline_def)
assert (environment_type.fields['resources'].config_type.name ==
'SomePipeline.Mode.TestMode.Resources')
solids_type = environment_type.fields['solids'].config_type
assert solids_type.name == 'SomePipeline.SolidsConfigDictionary'
assert (solids_type.fields['int_config_solid'].config_type.name ==
'SomePipeline.SolidConfig.IntConfigSolid')
assert (environment_type.fields['expectations'].config_type.name ==
'SomePipeline.ExpectationsConfig')
env = EnvironmentConfig.from_dict(
throwing_evaluate_config_value(
environment_type,
{
'resources': {
'test_resource': {
'config': 1
}
},
'solids': {
'int_config_solid': {
'config': 123
}
},
},
))
assert isinstance(env, EnvironmentConfig)
assert env.solids == {'int_config_solid': SolidConfig(123)}
assert env.expectations == ExpectationsConfig(evaluate=True)
assert env.resources == {'test_resource': {'config': 1}}
def test_provided_default_config():
pipeline_def = PipelineDefinition(
context_definitions={
'some_context': PipelineContextDefinition(
config_field=Field(
Dict({'with_default_int': Field(Int, is_optional=True, default_value=23434)})
),
context_fn=lambda *args: None,
)
},
solids=[
SolidDefinition(
name='some_solid', inputs=[], outputs=[], transform_fn=lambda *args: None
)
],
)
env_type = pipeline_def.environment_type
some_context_field = env_type.fields['context'].config_type.fields['some_context']
assert some_context_field.is_optional
some_context_config_field = some_context_field.config_type.fields['config']
assert some_context_config_field.is_optional
assert some_context_config_field.default_value == {'with_default_int': 23434}
assert some_context_field.default_value == {
'config': {'with_default_int': 23434},
'resources': {},
}
value = construct_environment_config(
throwing_evaluate_config_value(pipeline_def.environment_type, {})
)
assert value.context.name == 'some_context'
assert env_type.type_attributes.is_system_config
def test_solid_config():
solid_config_type = define_solid_config_cls('kdjfkd', Field(Int), None,
None).inst()
solid_inst = throwing_evaluate_config_value(solid_config_type,
{'config': 1})
assert solid_inst['config'] == 1
assert solid_config_type.inst().type_attributes.is_system_config
def test_resource_config_any():
resource_type = define_resource_cls(
'Parent', 'Foo', ResourceDefinition(lambda: None, Field(Any))
).inst()
assert resource_type.type_attributes.is_system_config
assert throwing_evaluate_config_value(resource_type, {'config': 1}) == {'config': 1}
def test_files_default_config():
pipeline_def = PipelineDefinition(name='pipeline', solid_defs=[])
env_type = create_environment_type(pipeline_def)
assert 'storage' in env_type.fields
config_value = throwing_evaluate_config_value(env_type, {})
assert 'storage' not in config_value
def test_storage_in_memory_config():
pipeline_def = PipelineDefinition(name='pipeline', solid_defs=[])
env_type = create_environment_type(pipeline_def)
assert 'storage' in env_type.fields
config_value = throwing_evaluate_config_value(env_type, {'storage': {'in_memory': {}}})
assert config_value['storage'] == {'in_memory': {}}
def test_optional_and_required_context():
pipeline_def = PipelineDefinition(
name='some_pipeline',
solid_defs=[],
mode_defs=[
ModeDefinition(
name='mixed',
resource_defs={
'optional_resource': ResourceDefinition(
lambda: None,
config_field=Field(
dagster_type=Dict(
fields={'optional_field': Field(String, is_optional=True)}
)
),
),
'required_resource': ResourceDefinition(
lambda: None,
config_field=Field(
dagster_type=Dict(fields={'required_field': Field(String)})
),
),
},
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields['solids'].is_optional
assert env_type.fields['execution'].is_optional
assert env_type.fields['expectations'].is_optional
assert nested_field(env_type, 'resources').is_required
assert nested_field(env_type, 'resources', 'optional_resource').is_optional
assert nested_field(env_type, 'resources', 'optional_resource', 'config').is_optional
assert nested_field(
env_type, 'resources', 'optional_resource', 'config', 'optional_field'
).is_optional
assert nested_field(env_type, 'resources', 'required_resource').is_required
assert nested_field(env_type, 'resources', 'required_resource', 'config').is_required
assert nested_field(
env_type, 'resources', 'required_resource', 'config', 'required_field'
).is_required
env_obj = EnvironmentConfig.from_dict(
throwing_evaluate_config_value(
env_type, {'resources': {'required_resource': {'config': {'required_field': 'foo'}}}}
)
)
assert env_obj.resources == {
'optional_resource': {'config': {}},
'required_resource': {'config': {'required_field': 'foo'}},
}
def test_whole_environment():
pipeline_def = PipelineDefinition(
name='some_pipeline',
context_definitions={
'test':
PipelineContextDefinition(
config_field=Field(Any),
context_fn=lambda *args: ExecutionContext())
},
solids=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
inputs=[],
outputs=[],
transform_fn=lambda *args: None,
),
SolidDefinition(name='no_config_solid',
inputs=[],
outputs=[],
transform_fn=lambda *args: None),
],
)
environment_type = pipeline_def.environment_type
assert environment_type.fields[
'context'].config_type.name == 'SomePipeline.ContextConfig'
solids_type = environment_type.fields['solids'].config_type
assert solids_type.name == 'SomePipeline.SolidsConfigDictionary'
assert (solids_type.fields['int_config_solid'].config_type.name ==
'SomePipeline.SolidConfig.IntConfigSolid')
assert (environment_type.fields['expectations'].config_type.name ==
'SomePipeline.ExpectationsConfig')
env = construct_environment_config(
throwing_evaluate_config_value(
pipeline_def.environment_type,
{
'context': {
'test': {
'config': 1
}
},
'solids': {
'int_config_solid': {
'config': 123
}
}
},
))
assert isinstance(env, EnvironmentConfig)
assert env.context == ContextConfig('test', 1, persistence={'file': {}})
assert env.solids == {'int_config_solid': SolidConfig(123)}
assert env.expectations == ExpectationsConfig(evaluate=True)
def test_provided_default_on_resources_config():
pipeline_def = PipelineDefinition(
mode_definitions=[
ModeDefinition(
name='some_mode',
resources={
'some_resource':
ResourceDefinition(
resource_fn=lambda: None,
config_field=Field(
Dict({
'with_default_int':
Field(Int,
is_optional=True,
default_value=23434)
})),
)
},
)
],
solid_defs=[
SolidDefinition(name='some_solid',
inputs=[],
outputs=[],
compute_fn=lambda *args: None)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.type_attributes.is_system_config
some_resource_field = env_type.fields['resources'].config_type.fields[
'some_resource']
assert some_resource_field.is_optional
some_resource_config_field = some_resource_field.config_type.fields[
'config']
assert some_resource_config_field.is_optional
assert some_resource_config_field.default_value == {
'with_default_int': 23434
}
assert some_resource_field.default_value == {
'config': {
'with_default_int': 23434
}
}
value = EnvironmentConfig.from_dict(
throwing_evaluate_config_value(env_type, {}))
assert value.resources == {
'some_resource': {
'config': {
'with_default_int': 23434
}
}
}
def test_default_environment():
@solid(name='some_solid', input_defs=[], output_defs=[])
def some_solid(_):
return None
@pipeline
def pipeline_def():
some_solid()
EnvironmentConfig.from_config_value(
throwing_evaluate_config_value(create_environment_type(pipeline_def),
{}), {})
def test_default_environment():
pipeline_def = PipelineDefinition(solids=[
SolidDefinition(name='some_solid',
inputs=[],
outputs=[],
transform_fn=lambda *args: None)
])
env_obj = construct_environment_config(
throwing_evaluate_config_value(pipeline_def.environment_type, {}))
assert env_obj.expectations.evaluate is True
def test_default_environment():
pipeline_def = PipelineDefinition(
solids=[
SolidDefinition(name='some_solid', inputs=[], outputs=[], compute_fn=lambda *args: None)
]
)
env_obj = EnvironmentConfig.from_dict(
throwing_evaluate_config_value(create_environment_type(pipeline_def), {})
)
assert env_obj.expectations.evaluate is True
def test_default_environment():
@solid(name='some_solid', input_defs=[], output_defs=[])
def some_solid(_):
return None
@pipeline
def pipeline_def():
some_solid()
env_obj = EnvironmentConfig.from_config_value(
throwing_evaluate_config_value(create_environment_type(pipeline_def),
{}), {})
assert env_obj.expectations.evaluate is True
def test_optional_and_required_context():
pipeline_def = PipelineDefinition(
name='some_pipeline',
solids=[],
context_definitions={
'optional_field_context':
PipelineContextDefinition(
context_fn=lambda *args: None,
config_field=Field(dagster_type=Dict(
fields={'optional_field': Field(String, is_optional=True)
})),
),
'required_field_context':
PipelineContextDefinition(
context_fn=lambda *args: None,
config_field=Field(dagster_type=Dict(
fields={'required_field': Field(String)})),
),
},
)
env_type = pipeline_def.environment_type
assert env_type.fields['solids'].is_optional
assert env_type.fields['context'].is_optional is False
context_type = env_type.fields['context'].config_type
assert context_type.fields['optional_field_context'].is_optional
assert context_type.fields['required_field_context'].is_optional
assert env_type.fields['execution'].is_optional
assert env_type.fields['expectations'].is_optional
env_obj = construct_environment_config(
throwing_evaluate_config_value(
pipeline_def.environment_type,
{
'context': {
'optional_field_context': {
'config': {
'optional_field': 'foobar'
}
}
}
},
))
assert env_obj.context.name == 'optional_field_context'
assert env_obj.context.config == {'optional_field': 'foobar'}
def test_context_config_any():
context_defs = {
'test': PipelineContextDefinition(
config_field=Field(Any), context_fn=lambda *args: ExecutionContext()
)
}
context_config_type = define_context_context_cls('something', context_defs).inst()
assert context_config_type.type_attributes.is_system_config
output = construct_context_config(
throwing_evaluate_config_value(context_config_type, {'test': {'config': 1}})
)
assert output.name == 'test'
assert output.config == 1
def test_whole_environment():
pipeline_def = PipelineDefinition(
name='some_pipeline',
mode_defs=[
ModeDefinition(
name='test_mode',
resource_defs={
'test_resource': ResourceDefinition(
resource_fn=lambda: None, config_field=Field(Any)
)
},
)
],
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(
name='no_config_solid', input_defs=[], output_defs=[], compute_fn=lambda *args: None
),
],
)
environment_type = create_environment_type(pipeline_def)
env = EnvironmentConfig.from_config_value(
throwing_evaluate_config_value(
environment_type,
{
'resources': {'test_resource': {'config': 1}},
'solids': {'int_config_solid': {'config': 123}},
},
),
{
'resources': {'test_resource': {'config': 1}},
'solids': {'int_config_solid': {'config': 123}},
},
)
assert isinstance(env, EnvironmentConfig)
assert env.solids == {'int_config_solid': SolidConfig(123)}
assert env.resources == {'test_resource': {'config': 1}}
def test_context_config():
context_defs = {
'test': PipelineContextDefinition(
config_field=Field(dagster_type=Dict({'some_str': Field(String)})),
context_fn=lambda *args: ExecutionContext(),
)
}
context_config_type = define_context_context_cls('something', context_defs).inst()
output = construct_context_config(
throwing_evaluate_config_value(
context_config_type, {'test': {'config': {'some_str': 'something'}}}
)
)
assert isinstance(output, ContextConfig)
assert output.name == 'test'
assert output.config == {'some_str': 'something'}
def test_default_context_config():
pipeline_def = PipelineDefinition(solids=[
SolidDefinition(name='some_solid',
inputs=[],
outputs=[],
transform_fn=lambda *args: None)
])
context_config_type = define_context_context_cls(
pipeline_def.name, pipeline_def.context_definitions).inst()
assert 'default' in context_config_type.fields
assert context_config_type.fields['default'].is_optional
default_context_config_type = context_config_type.fields[
'default'].config_type
assert 'config' in default_context_config_type.fields
context_dict = throwing_evaluate_config_value(context_config_type, {})
assert 'default' in context_dict
