def test_whole_environment():
pipeline_def = PipelineDefinition(
name='some_pipeline',
mode_definitions=[
ModeDefinition(
name='test_mode',
resources={
'test_resource':
ResourceDefinition(resource_fn=lambda: None,
config_field=Field(Any))
},
)
],
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
inputs=[],
outputs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(name='no_config_solid',
inputs=[],
outputs=[],
compute_fn=lambda *args: None),
],
)
environment_type = create_environment_type(pipeline_def)
assert (environment_type.fields['resources'].config_type.name ==
'SomePipeline.Mode.TestMode.Resources')
solids_type = environment_type.fields['solids'].config_type
assert solids_type.name == 'SomePipeline.SolidsConfigDictionary'
assert (solids_type.fields['int_config_solid'].config_type.name ==
'SomePipeline.SolidConfig.IntConfigSolid')
assert (environment_type.fields['expectations'].config_type.name ==
'SomePipeline.ExpectationsConfig')
env = EnvironmentConfig.from_dict(
throwing_evaluate_config_value(
environment_type,
{
'resources': {
'test_resource': {
'config': 1
}
},
'solids': {
'int_config_solid': {
'config': 123
}
},
},
))
assert isinstance(env, EnvironmentConfig)
assert env.solids == {'int_config_solid': SolidConfig(123)}
assert env.expectations == ExpectationsConfig(evaluate=True)
assert env.resources == {'test_resource': {'config': 1}}
def test_optional_solid_with_optional_scalar_config():
def _assert_config_none(context, value):
assert context.solid_config is value
pipeline_def = PipelineDefinition(
name='some_pipeline',
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int, is_optional=True),
input_defs=[],
output_defs=[],
compute_fn=lambda context, _inputs: _assert_config_none(context, 234),
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields['solids'].is_optional is True
solids_type = env_type.fields['solids'].config_type
assert solids_type.fields['int_config_solid'].is_optional is True
solids_default_obj = construct_solid_dictionary(throwing_evaluate_config_value(solids_type, {}))
assert solids_default_obj['int_config_solid'].config is None
env_obj = EnvironmentConfig.from_dict(throwing_evaluate_config_value(env_type, {}))
assert env_obj.solids['int_config_solid'].config is None
def test_optional_and_required_context():
pipeline_def = PipelineDefinition(
name='some_pipeline',
solid_defs=[],
mode_defs=[
ModeDefinition(
name='mixed',
resource_defs={
'optional_resource': ResourceDefinition(
lambda: None,
config_field=Field(
dagster_type=Dict(
fields={'optional_field': Field(String, is_optional=True)}
)
),
),
'required_resource': ResourceDefinition(
lambda: None,
config_field=Field(
dagster_type=Dict(fields={'required_field': Field(String)})
),
),
},
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields['solids'].is_optional
assert env_type.fields['execution'].is_optional
assert env_type.fields['expectations'].is_optional
assert nested_field(env_type, 'resources').is_required
assert nested_field(env_type, 'resources', 'optional_resource').is_optional
assert nested_field(env_type, 'resources', 'optional_resource', 'config').is_optional
assert nested_field(
env_type, 'resources', 'optional_resource', 'config', 'optional_field'
).is_optional
assert nested_field(env_type, 'resources', 'required_resource').is_required
assert nested_field(env_type, 'resources', 'required_resource', 'config').is_required
assert nested_field(
env_type, 'resources', 'required_resource', 'config', 'required_field'
).is_required
env_obj = EnvironmentConfig.from_dict(
throwing_evaluate_config_value(
env_type, {'resources': {'required_resource': {'config': {'required_field': 'foo'}}}}
)
)
assert env_obj.resources == {
'optional_resource': {'config': {}},
'required_resource': {'config': {'required_field': 'foo'}},
}
def test_provided_default_on_resources_config():
pipeline_def = PipelineDefinition(
mode_definitions=[
ModeDefinition(
name='some_mode',
resources={
'some_resource':
ResourceDefinition(
resource_fn=lambda: None,
config_field=Field(
Dict({
'with_default_int':
Field(Int,
is_optional=True,
default_value=23434)
})),
)
},
)
],
solid_defs=[
SolidDefinition(name='some_solid',
inputs=[],
outputs=[],
compute_fn=lambda *args: None)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.type_attributes.is_system_config
some_resource_field = env_type.fields['resources'].config_type.fields[
'some_resource']
assert some_resource_field.is_optional
some_resource_config_field = some_resource_field.config_type.fields[
'config']
assert some_resource_config_field.is_optional
assert some_resource_config_field.default_value == {
'with_default_int': 23434
}
assert some_resource_field.default_value == {
'config': {
'with_default_int': 23434
}
}
value = EnvironmentConfig.from_dict(
throwing_evaluate_config_value(env_type, {}))
assert value.resources == {
'some_resource': {
'config': {
'with_default_int': 23434
}
}
}
def test_default_environment():
pipeline_def = PipelineDefinition(
solids=[
SolidDefinition(name='some_solid', inputs=[], outputs=[], compute_fn=lambda *args: None)
]
)
env_obj = EnvironmentConfig.from_dict(
throwing_evaluate_config_value(create_environment_type(pipeline_def), {})
)
assert env_obj.expectations.evaluate is True
def create_environment_config(pipeline, environment_dict=None, mode=None):
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
check.opt_dict_param(environment_dict, 'environment')
mode = check.opt_str_param(mode, 'mode', default=pipeline.get_default_mode_name())
environment_type = create_environment_type(pipeline, mode)
result = evaluate_config_value(environment_type, environment_dict)
if not result.success:
raise PipelineConfigEvaluationError(pipeline, result.errors, environment_dict)
return EnvironmentConfig.from_dict(result.value)
def test_default_environment():
@solid(name='some_solid', input_defs=[], output_defs=[])
def some_solid(_):
return None
@pipeline
def pipeline_def():
some_solid()
env_obj = EnvironmentConfig.from_dict(
throwing_evaluate_config_value(create_environment_type(pipeline_def),
{}))
assert env_obj.expectations.evaluate is True
def test_required_solid_with_required_subfield():
pipeline_def = PipelineDefinition(
name='some_pipeline',
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Dict({'required_field': Field(String)})),
inputs=[],
outputs=[],
compute_fn=lambda *_args: None,
)
],
)
env_type = create_environment_type(pipeline_def)
assert env_type.fields['solids'].is_optional is False
assert env_type.fields['solids'].config_type
solids_type = env_type.fields['solids'].config_type
assert solids_type.fields['int_config_solid'].is_optional is False
int_config_solid_type = solids_type.fields['int_config_solid'].config_type
assert int_config_solid_type.fields['config'].is_optional is False
assert env_type.fields['execution'].is_optional
assert env_type.fields['expectations'].is_optional
env_obj = EnvironmentConfig.from_dict(
throwing_evaluate_config_value(
env_type, {
'solids': {
'int_config_solid': {
'config': {
'required_field': 'foobar'
}
}
}
}))
assert env_obj.solids['int_config_solid'].config[
'required_field'] == 'foobar'
with pytest.raises(DagsterEvaluateConfigValueError):
throwing_evaluate_config_value(env_type, {'solids': {}})
with pytest.raises(DagsterEvaluateConfigValueError):
throwing_evaluate_config_value(env_type, {})
def create_environment_config(pipeline,
environment_dict=None,
run_config=None):
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
check.opt_dict_param(environment_dict, 'environment')
run_config = check.opt_inst_param(run_config,
'run_config',
RunConfig,
default=RunConfig())
mode = run_config.mode or pipeline.get_default_mode_name()
environment_type = create_environment_type(pipeline, mode)
result = evaluate_config(environment_type, environment_dict, pipeline,
run_config)
if not result.success:
raise DagsterInvalidConfigError(pipeline, result.errors,
environment_dict)
return EnvironmentConfig.from_dict(result.value)