def _get_mapped_solids_dict(composite_def, current_stack,
current_solid_config):
# the spec of the config mapping function is that it takes the dictionary at:
# solid_name:
# config: {dict_passed_to_user}
# and it returns the dictionary rooted at solids
# solid_name:
# solids: {return_value_of_config_fn}
# We must call the config mapping function and then validate it against
# the child schema.
with user_code_error_boundary(DagsterConfigMappingFunctionError,
_get_error_lambda(current_stack)):
mapped_solids_config = composite_def.config_mapping.config_fn(
current_solid_config.get('config', {}))
# Dynamically construct the type that the output of the config mapping function will
# be evaluated against
type_to_evaluate_against = define_solid_dictionary_cls(
composite_def.solids, composite_def.dependency_structure,
current_stack.handle)
# process against that new type
evr = process_config(type_to_evaluate_against, mapped_solids_config)
if not evr.success:
raise_composite_descent_config_error(current_stack,
mapped_solids_config, evr)
return evr.value
def test_solid_dictionary_some_no_config():
pipeline_def = PipelineDefinition(solids=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
inputs=[],
outputs=[],
transform_fn=lambda *args: None,
),
SolidDefinition(name='no_config_solid',
inputs=[],
outputs=[],
transform_fn=lambda *args: None),
])
solid_dict_type = define_solid_dictionary_cls(
'foobar', create_creation_data(pipeline_def)).inst()
value = construct_solid_dictionary(
throwing_evaluate_config_value(solid_dict_type,
{'int_config_solid': {
'config': 1
}}))
assert set(['int_config_solid']) == set(value.keys())
assert value == {'int_config_solid': SolidConfig(1)}
def test_solid_dictionary_some_no_config():
pipeline_def = PipelineDefinition(
solid_defs=[
SolidDefinition(
name='int_config_solid',
config_field=Field(Int),
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(
name='no_config_solid', input_defs=[], output_defs=[], compute_fn=lambda *args: None
),
]
)
solid_dict_type = define_solid_dictionary_cls(
'foobar', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name
).inst()
value = construct_solid_dictionary(
throwing_evaluate_config_value(solid_dict_type, {'int_config_solid': {'config': 1}})
)
assert set(['int_config_solid']) == set(value.keys())
assert value == {'int_config_solid': SolidConfig(1)}
def test_solid_dictionary_type():
pipeline_def = define_test_solids_config_pipeline()
solid_dict_type = define_solid_dictionary_cls(
'foobar', pipeline_def.solids, pipeline_def.dependency_structure,
pipeline_def.name).inst()
value = construct_solid_dictionary(
throwing_evaluate_config_value(
solid_dict_type,
{
'int_config_solid': {
'config': 1
},
'string_config_solid': {
'config': 'bar'
}
},
))
assert set(['int_config_solid',
'string_config_solid']) == set(value.keys())
assert value == {
'int_config_solid': SolidConfig(1),
'string_config_solid': SolidConfig('bar')
}
assert solid_dict_type.type_attributes.is_system_config
for specific_solid_config_field in solid_dict_type.fields.values():
specific_solid_config_type = specific_solid_config_field.config_type
assert specific_solid_config_type.type_attributes.is_system_config
user_config_field = specific_solid_config_field.config_type.fields[
'config']
assert user_config_field.config_type.type_attributes.is_system_config is False
def test_solid_dictionary_some_no_config():
@solid(name='int_config_solid',
config_field=Field(Int),
input_defs=[],
output_defs=[])
def int_config_solid(_):
return None
@solid(name='no_config_solid', input_defs=[], output_defs=[])
def no_config_solid(_):
return None
@pipeline
def pipeline_def():
int_config_solid()
no_config_solid()
solid_dict_type = define_solid_dictionary_cls(
'foobar', pipeline_def.solids, pipeline_def.dependency_structure,
pipeline_def.name).inst()
value = construct_solid_dictionary(
throwing_evaluate_config_value(solid_dict_type,
{'int_config_solid': {
'config': 1
}}))
assert set(['int_config_solid']) == set(value.keys())
assert value == {'int_config_solid': SolidConfig(1)}
def test_solid_config_error():
solid_dict_type = define_solid_dictionary_cls(
'slkdfjkjdsf', create_creation_data(define_test_solids_config_pipeline())
).inst()
int_solid_config_type = solid_dict_type.fields['int_config_solid'].config_type
with pytest.raises(DagsterEvaluateConfigValueError, match='Field "notconfig" is not defined'):
throwing_evaluate_config_value(int_solid_config_type, {'notconfig': 1})
with pytest.raises(DagsterEvaluateConfigValueError):
throwing_evaluate_config_value(int_solid_config_type, 1)
def test_solid_config_error():
pipeline_def = define_test_solids_config_pipeline()
solid_dict_type = define_solid_dictionary_cls(
'slkdfjkjdsf', pipeline_def.solids, pipeline_def.dependency_structure, pipeline_def.name
).inst()
int_solid_config_type = solid_dict_type.fields['int_config_solid'].config_type
with pytest.raises(DagsterEvaluateConfigValueError, match='Field "notconfig" is not defined'):
throwing_evaluate_config_value(int_solid_config_type, {'notconfig': 1})
with pytest.raises(DagsterEvaluateConfigValueError):
throwing_evaluate_config_value(int_solid_config_type, 1)
def test_solid_config_error():
pipeline_def = define_test_solids_config_pipeline()
solid_dict_type = define_solid_dictionary_cls(
pipeline_def.solids, pipeline_def.dependency_structure,
)
int_solid_config_type = solid_dict_type.fields['int_config_solid'].config_type
with pytest.raises(DagsterEvaluateConfigValueError, match='Undefined field "notconfig"'):
throwing_validate_config_value(int_solid_config_type, {'notconfig': 1})
with pytest.raises(DagsterEvaluateConfigValueError):
throwing_validate_config_value(int_solid_config_type, 1)
def test_solid_config_error():
pipeline_def = define_test_solids_config_pipeline()
solid_dict_type = define_solid_dictionary_cls(
pipeline_def.solids, pipeline_def.dependency_structure,
)
int_solid_config_type = solid_dict_type.fields['int_config_solid'].config_type
res = process_config(int_solid_config_type, {'notconfig': 1})
assert not res.success
assert re.match('Undefined field "notconfig"', res.errors[0].message)
res = process_config(int_solid_config_type, 1)
assert not res.success
def _get_mapped_solids_dict(composite, graph_def, current_stack,
current_solid_config, resource_defs):
# the spec of the config mapping function is that it takes the dictionary at:
# solid_name:
# config: {dict_passed_to_user}
# and it returns the dictionary rooted at solids
# solid_name:
# solids: {return_value_of_config_fn}
# We must call the config mapping function and then validate it against
# the child schema.
# apply @configured config mapping to the composite's incoming config before we get to the
# composite's own config mapping process
config_mapped_solid_config = graph_def.apply_config_mapping(
current_solid_config)
if not config_mapped_solid_config.success:
raise DagsterInvalidConfigError(
"Error in config for composite solid {}".format(composite.name),
config_mapped_solid_config.errors,
config_mapped_solid_config,
)
with user_code_error_boundary(DagsterConfigMappingFunctionError,
_get_error_lambda(current_stack)):
mapped_solids_config = graph_def.config_mapping.config_fn(
config_mapped_solid_config.value.get("config", {}))
# Dynamically construct the type that the output of the config mapping function will
# be evaluated against
type_to_evaluate_against = define_solid_dictionary_cls(
solids=graph_def.solids,
ignored_solids=None,
dependency_structure=graph_def.dependency_structure,
parent_handle=current_stack.handle,
resource_defs=resource_defs,
)
# process against that new type
evr = process_config(type_to_evaluate_against, mapped_solids_config)
if not evr.success:
raise_composite_descent_config_error(current_stack,
mapped_solids_config, evr)
return evr.value
def test_solid_config_error():
pipeline_def = define_test_solids_config_pipeline()
solid_dict_type = define_solid_dictionary_cls(
solids=pipeline_def.solids,
ignored_solids=None,
dependency_structure=pipeline_def.dependency_structure,
parent_handle=None,
resource_defs={},
)
int_solid_config_type = solid_dict_type.fields["int_config_solid"].config_type
res = process_config(int_solid_config_type, {"notconfig": 1})
assert not res.success
assert re.match('Received unexpected config entry "notconfig"', res.errors[0].message)
res = process_config(int_solid_config_type, 1)
assert not res.success
def test_solid_dictionary_type():
pipeline_def = define_test_solids_config_pipeline()
solid_dict_type = define_solid_dictionary_cls(
pipeline_def.solids, pipeline_def.dependency_structure)
env_obj = EnvironmentConfig.build(
pipeline_def,
{
'solids': {
'int_config_solid': {
'config': 1
},
'string_config_solid': {
'config': 'bar'
}
},
},
)
value = env_obj.solids
assert set(['int_config_solid',
'string_config_solid']) == set(value.keys())
assert value == {
'int_config_solid': SolidConfig(1),
'string_config_solid': SolidConfig('bar')
}
assert solid_dict_type.type_attributes.is_system_config
for specific_solid_config_field in solid_dict_type.fields.values():
specific_solid_config_type = specific_solid_config_field.config_type
assert specific_solid_config_type.type_attributes.is_system_config
user_config_field = specific_solid_config_field.config_type.fields[
'config']
assert user_config_field.config_type.type_attributes.is_system_config is False
