def test_solid_dictionary_type():
pipeline_def = define_test_solids_config_pipeline()
env_obj = ResolvedRunConfig.build(
pipeline_def,
{
"solids": {
"int_config_solid": {
"config": 1
},
"string_config_solid": {
"config": "bar"
}
},
},
)
value = env_obj.solids
assert set(["int_config_solid",
"string_config_solid"]) == set(value.keys())
assert value == {
"int_config_solid": SolidConfig.from_dict({"config": 1}),
"string_config_solid": SolidConfig.from_dict({"config": "bar"}),
}
def test_solid_dictionary_some_no_config():
@solid(name="int_config_solid",
config_schema=Int,
input_defs=[],
output_defs=[])
def int_config_solid(_):
return None
@solid(name="no_config_solid", input_defs=[], output_defs=[])
def no_config_solid(_):
return None
@pipeline
def pipeline_def():
int_config_solid()
no_config_solid()
env = ResolvedRunConfig.build(
pipeline_def, {"solids": {
"int_config_solid": {
"config": 1
}
}})
assert {"int_config_solid", "no_config_solid"} == set(env.solids.keys())
assert env.solids == {
"int_config_solid": SolidConfig.from_dict({"config": 1}),
"no_config_solid": SolidConfig.from_dict({}),
}
def _composite_descent(parent_stack, solids_config_dict, resource_defs):
"""
The core implementation of composite_descent. This yields a stream of
SolidConfigEntry. This is used by composite_descent to construct a
dictionary.
It descends over the entire solid hierarchy, constructing an entry
for every handle. If it encounters a composite solid instance
with a config mapping, it will invoke that config mapping fn,
producing the config that is necessary to configure the child solids.
This process unrolls recursively as you descend down the tree.
"""
for solid in parent_stack.current_container.solids:
current_stack = parent_stack.descend(solid)
current_handle = current_stack.handle
current_solid_config = solids_config_dict.get(solid.name, {})
# the base case
if isinstance(solid.definition, SolidDefinition):
config_mapped_solid_config = solid.definition.apply_config_mapping(
{"config": current_solid_config.get("config")})
if not config_mapped_solid_config.success:
raise DagsterInvalidConfigError(
"Error in config for solid {}".format(solid.name),
config_mapped_solid_config.errors,
config_mapped_solid_config,
)
complete_config_object = merge_dicts(
current_solid_config, config_mapped_solid_config.value)
yield SolidConfigEntry(
current_handle, SolidConfig.from_dict(complete_config_object))
continue
graph_def = check.inst(solid.definition, GraphDefinition)
yield SolidConfigEntry(
current_handle,
SolidConfig.from_dict({
"inputs":
current_solid_config.get("inputs"),
"outputs":
current_solid_config.get("outputs"),
}),
)
# If there is a config mapping, invoke it and get the descendent solids
# config that way. Else just grabs the solids entry of the current config
solids_dict = (_get_mapped_solids_dict(
solid, graph_def, current_stack, current_solid_config,
resource_defs) if graph_def.config_mapping else
current_solid_config.get("solids", {}))
yield from _composite_descent(current_stack, solids_dict,
resource_defs)
def _composite_descent(parent_stack, solids_config_dict):
'''
The core implementation of composite_descent. This yields a stream of
SolidConfigEntry. This is used by composite_descent to construct a
dictionary.
It descends over the entire solid hierarchy, constructing an entry
for every handle. If it encounters a composite solid instance
with a config mapping, it will invoke that config mapping fn,
producing the config that is necessary to configure the child solids.
This process unrolls recursively as you descend down the tree.
'''
for solid in parent_stack.current_container.solids:
current_stack = parent_stack.descend(solid)
current_handle = current_stack.handle
current_solid_config = solids_config_dict.get(solid.name, {})
# the base case
if not isinstance(solid.definition, CompositeSolidDefinition):
transformed_current_solid_config = solid.definition.apply_config_mapping(
current_solid_config)
if not transformed_current_solid_config.success:
raise DagsterInvalidConfigError(
'Error in config for solid {}'.format(solid.name),
transformed_current_solid_config.errors,
transformed_current_solid_config,
)
yield SolidConfigEntry(
current_handle,
SolidConfig.from_dict(transformed_current_solid_config.value))
continue
composite_def = check.inst(solid.definition, CompositeSolidDefinition)
yield SolidConfigEntry(
current_handle,
SolidConfig.from_dict({
'inputs':
current_solid_config.get('inputs'),
'outputs':
current_solid_config.get('outputs'),
}),
)
# If there is a config mapping, invoke it and get the descendent solids
# config that way. Else just grabs the solids entry of the current config
solids_dict = (_get_mapped_solids_dict(composite_def, current_stack,
current_solid_config)
if composite_def.config_mapping else
current_solid_config.get('solids', {}))
for sce in _composite_descent(current_stack, solids_dict):
yield sce
def test_whole_environment():
pipeline_def = PipelineDefinition(
name="some_pipeline",
mode_defs=[
ModeDefinition(
name="test_mode",
resource_defs={
"test_resource":
ResourceDefinition(resource_fn=lambda _: None,
config_schema=Any)
},
)
],
solid_defs=[
SolidDefinition(
name="int_config_solid",
config_schema=Int,
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None,
),
SolidDefinition(name="no_config_solid",
input_defs=[],
output_defs=[],
compute_fn=lambda *args: None),
],
)
env = EnvironmentConfig.build(
pipeline_def,
{
"resources": {
"test_resource": {
"config": 1
}
},
"solids": {
"int_config_solid": {
"config": 123
}
},
},
)
assert isinstance(env, EnvironmentConfig)
assert env.solids == {
"int_config_solid": SolidConfig.from_dict({"config": 123}),
"no_config_solid": SolidConfig.from_dict({}),
}
assert env.resources == {
"test_resource": {
"config": 1
},
"object_manager": {}
}
