def test_single_layer_pipeline_composite_descent():
@solid(config=int)
def return_int(context):
return context.solid_config
@composite_solid
def return_int_passthrough():
return_int()
@pipeline
def return_int_pipeline_passthrough():
return_int_passthrough()
solid_config_dict = composite_descent(
return_int_pipeline_passthrough,
{'return_int_passthrough': {'solids': {'return_int': {'config': 34}}}},
)
handle = 'return_int_passthrough.return_int'
assert solid_config_dict[handle].config == 34
result = execute_pipeline(
return_int_pipeline_passthrough,
{'solids': {'return_int_passthrough': {'solids': {'return_int': {'config': 34}}}},},
)
assert result.success
assert result.result_for_handle(handle).output_value() == 34
def test_double_nested_input_via_config_mapping():
@lambda_solid
def number(num):
return num
@composite_solid(
config_fn=lambda context, _: {'number': {'inputs': {'num': {'value': 4}}}}, config={}
)
def wrap_solid(): # pylint: disable=unused-variable
return number()
@composite_solid
def double_wrap(num):
number(num)
return wrap_solid()
@pipeline
def wrap_pipeline_double_nested_input():
return double_wrap()
solid_handle_dict = composite_descent(
wrap_pipeline_double_nested_input, {'double_wrap': {'inputs': {'num': {'value': 2}}}}
)
assert solid_handle_dict['double_wrap.wrap_solid.number'].inputs == {'num': {'value': 4}}
assert solid_handle_dict['double_wrap'].inputs == {'num': {'value': 2}}
result = execute_pipeline(
wrap_pipeline_double_nested_input,
{'solids': {'double_wrap': {'inputs': {'num': {'value': 2}}}}},
)
assert result.success
def build(pipeline, environment_dict=None, run_config=None):
from dagster.core.types.config.evaluator.composite_descent import composite_descent
from dagster.core.types.config.evaluator.validate import process_config
check.inst_param(pipeline, 'pipeline', PipelineDefinition)
environment_dict = check.opt_dict_param(environment_dict, 'environment_dict')
run_config = check.opt_inst_param(run_config, 'run_config', IRunConfig, default=RunConfig())
mode = run_config.mode or pipeline.get_default_mode_name()
environment_type = create_environment_type(pipeline, mode)
config_evr = process_config(environment_type, environment_dict)
if not config_evr.success:
raise DagsterInvalidConfigError(
'Error in config for pipeline {}'.format(pipeline.name),
config_evr.errors,
environment_dict,
)
config_value = config_evr.value
solid_config_dict = composite_descent(pipeline, config_value.get('solids', {}), run_config)
return EnvironmentConfig(
solids=solid_config_dict,
execution=ExecutionConfig.from_dict(config_value.get('execution')),
storage=StorageConfig.from_dict(config_value.get('storage')),
loggers=config_value.get('loggers'),
original_config_dict=environment_dict,
resources=config_value.get('resources'),
)
def test_direct_composite_descent_with_error():
@composite_solid(
config={'override_str': Field(int)},
config_fn=lambda _, cfg: {'layer2': {'config': cfg['override_str']}},
)
def wrap_coerce_to_wrong_type():
return scalar_config_solid.alias('layer2')()
@composite_solid(
config={'nesting_override': Field(int)},
config_fn=lambda _, cfg: {'layer1': {'config': {'override_str': cfg['nesting_override']}}},
)
def nesting_wrap_wrong_type_at_leaf():
return wrap_coerce_to_wrong_type.alias('layer1')()
@pipeline
def wrap_pipeline_with_error():
return nesting_wrap_wrong_type_at_leaf.alias('layer0')()
with pytest.raises(DagsterInvalidConfigError) as exc_info:
composite_descent(
wrap_pipeline_with_error, {'layer0': {'config': {'nesting_override': 214}}}
)
assert 'In pipeline wrap_pipeline_with_error at stack layer0:layer1:' in str(exc_info.value)
assert (
'Solid "layer1" with definition "wrap_coerce_to_wrong_type" has a configuration error.'
in str(exc_info.value)
)
assert (
'''Error 1: Type failure at path "root:layer2:config" on type "String". '''
'''Value at path root:layer2:config is not valid. Expected "String".'''
in str(exc_info.value)
)
def test_single_layer_pipeline_hardcoded_config_mapping():
@solid(config=int)
def return_int(context):
return context.solid_config
@composite_solid(config={}, config_fn=lambda _ctx, _cfg: {'return_int': {'config': 35}})
def return_int_hardcode_wrap():
return_int()
@pipeline
def return_int_hardcode_wrap_pipeline():
return_int_hardcode_wrap()
solid_config_dict = composite_descent(return_int_hardcode_wrap_pipeline, {})
assert solid_config_dict['return_int_hardcode_wrap.return_int'].config == 35
def test_single_solid_pipeline_composite_descent():
@solid(config=int)
def return_int(context):
return context.solid_config
@pipeline
def return_int_pipeline():
return_int()
solid_config_dict = composite_descent(return_int_pipeline, {'return_int': {'config': 3}})
assert solid_config_dict['return_int'].config == 3
result = execute_pipeline(return_int_pipeline, {'solids': {'return_int': {'config': 3}}})
assert result.success
assert result.result_for_solid('return_int').output_value() == 3
def test_nested_input_via_config_mapping():
@solid
def add_one(_, num):
return num + 1
@composite_solid(
config={}, config_fn=lambda _cxt, _cfg: {'add_one': {'inputs': {'num': {'value': 2}}}}
)
def wrap_add_one():
add_one()
@pipeline
def wrap_add_one_pipeline():
wrap_add_one()
solid_config_dict = composite_descent(wrap_add_one_pipeline, {})
assert solid_config_dict['wrap_add_one.add_one'].inputs == {'num': {'value': 2}}
result = execute_pipeline(wrap_add_one_pipeline)
assert result.success
assert result.result_for_handle('wrap_add_one.add_one').output_value() == 3
def test_single_layer_pipeline_computed_config_mapping():
@solid(config=int)
def return_int(context):
return context.solid_config
def _config_fn(_, cfg):
return {'return_int': {'config': cfg['number'] + 1}}
@composite_solid(config={'number': int}, config_fn=_config_fn)
def return_int_plus_one():
return_int()
@pipeline
def return_int_hardcode_wrap_pipeline():
return_int_plus_one()
solid_config_dict = composite_descent(
return_int_hardcode_wrap_pipeline, {'return_int_plus_one': {'config': {'number': 23}}}
)
assert solid_config_dict['return_int_plus_one.return_int'].config == 24
def test_mix_layer_computed_mapping():
@solid(config=int)
def return_int(context):
return context.solid_config
@composite_solid(
config={'number': int},
config_fn=lambda _, cfg: {'return_int': {'config': cfg['number'] + 1}},
)
def layer_three_wrap():
return_int()
def _layer_two_double_wrap_cfg_fn(_, cfg):
if cfg['inject_error']:
return {'layer_three_wrap': {'config': {'number': 'a_string'}}}
else:
return {'layer_three_wrap': {'config': {'number': cfg['number'] + 1}}}
@composite_solid(
config={'number': int, 'inject_error': bool}, config_fn=_layer_two_double_wrap_cfg_fn
)
def layer_two_double_wrap():
layer_three_wrap()
@composite_solid
def layer_two_passthrough():
return_int()
@composite_solid
def layer_one():
layer_two_passthrough()
layer_two_double_wrap()
@pipeline
def layered_config():
layer_one()
solid_config_dict = composite_descent(
layered_config,
{
'layer_one': {
'solids': {
'layer_two_passthrough': {'solids': {'return_int': {'config': 234}}},
'layer_two_double_wrap': {'config': {'number': 5, 'inject_error': False}},
}
}
},
)
assert solid_config_dict['layer_one.layer_two_passthrough.return_int'].config == 234
# this passed through both config fns which each added one
assert (
solid_config_dict['layer_one.layer_two_double_wrap.layer_three_wrap.return_int'].config == 7
)
with pytest.raises(DagsterInvalidConfigError) as exc_info:
composite_descent(
layered_config,
{
'layer_one': {
'solids': {
'layer_two_passthrough': {'solids': {'return_int': {'config': 234}}},
'layer_two_double_wrap': {'config': {'number': 234, 'inject_error': True}},
}
}
},
)
assert 'Solid "layer_two_double_wrap" with definition "layer_two_double_wrap"' in str(
exc_info.value
)
assert (
'Error 1: Type failure at path "root:layer_three_wrap:config:number" on type "Int". '
'Value at path root:layer_three_wrap:config:number is not valid. Expected "Int"'
) in str(exc_info.value)
result = execute_pipeline(
layered_config,
{
'solids': {
'layer_one': {
'solids': {
'layer_two_passthrough': {'solids': {'return_int': {'config': 55}}},
'layer_two_double_wrap': {'config': {'number': 7, 'inject_error': False}},
}
}
},
},
)
assert (
result.result_for_handle('layer_one.layer_two_passthrough.return_int').output_value() == 55
)
assert (
result.result_for_handle(
'layer_one.layer_two_double_wrap.layer_three_wrap.return_int'
).output_value()
== 9
)
