def test_direct_composite_descent_with_error():
@composite_solid(
config={'override_str': Field(int)},
config_fn=lambda cfg: {'layer2': {'config': cfg['override_str']}},
)
def wrap_coerce_to_wrong_type():
return scalar_config_solid.alias('layer2')()
@composite_solid(
config={'nesting_override': Field(int)},
config_fn=lambda cfg: {'layer1': {'config': {'override_str': cfg['nesting_override']}}},
)
def nesting_wrap_wrong_type_at_leaf():
return wrap_coerce_to_wrong_type.alias('layer1')()
@pipeline
def wrap_pipeline_with_error():
return nesting_wrap_wrong_type_at_leaf.alias('layer0')()
with pytest.raises(DagsterInvalidConfigError) as exc_info:
composite_descent(
wrap_pipeline_with_error, {'layer0': {'config': {'nesting_override': 214}}}
)
assert 'In pipeline wrap_pipeline_with_error at stack layer0:layer1:' in str(exc_info.value)
assert (
'Solid "layer1" with definition "wrap_coerce_to_wrong_type" has a configuration error.'
in str(exc_info.value)
)
print(str(exc_info.value))
assert 'Error 1: Invalid scalar at path root:layer2:config value "214"' in str(exc_info.value)
def test_direct_composite_descent_with_error():
@composite_solid(
config_schema={"override_str": Field(int)},
config_fn=lambda cfg: {"layer2": {"config": cfg["override_str"]}},
)
def wrap_coerce_to_wrong_type():
return scalar_config_solid.alias("layer2")()
@composite_solid(
config_schema={"nesting_override": Field(int)},
config_fn=lambda cfg: {"layer1": {"config": {"override_str": cfg["nesting_override"]}}},
)
def nesting_wrap_wrong_type_at_leaf():
return wrap_coerce_to_wrong_type.alias("layer1")()
@pipeline
def wrap_pipeline_with_error():
nesting_wrap_wrong_type_at_leaf.alias("layer0")()
with pytest.raises(DagsterInvalidConfigError) as exc_info:
composite_descent(
wrap_pipeline_with_error, {"layer0": {"config": {"nesting_override": 214}}}
)
assert "In pipeline wrap_pipeline_with_error at stack layer0:layer1:" in str(exc_info.value)
assert (
'Solid "layer1" with definition "wrap_coerce_to_wrong_type" has a configuration error.'
in str(exc_info.value)
)
print(str(exc_info.value))
assert 'Error 1: Invalid scalar at path root:layer2:config. Value "214"' in str(exc_info.value)
def test_single_solid_pipeline_composite_descent():
@solid(config_schema=int)
def return_int(context):
return context.solid_config
@pipeline
def return_int_pipeline():
return_int()
solid_config_dict = composite_descent(return_int_pipeline,
{"return_int": {
"config": 3
}},
resource_defs={})
assert solid_config_dict["return_int"].config == 3
result = execute_pipeline(return_int_pipeline,
{"solids": {
"return_int": {
"config": 3
}
}})
assert result.success
assert result.result_for_solid("return_int").output_value() == 3
def test_nested_input_via_config_mapping():
@solid
def add_one(_, num):
return num + 1
@composite_solid(
config_schema={},
config_fn=lambda _cfg: {"add_one": {
"inputs": {
"num": {
"value": 2
}
}
}})
def wrap_add_one():
add_one()
@pipeline
def wrap_add_one_pipeline():
wrap_add_one()
solid_config_dict = composite_descent(wrap_add_one_pipeline, {},
resource_defs={})
assert solid_config_dict["wrap_add_one.add_one"].inputs == {
"num": {
"value": 2
}
}
result = execute_pipeline(wrap_add_one_pipeline)
assert result.success
assert result.result_for_handle("wrap_add_one.add_one").output_value() == 3
def test_single_layer_pipeline_computed_config_mapping():
@solid(config_schema=int)
def return_int(context):
return context.solid_config
def _config_fn(cfg):
return {"return_int": {"config": cfg["number"] + 1}}
@composite_solid(config_schema={"number": int}, config_fn=_config_fn)
def return_int_plus_one():
return_int()
@pipeline
def return_int_hardcode_wrap_pipeline():
return_int_plus_one()
solid_config_dict = composite_descent(
return_int_hardcode_wrap_pipeline,
{"return_int_plus_one": {
"config": {
"number": 23
}
}},
resource_defs={},
)
assert solid_config_dict["return_int_plus_one.return_int"].config == 24
def test_single_layer_pipeline_composite_descent():
@solid(config=int)
def return_int(context):
return context.solid_config
@composite_solid
def return_int_passthrough():
return_int()
@pipeline
def return_int_pipeline_passthrough():
return_int_passthrough()
solid_config_dict = composite_descent(
return_int_pipeline_passthrough,
{'return_int_passthrough': {'solids': {'return_int': {'config': 34}}}},
)
handle = 'return_int_passthrough.return_int'
assert solid_config_dict[handle].config == 34
result = execute_pipeline(
return_int_pipeline_passthrough,
{'solids': {'return_int_passthrough': {'solids': {'return_int': {'config': 34}}}},},
)
assert result.success
assert result.result_for_handle(handle).output_value() == 34
def test_double_nested_input_via_config_mapping():
@lambda_solid
def number(num):
return num
@composite_solid(config_fn=lambda _: {'number': {'inputs': {'num': {'value': 4}}}}, config={})
def wrap_solid(): # pylint: disable=unused-variable
return number()
@composite_solid
def double_wrap(num):
number(num)
return wrap_solid()
@pipeline
def wrap_pipeline_double_nested_input():
return double_wrap()
solid_handle_dict = composite_descent(
wrap_pipeline_double_nested_input, {'double_wrap': {'inputs': {'num': {'value': 2}}}}
)
assert solid_handle_dict['double_wrap.wrap_solid.number'].inputs == {'num': {'value': 4}}
assert solid_handle_dict['double_wrap'].inputs == {'num': {'value': 2}}
result = execute_pipeline(
wrap_pipeline_double_nested_input,
{'solids': {'double_wrap': {'inputs': {'num': {'value': 2}}}}},
)
assert result.success
def test_single_solid_pipeline_composite_descent():
@solid(config=int)
def return_int(context):
return context.solid_config
@pipeline
def return_int_pipeline():
return_int()
solid_config_dict = composite_descent(return_int_pipeline,
{'return_int': {
'config': 3
}})
assert solid_config_dict['return_int'].config == 3
result = execute_pipeline(return_int_pipeline,
{'solids': {
'return_int': {
'config': 3
}
}})
assert result.success
assert result.result_for_solid('return_int').output_value() == 3
def test_nested_input_via_config_mapping():
@solid
def add_one(_, num):
return num + 1
@composite_solid(
config={},
config_fn=lambda _cfg: {'add_one': {
'inputs': {
'num': {
'value': 2
}
}
}})
def wrap_add_one():
add_one()
@pipeline
def wrap_add_one_pipeline():
wrap_add_one()
solid_config_dict = composite_descent(wrap_add_one_pipeline, {})
assert solid_config_dict['wrap_add_one.add_one'].inputs == {
'num': {
'value': 2
}
}
result = execute_pipeline(wrap_add_one_pipeline)
assert result.success
assert result.result_for_handle('wrap_add_one.add_one').output_value() == 3
def test_single_layer_pipeline_computed_config_mapping():
@solid(config=int)
def return_int(context):
return context.solid_config
def _config_fn(cfg):
return {'return_int': {'config': cfg['number'] + 1}}
@composite_solid(config={'number': int}, config_fn=_config_fn)
def return_int_plus_one():
return_int()
@pipeline
def return_int_hardcode_wrap_pipeline():
return_int_plus_one()
solid_config_dict = composite_descent(
return_int_hardcode_wrap_pipeline,
{'return_int_plus_one': {
'config': {
'number': 23
}
}})
assert solid_config_dict['return_int_plus_one.return_int'].config == 24
def test_double_nested_input_via_config_mapping():
@lambda_solid
def number(num):
return num
@composite_solid(
config_fn=lambda _: {"number": {"inputs": {"num": {"value": 4}}}}, config_schema={}
)
def wrap_solid(): # pylint: disable=unused-variable
return number()
@composite_solid
def double_wrap(num):
number(num)
return wrap_solid()
@pipeline
def wrap_pipeline_double_nested_input():
double_wrap()
solid_handle_dict = composite_descent(
wrap_pipeline_double_nested_input, {"double_wrap": {"inputs": {"num": {"value": 2}}}}
)
assert solid_handle_dict["double_wrap.wrap_solid.number"].inputs == {"num": {"value": 4}}
assert solid_handle_dict["double_wrap"].inputs == {"num": {"value": 2}}
result = execute_pipeline(
wrap_pipeline_double_nested_input,
{"solids": {"double_wrap": {"inputs": {"num": {"value": 2}}}}},
)
assert result.success
def test_single_layer_pipeline_composite_descent():
@solid(config_schema=int)
def return_int(context):
return context.solid_config
@composite_solid
def return_int_passthrough():
return_int()
@pipeline
def return_int_pipeline_passthrough():
return_int_passthrough()
solid_config_dict = composite_descent(
return_int_pipeline_passthrough,
{"return_int_passthrough": {"solids": {"return_int": {"config": 34}}}},
resource_defs={"io_manager": mem_io_manager},
)
handle = "return_int_passthrough.return_int"
assert solid_config_dict[handle].config == 34
result = execute_pipeline(
return_int_pipeline_passthrough,
{
"solids": {"return_int_passthrough": {"solids": {"return_int": {"config": 34}}}},
},
)
assert result.success
assert result.result_for_handle(handle).output_value() == 34
def test_single_layer_pipeline_hardcoded_config_mapping():
@solid(config=int)
def return_int(context):
return context.solid_config
@composite_solid(config={}, config_fn=lambda _cfg: {'return_int': {'config': 35}})
def return_int_hardcode_wrap():
return_int()
@pipeline
def return_int_hardcode_wrap_pipeline():
return_int_hardcode_wrap()
solid_config_dict = composite_descent(return_int_hardcode_wrap_pipeline, {})
assert solid_config_dict['return_int_hardcode_wrap.return_int'].config == 35
def test_single_layer_pipeline_hardcoded_config_mapping():
@solid(config_schema=int)
def return_int(context):
return context.solid_config
@composite_solid(config_schema={}, config_fn=lambda _cfg: {"return_int": {"config": 35}})
def return_int_hardcode_wrap():
return_int()
@pipeline
def return_int_hardcode_wrap_pipeline():
return_int_hardcode_wrap()
solid_config_dict = composite_descent(
return_int_hardcode_wrap_pipeline, {}, resource_defs={"io_manager": mem_io_manager}
)
assert solid_config_dict["return_int_hardcode_wrap.return_int"].config == 35
def test_mix_layer_computed_mapping():
@solid(config_schema=int)
def return_int(context):
return context.solid_config
@composite_solid(
config_schema={"number": int},
config_fn=lambda cfg: {"return_int": {
"config": cfg["number"] + 1
}},
)
def layer_three_wrap():
return_int()
def _layer_two_double_wrap_cfg_fn(cfg):
if cfg["inject_error"]:
return {"layer_three_wrap": {"config": {"number": "a_string"}}}
else:
return {
"layer_three_wrap": {
"config": {
"number": cfg["number"] + 1
}
}
}
@composite_solid(config_schema={
"number": int,
"inject_error": bool
},
config_fn=_layer_two_double_wrap_cfg_fn)
def layer_two_double_wrap():
layer_three_wrap()
@composite_solid
def layer_two_passthrough():
return_int()
@composite_solid
def layer_one():
layer_two_passthrough()
layer_two_double_wrap()
@pipeline
def layered_config():
layer_one()
solid_config_dict = composite_descent(
layered_config,
{
"layer_one": {
"solids": {
"layer_two_passthrough": {
"solids": {
"return_int": {
"config": 234
}
}
},
"layer_two_double_wrap": {
"config": {
"number": 5,
"inject_error": False
}
},
}
}
},
resource_defs={},
)
assert solid_config_dict[
"layer_one.layer_two_passthrough.return_int"].config == 234
# this passed through both config fns which each added one
assert (solid_config_dict[
"layer_one.layer_two_double_wrap.layer_three_wrap.return_int"].config
== 7)
with pytest.raises(DagsterInvalidConfigError) as exc_info:
composite_descent(
layered_config,
{
"layer_one": {
"solids": {
"layer_two_passthrough": {
"solids": {
"return_int": {
"config": 234
}
}
},
"layer_two_double_wrap": {
"config": {
"number": 234,
"inject_error": True
}
},
}
}
},
resource_defs={},
)
assert 'Solid "layer_two_double_wrap" with definition "layer_two_double_wrap"' in str(
exc_info.value)
print(str(exc_info.value))
assert (
'Error 1: Invalid scalar at path root:layer_three_wrap:config:number. Value "a_string"'
) in str(exc_info.value)
result = execute_pipeline(
layered_config,
{
"solids": {
"layer_one": {
"solids": {
"layer_two_passthrough": {
"solids": {
"return_int": {
"config": 55
}
}
},
"layer_two_double_wrap": {
"config": {
"number": 7,
"inject_error": False
}
},
}
}
},
},
)
assert (result.result_for_handle(
"layer_one.layer_two_passthrough.return_int").output_value() == 55)
assert (result.result_for_handle(
"layer_one.layer_two_double_wrap.layer_three_wrap.return_int").
output_value() == 9)
def test_mix_layer_computed_mapping():
@solid(config=int)
def return_int(context):
return context.solid_config
@composite_solid(
config={'number': int}, config_fn=lambda cfg: {'return_int': {'config': cfg['number'] + 1}},
)
def layer_three_wrap():
return_int()
def _layer_two_double_wrap_cfg_fn(cfg):
if cfg['inject_error']:
return {'layer_three_wrap': {'config': {'number': 'a_string'}}}
else:
return {'layer_three_wrap': {'config': {'number': cfg['number'] + 1}}}
@composite_solid(
config={'number': int, 'inject_error': bool}, config_fn=_layer_two_double_wrap_cfg_fn
)
def layer_two_double_wrap():
layer_three_wrap()
@composite_solid
def layer_two_passthrough():
return_int()
@composite_solid
def layer_one():
layer_two_passthrough()
layer_two_double_wrap()
@pipeline
def layered_config():
layer_one()
solid_config_dict = composite_descent(
layered_config,
{
'layer_one': {
'solids': {
'layer_two_passthrough': {'solids': {'return_int': {'config': 234}}},
'layer_two_double_wrap': {'config': {'number': 5, 'inject_error': False}},
}
}
},
)
assert solid_config_dict['layer_one.layer_two_passthrough.return_int'].config == 234
# this passed through both config fns which each added one
assert (
solid_config_dict['layer_one.layer_two_double_wrap.layer_three_wrap.return_int'].config == 7
)
with pytest.raises(DagsterInvalidConfigError) as exc_info:
composite_descent(
layered_config,
{
'layer_one': {
'solids': {
'layer_two_passthrough': {'solids': {'return_int': {'config': 234}}},
'layer_two_double_wrap': {'config': {'number': 234, 'inject_error': True}},
}
}
},
)
assert 'Solid "layer_two_double_wrap" with definition "layer_two_double_wrap"' in str(
exc_info.value
)
assert (
'Error 1: Type failure at path "root:layer_three_wrap:config:number" on type "Int". '
'Value at path root:layer_three_wrap:config:number is not valid. Expected "Int"'
) in str(exc_info.value)
result = execute_pipeline(
layered_config,
{
'solids': {
'layer_one': {
'solids': {
'layer_two_passthrough': {'solids': {'return_int': {'config': 55}}},
'layer_two_double_wrap': {'config': {'number': 7, 'inject_error': False}},
}
}
},
},
)
assert (
result.result_for_handle('layer_one.layer_two_passthrough.return_int').output_value() == 55
)
assert (
result.result_for_handle(
'layer_one.layer_two_double_wrap.layer_three_wrap.return_int'
).output_value()
== 9
)