def test_nested_graph_config_mapping():
@op(config_schema=str)
def my_op(context):
return context.op_config
def _nested_config_fn(outer):
return {"my_op": {"config": outer}}
@graph(config=ConfigMapping(config_fn=_nested_config_fn,
config_schema=str))
def my_nested_graph():
my_op()
def _config_fn(outer):
return {"my_nested_graph": {"config": outer}}
@graph(config=ConfigMapping(config_fn=_config_fn, config_schema=str))
def my_graph():
my_nested_graph()
result = my_graph.to_job().execute_in_process(
run_config={"ops": {
"config": "foo"
}})
assert result.success
assert result.output_for_node("my_nested_graph.my_op") == "foo"
def test_graph_configured_error_in_fn():
@op(config_schema=str)
def my_op(context):
return context.op_config
def _config_fn(outer):
return {"my_op": {"config": outer}}
@graph(config=ConfigMapping(config_fn=_config_fn, config_schema=str))
def my_graph():
my_op()
def _bad_config_fn(_):
raise Exception("Uh oh")
configured_graph = my_graph.configured(name="blah",
config_or_config_fn=_bad_config_fn)
with pytest.raises(
DagsterConfigMappingFunctionError,
match=
"The config mapping function on a `configured` GraphDefinition has thrown an "
"unexpected error during its execution.",
):
configured_graph.execute_in_process()
def test_op_selection_with_config_mapping():
def my_config_fn(val):
config_val = {"config": {"foo": val["foo"]}}
return {
"ops": {
"my_op": config_val,
"my_other_op": config_val,
}
}
@op
def my_op(context):
return context.op_config["foo"]
@graph
def my_graph():
my_op()
my_op.alias("my_other_op")()
my_job = my_graph.to_job(config=ConfigMapping(my_config_fn))
result = my_job.execute_in_process(run_config={"foo": "bar"})
assert result.success
assert result.output_for_node("my_op") == "bar"
assert result.output_for_node("my_other_op") == "bar"
subset_result = my_job.execute_in_process(run_config={"foo": "bar"},
op_selection=["my_other_op"])
assert subset_result.success
assert subset_result.output_for_node("my_other_op") == "bar"
def test_graph_with_configured():
@op(config_schema=str)
def my_op(context):
return context.op_config
def _config_fn(outer):
return {"my_op": {"config": outer}}
@graph(config=ConfigMapping(config_fn=_config_fn, config_schema=str))
def my_graph():
my_op()
result = my_graph.configured(
name="my_graph", config_or_config_fn="foo").execute_in_process()
assert result.success
assert result.output_for_node("my_op") == "foo"
def _configured_use_fn(outer):
return outer
result = (my_graph.configured(
name="my_graph",
config_or_config_fn=_configured_use_fn,
config_schema=str).to_job().execute_in_process(
run_config={"ops": {
"config": "foo"
}}))
assert result.success
assert result.output_for_node("my_op") == "foo"
def test_config_mapping_fn():
@resource(config_schema=str)
def date(context) -> str:
return context.resource_config
@op(
required_resource_keys={"date"},
config_schema={"msg": str},
)
def do_stuff(context):
return f"{context.op_config['msg'] } on {context.resources.date}"
@graph
def needs_config():
do_stuff()
def _mapped(val):
return {
"ops": {"do_stuff": {"config": {"msg": "i am here"}}},
"resources": {"date": {"config": val["date"]}},
}
job = needs_config.to_job(
resource_defs={"date": date},
config=ConfigMapping(
config_schema={"date": str}, # top level has to be dict
config_fn=_mapped,
),
)
result = job.execute_in_process(run_config={"date": "6/4"})
assert result.success
assert result.output_for_node("do_stuff") == "i am here on 6/4"
def test_top_level_config_mapping_graph():
@op(config_schema=str)
def my_op(context):
return context.op_config
def _config_fn(_):
return {"my_op": {"config": "foo"}}
@graph(config=ConfigMapping(config_fn=_config_fn))
def my_graph():
my_op()
result = my_graph.execute_in_process()
assert result.success
assert result.output_for_node("my_op") == "foo"
def test_top_level_graph_outer_config_failure():
@op(config_schema=str)
def my_op(context):
return context.op_config
def _config_fn(outer):
return {"my_op": {"config": outer}}
@graph(config=ConfigMapping(config_fn=_config_fn, config_schema=str))
def my_graph():
my_op()
with pytest.raises(DagsterInvalidConfigError, match="Invalid scalar at path root:ops:config"):
my_graph.to_job().execute_in_process(run_config={"ops": {"config": {"bad_type": "foo"}}})
with pytest.raises(DagsterInvalidConfigError, match="Invalid scalar at path root:config"):
my_graph.to_job(config={"ops": {"config": {"bad_type": "foo"}}})
def test_top_level_graph_config_mapping_failure():
@op(config_schema=str)
def my_op(context):
return context.op_config
def _nested_config_fn(_):
return "foo"
@graph(config=ConfigMapping(config_fn=_nested_config_fn))
def my_nested_graph():
my_op()
with pytest.raises(
DagsterInvalidConfigError,
match="In pipeline 'my_nested_graph', top level graph 'my_nested_graph' has a configuration error.",
):
my_nested_graph.execute_in_process()
def test_job_config_mapping():
@op(config_schema=str)
def my_op(context):
return context.op_config
def _config_fn(outer):
return {"ops": {"my_op": {"config": outer["foo_schema"]}}}
config_mapping = ConfigMapping(config_fn=_config_fn,
config_schema={"foo_schema": str})
@job(config=config_mapping)
def my_job():
my_op()
result = my_job.execute_in_process(run_config={"foo_schema": "foo"})
assert result.success
assert result.output_for_node("my_op") == "foo"
def test_graph_configured_error_in_config():
@op(config_schema=str)
def my_op(context):
return context.op_config
def _config_fn(outer):
return {"my_op": {"config": outer}}
@graph(config=ConfigMapping(config_fn=_config_fn, config_schema=str))
def my_graph():
my_op()
def _bad_config_fn(_):
return 2
configured_graph = my_graph.configured(name="blah", config_or_config_fn=_bad_config_fn)
with pytest.raises(DagsterInvalidConfigError, match="Error in config for graph blah"):
configured_graph.execute_in_process()
def test_enum_config_mapping():
@op(
config_schema={
"my_enum":
Field(Enum.from_python_enum(TestEnum),
is_required=False,
default_value="ONE")
})
def my_op(context):
return context.op_config["my_enum"]
@graph
def my_graph():
my_op()
def _use_defaults_mapping(_):
return {}
use_defaults = my_graph.to_job(config=ConfigMapping(
config_fn=_use_defaults_mapping))
result = use_defaults.execute_in_process()
assert result.success
assert result.output_for_node("my_op") == TestEnum.ONE
def _override_defaults_mapping(_):
return {"ops": {"my_op": {"config": {"my_enum": "TWO"}}}}
override_defaults = my_graph.to_job(config=ConfigMapping(
config_fn=_override_defaults_mapping))
result = override_defaults.execute_in_process()
assert result.success
assert result.output_for_node("my_op") == TestEnum.TWO
def _ingest_config_mapping(x):
return {"ops": {"my_op": {"config": {"my_enum": x["my_field"]}}}}
default_config_mapping = ConfigMapping(
config_fn=_ingest_config_mapping,
config_schema=Shape({
"my_field":
Field(Enum.from_python_enum(TestEnum),
is_required=False,
default_value="TWO")
}),
receive_processed_config_values=False,
)
ingest_mapping = my_graph.to_job(config=default_config_mapping)
result = ingest_mapping.execute_in_process()
assert result.success
assert result.output_for_node("my_op") == TestEnum.TWO
no_default_config_mapping = ConfigMapping(
config_fn=_ingest_config_mapping,
config_schema=Shape({
"my_field":
Field(Enum.from_python_enum(TestEnum), is_required=True)
}),
receive_processed_config_values=False,
)
ingest_mapping_no_default = my_graph.to_job(
config=no_default_config_mapping)
result = ingest_mapping_no_default.execute_in_process(
run_config={"my_field": "TWO"})
assert result.success
assert result.output_for_node("my_op") == TestEnum.TWO
def _ingest_post_processed_config(x):
assert x["my_field"] == TestEnum.TWO
return {"ops": {"my_op": {"config": {"my_enum": "TWO"}}}}
config_mapping_with_preprocessing = ConfigMapping(
config_fn=_ingest_post_processed_config,
config_schema=Shape({
"my_field":
Field(Enum.from_python_enum(TestEnum), is_required=True)
}),
)
ingest_preprocessing = my_graph.to_job(
config=config_mapping_with_preprocessing)
result = ingest_preprocessing.execute_in_process(
run_config={"my_field": "TWO"})
assert result.success
assert result.output_for_node("my_op") == TestEnum.TWO
def test_nested_op_selection_with_config_mapping():
@op(config_schema=str)
def concat(context, x: str):
return x + context.op_config
@op(config_schema=str)
def my_op(context):
return context.op_config
def _nested_config_fn(outer):
return {"my_op": {"config": outer}, "concat": {"config": outer}}
@graph(config=ConfigMapping(config_fn=_nested_config_fn, config_schema=str))
def my_nested_graph():
concat(my_op())
def _config_fn(outer):
return {"my_nested_graph": {"config": outer}}
@graph(config=ConfigMapping(config_fn=_config_fn, config_schema=str))
def my_graph():
my_nested_graph()
result = my_graph.to_job().execute_in_process(run_config={"ops": {"config": "foo"}})
assert result.success
assert result.output_for_node("my_nested_graph.concat") == "foofoo"
result_sub_1 = my_graph.to_job().execute_in_process(
op_selection=["my_nested_graph.my_op", "my_nested_graph.concat"],
run_config={"ops": {"config": "hello"}},
)
assert result_sub_1.success
assert result_sub_1.output_for_node("my_nested_graph.concat") == "hellohello"
# when config mapping generates values for unselected nodes, the excess values are ignored
result_sub_2 = my_graph.to_job().execute_in_process(
op_selection=["my_nested_graph.my_op"],
run_config={"ops": {"config": "hello"}},
)
assert result_sub_2.success
assert result_sub_2.output_for_node("my_nested_graph.my_op") == "hello"
# sub sub graph
@graph
def my_super_graph():
my_graph()
my_subselected_super_job = my_super_graph.to_job(
op_selection=["my_graph.my_nested_graph.my_op"]
)
result_sub_3_1 = my_subselected_super_job.execute_in_process(
run_config={"ops": {"my_graph": {"config": "hello"}}},
)
assert result_sub_3_1.success
assert set(_success_step_keys(result_sub_3_1)) == {"my_graph.my_nested_graph.my_op"}
assert result_sub_3_1.output_for_node("my_graph.my_nested_graph.my_op") == "hello"
# execute a subselected job with op_selection
result_sub_3_2 = my_subselected_super_job.execute_in_process(
op_selection=["*"],
run_config={"ops": {"my_graph": {"config": "hello"}}},
)
assert result_sub_3_2.success
assert set(_success_step_keys(result_sub_3_2)) == {"my_graph.my_nested_graph.my_op"}
assert result_sub_3_2.output_for_node("my_graph.my_nested_graph.my_op") == "hello"
@op(config_schema={"param": str})
def do_something(_):
...
@graph
def do_it_all():
do_something()
# start
do_it_all_job = do_it_all.to_job(config=ConfigMapping(
config_fn=lambda conf:
{"solids": {
"do_something": {
"config": {
"param": conf["arg"]
}
}
}},
config_schema={"arg": str},
))
# end
def execute_do_it_all():
# start_execute
do_it_all_job.execute_in_process(run_config={"arg": "some_value"})
# end_execute