def test_aliased_solids():
@lambda_solid()
def first():
return ["first"]
@lambda_solid(input_defs=[InputDefinition(name="prev")])
def not_first(prev):
return prev + ["not_first"]
pipeline = PipelineDefinition(
solid_defs=[first, not_first],
name="test",
dependencies={
"not_first": {
"prev": DependencyDefinition("first")
},
NodeInvocation("not_first", alias="second"): {
"prev": DependencyDefinition("not_first")
},
NodeInvocation("not_first", alias="third"): {
"prev": DependencyDefinition("second")
},
},
)
result = execute_pipeline(pipeline)
assert result.success
solid_result = result.result_for_solid("third")
assert solid_result.output_value() == [
"first", "not_first", "not_first", "not_first"
]
def test_failure_hook_event():
@failure_hook
def a_hook(_):
pass
@solid
def a_solid(_):
pass
@solid
def failed_solid(_):
raise SomeUserException()
a_pipeline = PipelineDefinition(
solid_defs=[a_solid, failed_solid],
name="test",
dependencies={
NodeInvocation("a_solid", hook_defs={a_hook}): {},
NodeInvocation("failed_solid", hook_defs={a_hook}): {},
},
)
result = execute_pipeline(a_pipeline, raise_on_error=False)
assert not result.success
hook_events = list(filter(lambda event: event.is_hook_event, result.event_list))
# when a hook is not triggered, we fire hook skipped event instead of completed
assert len(hook_events) == 2
for event in hook_events:
if event.event_type == DagsterEventType.HOOK_COMPLETED:
assert event.solid_name == "failed_solid"
if event.event_type == DagsterEventType.HOOK_SKIPPED:
assert event.solid_name == "a_solid"
def test_required_inputs():
@lambda_solid(input_defs=[InputDefinition("num", Int)],
output_def=OutputDefinition(Int))
def add_one(num):
return num + 1
pipeline_def = PipelineDefinition(
name="required_int_input",
solid_defs=[add_one],
dependencies={
NodeInvocation("add_one", "first_add"): {},
NodeInvocation("add_one", "second_add"): {
"num": DependencyDefinition("first_add")
},
},
)
env_type = create_run_config_schema_type(pipeline_def)
solids_type = env_type.fields["solids"].config_type
first_add_fields = solids_type.fields["first_add"].config_type.fields
assert "inputs" in first_add_fields
inputs_field = first_add_fields["inputs"]
assert inputs_field.is_required
assert inputs_field.config_type.fields["num"].is_required
# second_add has a dependency so the input is not available
assert "inputs" not in solids_type.fields["second_add"].config_type.fields
def test_success_hook():
called_hook_to_solids = defaultdict(list)
@success_hook
def a_success_hook(context):
called_hook_to_solids[context.hook_def.name].append(context.solid.name)
@success_hook(name="a_named_success_hook")
def named_success_hook(context):
called_hook_to_solids[context.hook_def.name].append(context.solid.name)
@success_hook(required_resource_keys={"resource_a"})
def success_hook_resource(context):
called_hook_to_solids[context.hook_def.name].append(context.solid.name)
assert context.resources.resource_a == 1
@solid
def succeeded_solid(_):
pass
@solid
def failed_solid(_):
# this solid shouldn't trigger success hooks
raise SomeUserException()
a_pipeline = PipelineDefinition(
solid_defs=[succeeded_solid, failed_solid],
name="test",
dependencies={
NodeInvocation(
"succeeded_solid",
"succeeded_solid_with_hook",
hook_defs={
a_success_hook, named_success_hook, success_hook_resource
},
): {},
NodeInvocation(
"failed_solid",
"failed_solid_with_hook",
hook_defs={a_success_hook, named_success_hook},
): {},
},
mode_defs=[ModeDefinition(resource_defs={"resource_a": resource_a})],
)
result = execute_pipeline(a_pipeline, raise_on_error=False)
assert not result.success
# test if hooks are run for the given solids
assert "succeeded_solid_with_hook" in called_hook_to_solids[
"a_success_hook"]
assert "succeeded_solid_with_hook" in called_hook_to_solids[
"a_named_success_hook"]
assert "succeeded_solid_with_hook" in called_hook_to_solids[
"success_hook_resource"]
assert "failed_solid_with_hook" not in called_hook_to_solids[
"a_success_hook"]
assert "failed_solid_with_hook" not in called_hook_to_solids[
"a_named_success_hook"]
def test_aliased_configs():
@solid(input_defs=[], config_schema=Int)
def load_constant(context):
return context.solid_config
pipeline = PipelineDefinition(
solid_defs=[load_constant],
name="test",
dependencies={
NodeInvocation(load_constant.name, "load_a"): {},
NodeInvocation(load_constant.name, "load_b"): {},
},
)
result = execute_pipeline(
pipeline,
{"solids": {
"load_a": {
"config": 2
},
"load_b": {
"config": 3
}
}})
assert result.success
assert result.result_for_solid("load_a").output_value() == 2
assert result.result_for_solid("load_b").output_value() == 3
def test_nothing_inputs():
@lambda_solid(input_defs=[InputDefinition("never_defined", Nothing)])
def emit_one():
return 1
@lambda_solid
def emit_two():
return 2
@lambda_solid
def emit_three():
return 3
@lambda_solid(output_def=OutputDefinition(Nothing))
def emit_nothing():
pass
@solid(input_defs=[
InputDefinition("_one", Nothing),
InputDefinition("one", Int),
InputDefinition("_two", Nothing),
InputDefinition("two", Int),
InputDefinition("_three", Nothing),
InputDefinition("three", Int),
])
def adder(_context, one, two, three):
assert one == 1
assert two == 2
assert three == 3
return one + two + three
pipeline = PipelineDefinition(
name="input_test",
solid_defs=[emit_one, emit_two, emit_three, emit_nothing, adder],
dependencies={
NodeInvocation("emit_nothing", "_one"): {},
NodeInvocation("emit_nothing", "_two"): {},
NodeInvocation("emit_nothing", "_three"): {},
"adder": {
"_one": DependencyDefinition("_one"),
"_two": DependencyDefinition("_two"),
"_three": DependencyDefinition("_three"),
"one": DependencyDefinition("emit_one"),
"two": DependencyDefinition("emit_two"),
"three": DependencyDefinition("emit_three"),
},
},
)
result = execute_pipeline(pipeline)
assert result.success
def test_fanin_deps():
called = defaultdict(int)
@lambda_solid
def emit_two():
return 2
@lambda_solid(output_def=OutputDefinition(Nothing))
def emit_nothing():
called["emit_nothing"] += 1
@solid(input_defs=[
InputDefinition("ready", Nothing),
InputDefinition("num_1", Int),
InputDefinition("num_2", Int),
])
def adder(_context, num_1, num_2):
assert called["emit_nothing"] == 3
called["adder"] += 1
return num_1 + num_2
pipeline = PipelineDefinition(
name="input_test",
solid_defs=[emit_two, emit_nothing, adder],
dependencies={
NodeInvocation("emit_two", "emit_1"): {},
NodeInvocation("emit_two", "emit_2"): {},
NodeInvocation("emit_nothing", "_one"): {},
NodeInvocation("emit_nothing", "_two"): {},
NodeInvocation("emit_nothing", "_three"): {},
"adder": {
"ready":
MultiDependencyDefinition([
DependencyDefinition("_one"),
DependencyDefinition("_two"),
DependencyDefinition("_three"),
]),
"num_1":
DependencyDefinition("emit_1"),
"num_2":
DependencyDefinition("emit_2"),
},
},
)
result = execute_pipeline(pipeline)
assert result.success
assert called["adder"] == 1
assert called["emit_nothing"] == 3
def test_hook_resource_error():
@event_list_hook(required_resource_keys={"resource_b"})
def a_hook(context, event_list): # pylint: disable=unused-argument
return HookExecutionResult(hook_name="a_hook")
@solid
def a_solid(_):
pass
with pytest.raises(
DagsterInvalidDefinitionError,
match="resource key 'resource_b' is required by hook 'a_hook'",
):
PipelineDefinition(
solid_defs=[a_solid],
name="test",
dependencies={
NodeInvocation("a_solid",
"a_solid_with_hook",
hook_defs={a_hook}): {}
},
mode_defs=[
ModeDefinition(resource_defs={"resource_a": resource_a})
],
)
def test_hook_with_resource():
called = {}
@event_list_hook(required_resource_keys={"resource_a"})
def a_hook(context, _):
called[context.solid.name] = True
assert context.resources.resource_a == 1
return HookExecutionResult(hook_name="a_hook")
@solid
def a_solid(_):
pass
a_pipeline = PipelineDefinition(
solid_defs=[a_solid],
name="test",
dependencies={
NodeInvocation("a_solid", "a_solid_with_hook", hook_defs={a_hook}):
{}
},
mode_defs=[ModeDefinition(resource_defs={"resource_a": resource_a})],
)
result = execute_pipeline(a_pipeline)
assert result.success
assert called.get("a_solid_with_hook")
def test_hook():
called = {}
@event_list_hook
def a_hook(context, event_list):
called[context.hook_def.name] = context.solid.name
called["step_event_list"] = [i for i in event_list]
return HookExecutionResult(hook_name="a_hook")
@event_list_hook(name="a_named_hook")
def named_hook(context, _):
called[context.hook_def.name] = context.solid.name
return HookExecutionResult(hook_name="a_hook")
@solid
def a_solid(_):
pass
a_pipeline = PipelineDefinition(
solid_defs=[a_solid],
name="test",
dependencies={
NodeInvocation("a_solid", "a_solid_with_hook", hook_defs={a_hook, named_hook}): {}
},
)
result = execute_pipeline(a_pipeline)
assert result.success
assert called.get("a_hook") == "a_solid_with_hook"
assert called.get("a_named_hook") == "a_solid_with_hook"
assert set([event.event_type_value for event in called["step_event_list"]]) == set(
[event.event_type_value for event in result.step_event_list]
)
def test_hook_user_error():
@event_list_hook
def error_hook(context, _):
raise SomeUserException()
@solid
def a_solid(_):
return 1
a_pipeline = PipelineDefinition(
solid_defs=[a_solid],
name="test",
dependencies={
NodeInvocation("a_solid",
"a_solid_with_hook",
hook_defs={error_hook}): {}
},
)
result = execute_pipeline(a_pipeline)
assert result.success
hook_errored_events = list(
filter(lambda event: event.event_type == DagsterEventType.HOOK_ERRORED,
result.event_list))
assert len(hook_errored_events) == 1
assert hook_errored_events[0].solid_handle.name == "a_solid_with_hook"
def test_solid_instance_tags():
called = {}
@solid(tags={"foo": "bar", "baz": "quux"})
def metadata_solid(context):
assert context.solid.tags == {"foo": "oof", "baz": "quux", "bip": "bop"}
called["yup"] = True
pipeline = PipelineDefinition(
name="metadata_pipeline",
solid_defs=[metadata_solid],
dependencies={
NodeInvocation(
"metadata_solid",
alias="aliased_metadata_solid",
tags={"foo": "oof", "bip": "bop"},
): {}
},
)
result = execute_pipeline(
pipeline,
)
assert result.success
assert called["yup"]
def test_mapper_errors():
@lambda_solid
def solid_a():
return 1
with pytest.raises(DagsterInvalidDefinitionError) as excinfo_1:
PipelineDefinition(
solid_defs=[solid_a],
name="test",
dependencies={"solid_b": {"arg_a": DependencyDefinition("solid_a")}},
)
assert (
str(excinfo_1.value)
== 'Invalid dependencies: node "solid_b" in dependency dictionary not found in node list'
)
with pytest.raises(DagsterInvalidDefinitionError) as excinfo_2:
PipelineDefinition(
solid_defs=[solid_a],
name="test",
dependencies={
NodeInvocation("solid_b", alias="solid_c"): {
"arg_a": DependencyDefinition("solid_a")
}
},
)
assert (
str(excinfo_2.value)
== 'Invalid dependencies: node "solid_b" (aliased by "solid_c" in dependency dictionary) not found in node list'
)
def test_string_from_aliased_inputs():
called = {}
@solid(input_defs=[InputDefinition("string_input", String)])
def str_as_input(_context, string_input):
assert string_input == "foo"
called["yup"] = True
pipeline = PipelineDefinition(
solid_defs=[str_as_input],
name="test",
dependencies={NodeInvocation("str_as_input", alias="aliased"): {}},
)
result = execute_pipeline(pipeline, {
"solids": {
"aliased": {
"inputs": {
"string_input": {
"value": "foo"
}
}
}
}
})
assert result.success
assert called["yup"]
def test_cycle_detect():
@lambda_solid
def return_one():
return 1
@lambda_solid
def add(a, b):
return a + b
with pytest.raises(DagsterInvalidDefinitionError,
match="Circular dependencies exist"):
PipelineDefinition(
solid_defs=[return_one, add],
name="test",
dependencies={
NodeInvocation("add", alias="first"): {
"a": DependencyDefinition("return_one"),
"b": DependencyDefinition("second"),
},
NodeInvocation("add", alias="second"): {
"a": DependencyDefinition("first"),
"b": DependencyDefinition("return_one"),
},
},
)
with pytest.raises(DagsterInvalidDefinitionError,
match="Circular dependencies exist"):
CompositeSolidDefinition(
name="circletron",
solid_defs=[return_one, add],
dependencies={
NodeInvocation("add", alias="first"): {
"a": DependencyDefinition("return_one"),
"b": DependencyDefinition("second"),
},
NodeInvocation("add", alias="second"): {
"a": DependencyDefinition("first"),
"b": DependencyDefinition("return_one"),
},
},
)
def test_aliased_solids_context():
record = defaultdict(set)
@solid
def log_things(context):
solid_value = context.solid.name
solid_def_value = context.solid_def.name
record[solid_def_value].add(solid_value)
pipeline = PipelineDefinition(
solid_defs=[log_things],
name="test",
dependencies={
NodeInvocation("log_things", "log_a"): {},
NodeInvocation("log_things", "log_b"): {},
},
)
result = execute_pipeline(pipeline)
assert result.success
assert dict(record) == {"log_things": set(["log_a", "log_b"])}
def construct_graph_with_yaml(yaml_file, op_defs) -> GraphDefinition:
yaml_data = load_yaml_from_path(yaml_file)
deps = {}
for op_yaml_data in yaml_data["ops"]:
def_name = op_yaml_data["def"]
alias = op_yaml_data.get("alias", def_name)
op_deps_entry = {}
for input_name, input_data in op_yaml_data.get("deps", {}).items():
op_deps_entry[input_name] = DependencyDefinition(
solid=input_data["op"],
output=input_data.get("output", "result"))
deps[NodeInvocation(name=def_name, alias=alias)] = op_deps_entry
return GraphDefinition(
name=yaml_data["name"],
description=yaml_data.get("description"),
node_defs=op_defs,
dependencies=deps,
)
def _dep_key_of(solid):
return NodeInvocation(solid.definition.name, solid.name)