def test_success_hook():
called_hook_to_solids = defaultdict(list)
@success_hook
def a_success_hook(context):
called_hook_to_solids[context.hook_def.name].append(context.solid.name)
@success_hook(name="a_named_success_hook")
def named_success_hook(context):
called_hook_to_solids[context.hook_def.name].append(context.solid.name)
@success_hook(required_resource_keys={"resource_a"})
def success_hook_resource(context):
called_hook_to_solids[context.hook_def.name].append(context.solid.name)
assert context.resources.resource_a == 1
@solid
def succeeded_solid(_):
pass
@solid
def failed_solid(_):
# this solid shouldn't trigger success hooks
raise SomeUserException()
a_pipeline = PipelineDefinition(
solid_defs=[succeeded_solid, failed_solid],
name="test",
dependencies={
SolidInvocation(
"succeeded_solid",
"succeeded_solid_with_hook",
hook_defs={
a_success_hook, named_success_hook, success_hook_resource
},
): {},
SolidInvocation(
"failed_solid",
"failed_solid_with_hook",
hook_defs={a_success_hook, named_success_hook},
): {},
},
mode_defs=[ModeDefinition(resource_defs={"resource_a": resource_a})],
)
result = execute_pipeline(a_pipeline, raise_on_error=False)
assert not result.success
# test if hooks are run for the given solids
assert "succeeded_solid_with_hook" in called_hook_to_solids[
"a_success_hook"]
assert "succeeded_solid_with_hook" in called_hook_to_solids[
"a_named_success_hook"]
assert "succeeded_solid_with_hook" in called_hook_to_solids[
"success_hook_resource"]
assert "failed_solid_with_hook" not in called_hook_to_solids[
"a_success_hook"]
assert "failed_solid_with_hook" not in called_hook_to_solids[
"a_named_success_hook"]
def test_aliased_solids():
@lambda_solid()
def first():
return ["first"]
@lambda_solid(input_defs=[InputDefinition(name="prev")])
def not_first(prev):
return prev + ["not_first"]
pipeline = PipelineDefinition(
solid_defs=[first, not_first],
dependencies={
"not_first": {
"prev": DependencyDefinition("first")
},
SolidInvocation("not_first", alias="second"): {
"prev": DependencyDefinition("not_first")
},
SolidInvocation("not_first", alias="third"): {
"prev": DependencyDefinition("second")
},
},
)
result = execute_pipeline(pipeline)
assert result.success
solid_result = result.result_for_solid("third")
assert solid_result.output_value() == [
"first", "not_first", "not_first", "not_first"
]
def test_failure_hook_event():
@failure_hook
def a_hook(_):
pass
@solid
def a_solid(_):
pass
@solid
def failed_solid(_):
raise SomeUserException()
a_pipeline = PipelineDefinition(
solid_defs=[a_solid, failed_solid],
dependencies={
SolidInvocation('a_solid', hook_defs={a_hook}): {},
SolidInvocation('failed_solid', hook_defs={a_hook}): {},
},
)
result = execute_pipeline(a_pipeline, raise_on_error=False)
assert not result.success
hook_events = list(
filter(lambda event: event.is_hook_event, result.event_list))
# when a hook is not triggered, we fire hook skipped event instead of completed
assert len(hook_events) == 2
for event in hook_events:
if event.event_type == DagsterEventType.HOOK_COMPLETED:
assert event.solid_name == 'failed_solid'
if event.event_type == DagsterEventType.HOOK_SKIPPED:
assert event.solid_name == 'a_solid'
def test_aliased_configs():
@solid(input_defs=[], config_schema=Int)
def load_constant(context):
return context.solid_config
pipeline = PipelineDefinition(
solid_defs=[load_constant],
dependencies={
SolidInvocation(load_constant.name, "load_a"): {},
SolidInvocation(load_constant.name, "load_b"): {},
},
)
result = execute_pipeline(
pipeline,
{"solids": {
"load_a": {
"config": 2
},
"load_b": {
"config": 3
}
}})
assert result.success
assert result.result_for_solid("load_a").output_value() == 2
assert result.result_for_solid("load_b").output_value() == 3
def test_aliased_solids():
@lambda_solid()
def first():
return ['first']
@lambda_solid(input_defs=[InputDefinition(name="prev")])
def not_first(prev):
return prev + ['not_first']
pipeline = PipelineDefinition(
solid_defs=[first, not_first],
dependencies={
'not_first': {
'prev': DependencyDefinition('first')
},
SolidInvocation('not_first', alias='second'): {
'prev': DependencyDefinition('not_first')
},
SolidInvocation('not_first', alias='third'): {
'prev': DependencyDefinition('second')
},
},
)
result = execute_pipeline(pipeline)
assert result.success
solid_result = result.result_for_solid('third')
assert solid_result.output_value() == [
'first', 'not_first', 'not_first', 'not_first'
]
def test_aliased_configs():
@solid(input_defs=[], config=Int)
def load_constant(context):
return context.solid_config
pipeline = PipelineDefinition(
solid_defs=[load_constant],
dependencies={
SolidInvocation(load_constant.name, 'load_a'): {},
SolidInvocation(load_constant.name, 'load_b'): {},
},
)
result = execute_pipeline(
pipeline,
{'solids': {
'load_a': {
'config': 2
},
'load_b': {
'config': 3
}
}})
assert result.success
assert result.result_for_solid('load_a').output_value() == 2
assert result.result_for_solid('load_b').output_value() == 3
def test_required_inputs():
@lambda_solid(input_defs=[InputDefinition('num', Int)], output_def=OutputDefinition(Int))
def add_one(num):
return num + 1
pipeline_def = PipelineDefinition(
name='required_int_input',
solid_defs=[add_one],
dependencies={
SolidInvocation('add_one', 'first_add'): {},
SolidInvocation('add_one', 'second_add'): {'num': DependencyDefinition('first_add')},
},
)
env_type = create_environment_type(pipeline_def)
solids_type = env_type.fields['solids'].config_type
first_add_fields = solids_type.fields['first_add'].config_type.fields
assert 'inputs' in first_add_fields
inputs_field = first_add_fields['inputs']
assert inputs_field.is_required
assert inputs_field.config_type.fields['num'].is_required
# second_add has a dependency so the input is not available
assert 'inputs' not in solids_type.fields['second_add'].config_type.fields
def test_required_inputs():
@lambda_solid(input_defs=[InputDefinition("num", Int)], output_def=OutputDefinition(Int))
def add_one(num):
return num + 1
pipeline_def = PipelineDefinition(
name="required_int_input",
solid_defs=[add_one],
dependencies={
SolidInvocation("add_one", "first_add"): {},
SolidInvocation("add_one", "second_add"): {"num": DependencyDefinition("first_add")},
},
)
env_type = create_environment_type(pipeline_def)
solids_type = env_type.fields["solids"].config_type
first_add_fields = solids_type.fields["first_add"].config_type.fields
assert "inputs" in first_add_fields
inputs_field = first_add_fields["inputs"]
assert inputs_field.is_required
assert inputs_field.config_type.fields["num"].is_required
# second_add has a dependency so the input is not available
assert "inputs" not in solids_type.fields["second_add"].config_type.fields
def test_failure_hook():
called_hook_to_solids = defaultdict(list)
@failure_hook
def a_failure_hook(context):
called_hook_to_solids[context.hook_def.name].append(context.solid.name)
@failure_hook(name='a_named_failure_hook')
def named_failure_hook(context):
called_hook_to_solids[context.hook_def.name].append(context.solid.name)
@failure_hook(required_resource_keys={'resource_a'})
def failure_hook_resource(context):
called_hook_to_solids[context.hook_def.name].append(context.solid.name)
assert context.resources.resource_a == 1
@solid
def succeeded_solid(_):
# this solid shouldn't trigger failure hooks
pass
@solid
def failed_solid(_):
raise SomeUserException()
a_pipeline = PipelineDefinition(
solid_defs=[failed_solid, succeeded_solid],
dependencies={
SolidInvocation(
'failed_solid',
'failed_solid_with_hook',
hook_defs={
a_failure_hook, named_failure_hook, failure_hook_resource
},
): {},
SolidInvocation(
'succeeded_solid',
'succeeded_solid_with_hook',
hook_defs={a_failure_hook, named_failure_hook},
): {},
},
mode_defs=[ModeDefinition(resource_defs={'resource_a': resource_a})],
)
result = execute_pipeline(a_pipeline, raise_on_error=False)
assert not result.success
# test if hooks are run for the given solids
assert 'failed_solid_with_hook' in called_hook_to_solids['a_failure_hook']
assert 'failed_solid_with_hook' in called_hook_to_solids[
'a_named_failure_hook']
assert 'failed_solid_with_hook' in called_hook_to_solids[
'failure_hook_resource']
assert 'succeeded_solid_with_hook' not in called_hook_to_solids[
'a_failure_hook']
assert 'succeeded_solid_with_hook' not in called_hook_to_solids[
'a_named_failure_hook']
def define_tutorial_pipeline():
return PipelineDefinition(
name='tutorial_pipeline',
solid_defs=[clean_data_solid, LR_solid, RF_solid],
dependencies={
SolidInvocation('clean_data'): {},
SolidInvocation('linear_regression'): {'df': DependencyDefinition('clean_data')},
SolidInvocation('random_forest_regression'): {'df': DependencyDefinition('clean_data')},
},
)
def define_tutorial_pipeline():
return PipelineDefinition(
name="tutorial_pipeline",
solid_defs=[clean_data_solid, LR_solid, RF_solid],
dependencies={
SolidInvocation("clean_data"): {},
SolidInvocation("linear_regression"): {"df": DependencyDefinition("clean_data")},
SolidInvocation("random_forest_regression"): {
"df": DependencyDefinition("clean_data")
},
},
)
def test_nothing_inputs():
@lambda_solid(input_defs=[InputDefinition('never_defined', Nothing)])
def emit_one():
return 1
@lambda_solid
def emit_two():
return 2
@lambda_solid
def emit_three():
return 3
@lambda_solid(output_def=OutputDefinition(Nothing))
def emit_nothing():
pass
@solid(
input_defs=[
InputDefinition('_one', Nothing),
InputDefinition('one', Int),
InputDefinition('_two', Nothing),
InputDefinition('two', Int),
InputDefinition('_three', Nothing),
InputDefinition('three', Int),
]
)
def adder(_context, one, two, three):
assert one == 1
assert two == 2
assert three == 3
return one + two + three
pipeline = PipelineDefinition(
name='input_test',
solid_defs=[emit_one, emit_two, emit_three, emit_nothing, adder],
dependencies={
SolidInvocation('emit_nothing', '_one'): {},
SolidInvocation('emit_nothing', '_two'): {},
SolidInvocation('emit_nothing', '_three'): {},
'adder': {
'_one': DependencyDefinition('_one'),
'_two': DependencyDefinition('_two'),
'_three': DependencyDefinition('_three'),
'one': DependencyDefinition('emit_one'),
'two': DependencyDefinition('emit_two'),
'three': DependencyDefinition('emit_three'),
},
},
)
result = execute_pipeline(pipeline)
assert result.success
def test_nothing_inputs():
@lambda_solid(input_defs=[InputDefinition("never_defined", Nothing)])
def emit_one():
return 1
@lambda_solid
def emit_two():
return 2
@lambda_solid
def emit_three():
return 3
@lambda_solid(output_def=OutputDefinition(Nothing))
def emit_nothing():
pass
@solid(
input_defs=[
InputDefinition("_one", Nothing),
InputDefinition("one", Int),
InputDefinition("_two", Nothing),
InputDefinition("two", Int),
InputDefinition("_three", Nothing),
InputDefinition("three", Int),
]
)
def adder(_context, one, two, three):
assert one == 1
assert two == 2
assert three == 3
return one + two + three
pipeline = PipelineDefinition(
name="input_test",
solid_defs=[emit_one, emit_two, emit_three, emit_nothing, adder],
dependencies={
SolidInvocation("emit_nothing", "_one"): {},
SolidInvocation("emit_nothing", "_two"): {},
SolidInvocation("emit_nothing", "_three"): {},
"adder": {
"_one": DependencyDefinition("_one"),
"_two": DependencyDefinition("_two"),
"_three": DependencyDefinition("_three"),
"one": DependencyDefinition("emit_one"),
"two": DependencyDefinition("emit_two"),
"three": DependencyDefinition("emit_three"),
},
},
)
result = execute_pipeline(pipeline)
assert result.success
def test_composite_basic_execution():
a_source = define_stub_solid('A_source', [input_set('A_input')])
node_a = create_root_solid('A')
node_b = create_solid_with_deps('B', node_a)
node_c = create_solid_with_deps('C', node_a)
node_d = create_solid_with_deps('D', node_b, node_c)
diamond_composite = CompositeSolidDefinition(
name='diamond_composite',
solid_defs=[a_source, node_a, node_b, node_c, node_d],
dependencies={
'A': {
'A_input': DependencyDefinition('A_source')
},
'B': {
'A': DependencyDefinition('A')
},
'C': {
'A': DependencyDefinition('A')
},
'D': {
'B': DependencyDefinition('B'),
'C': DependencyDefinition('C')
},
},
)
result = execute_pipeline(
PipelineDefinition(solid_defs=[diamond_composite]))
assert result.success
result = execute_pipeline(
PipelineDefinition(
solid_defs=[diamond_composite],
dependencies={
SolidInvocation('diamond_composite', alias='D1'): {},
SolidInvocation('diamond_composite', alias='D2'): {},
},
))
assert result.success
wrapped_composite = CompositeSolidDefinition(
name='wrapped_composite', solid_defs=[diamond_composite])
result = execute_pipeline(
PipelineDefinition(solid_defs=[diamond_composite, wrapped_composite]))
assert result.success
empty_composite = CompositeSolidDefinition(name='empty', solid_defs=[])
result = execute_pipeline(PipelineDefinition(solid_defs=[empty_composite]))
assert result.success
def test_fanin_deps():
called = defaultdict(int)
@lambda_solid
def emit_two():
return 2
@lambda_solid(output_def=OutputDefinition(Nothing))
def emit_nothing():
called['emit_nothing'] += 1
@solid(
input_defs=[
InputDefinition('ready', Nothing),
InputDefinition('num_1', Int),
InputDefinition('num_2', Int),
]
)
def adder(_context, num_1, num_2):
assert called['emit_nothing'] == 3
called['adder'] += 1
return num_1 + num_2
pipeline = PipelineDefinition(
name='input_test',
solid_defs=[emit_two, emit_nothing, adder],
dependencies={
SolidInvocation('emit_two', 'emit_1'): {},
SolidInvocation('emit_two', 'emit_2'): {},
SolidInvocation('emit_nothing', '_one'): {},
SolidInvocation('emit_nothing', '_two'): {},
SolidInvocation('emit_nothing', '_three'): {},
'adder': {
'ready': MultiDependencyDefinition(
[
DependencyDefinition('_one'),
DependencyDefinition('_two'),
DependencyDefinition('_three'),
]
),
'num_1': DependencyDefinition('emit_1'),
'num_2': DependencyDefinition('emit_2'),
},
},
)
result = execute_pipeline(pipeline)
assert result.success
assert called['adder'] == 1
assert called['emit_nothing'] == 3
def test_fanin_deps():
called = defaultdict(int)
@lambda_solid
def emit_two():
return 2
@lambda_solid(output_def=OutputDefinition(Nothing))
def emit_nothing():
called["emit_nothing"] += 1
@solid(
input_defs=[
InputDefinition("ready", Nothing),
InputDefinition("num_1", Int),
InputDefinition("num_2", Int),
]
)
def adder(_context, num_1, num_2):
assert called["emit_nothing"] == 3
called["adder"] += 1
return num_1 + num_2
pipeline = PipelineDefinition(
name="input_test",
solid_defs=[emit_two, emit_nothing, adder],
dependencies={
SolidInvocation("emit_two", "emit_1"): {},
SolidInvocation("emit_two", "emit_2"): {},
SolidInvocation("emit_nothing", "_one"): {},
SolidInvocation("emit_nothing", "_two"): {},
SolidInvocation("emit_nothing", "_three"): {},
"adder": {
"ready": MultiDependencyDefinition(
[
DependencyDefinition("_one"),
DependencyDefinition("_two"),
DependencyDefinition("_three"),
]
),
"num_1": DependencyDefinition("emit_1"),
"num_2": DependencyDefinition("emit_2"),
},
},
)
result = execute_pipeline(pipeline)
assert result.success
assert called["adder"] == 1
assert called["emit_nothing"] == 3
def test_hook_resource_error():
@event_list_hook(required_resource_keys={"resource_b"})
def a_hook(context, event_list): # pylint: disable=unused-argument
return HookExecutionResult(hook_name="a_hook")
@solid
def a_solid(_):
pass
with pytest.raises(
DagsterInvalidDefinitionError,
match='Resource key "resource_b" is required by hook "a_hook"',
):
PipelineDefinition(
solid_defs=[a_solid],
name="test",
dependencies={
SolidInvocation("a_solid",
"a_solid_with_hook",
hook_defs={a_hook}): {}
},
mode_defs=[
ModeDefinition(resource_defs={"resource_a": resource_a})
],
)
def test_string_from_aliased_inputs():
called = {}
@solid(input_defs=[InputDefinition("string_input", String)])
def str_as_input(_context, string_input):
assert string_input == "foo"
called["yup"] = True
pipeline = PipelineDefinition(
solid_defs=[str_as_input],
name="test",
dependencies={SolidInvocation("str_as_input", alias="aliased"): {}},
)
result = execute_pipeline(pipeline, {
"solids": {
"aliased": {
"inputs": {
"string_input": {
"value": "foo"
}
}
}
}
})
assert result.success
assert called["yup"]
def test_solid_instance_tags():
called = {}
@solid(tags={"foo": "bar", "baz": "quux"})
def metadata_solid(context):
assert context.solid.tags == {
"foo": "oof",
"baz": "quux",
"bip": "bop"
}
called["yup"] = True
pipeline = PipelineDefinition(
name="metadata_pipeline",
solid_defs=[metadata_solid],
dependencies={
SolidInvocation(
"metadata_solid",
alias="aliased_metadata_solid",
tags={
"foo": "oof",
"bip": "bop"
},
): {}
},
)
result = execute_pipeline(pipeline, )
assert result.success
assert called["yup"]
def test_string_from_aliased_inputs():
called = {}
@solid(input_defs=[InputDefinition('string_input', String)])
def str_as_input(_context, string_input):
assert string_input == 'foo'
called['yup'] = True
pipeline = PipelineDefinition(
solid_defs=[str_as_input],
dependencies={SolidInvocation('str_as_input', alias='aliased'): {}},
)
result = execute_pipeline(pipeline, {
'solids': {
'aliased': {
'inputs': {
'string_input': {
'value': 'foo'
}
}
}
}
})
assert result.success
assert called['yup']
def test_hook_user_error():
@event_list_hook
def error_hook(context, _):
raise SomeUserException()
@solid
def a_solid(_):
return 1
a_pipeline = PipelineDefinition(
solid_defs=[a_solid],
dependencies={
SolidInvocation('a_solid',
'a_solid_with_hook',
hook_defs={error_hook}): {}
},
)
result = execute_pipeline(a_pipeline)
assert result.success
hook_errored_events = list(
filter(lambda event: event.event_type == DagsterEventType.HOOK_ERRORED,
result.event_list))
assert len(hook_errored_events) == 1
assert hook_errored_events[0].solid_handle.name == 'a_solid_with_hook'
def test_hook_with_resource():
called = {}
@event_list_hook(required_resource_keys={'resource_a'})
def a_hook(context, _):
called[context.solid.name] = True
assert context.resources.resource_a == 1
return HookExecutionResult(hook_name='a_hook')
@solid
def a_solid(_):
pass
a_pipeline = PipelineDefinition(
solid_defs=[a_solid],
dependencies={
SolidInvocation('a_solid', 'a_solid_with_hook', hook_defs={a_hook}):
{}
},
mode_defs=[ModeDefinition(resource_defs={'resource_a': resource_a})],
)
result = execute_pipeline(a_pipeline)
assert result.success
assert called.get('a_solid_with_hook')
def test_hook():
called = {}
@event_list_hook
def a_hook(context, event_list):
called[context.hook_def.name] = context.solid.name
called['step_event_list'] = [i for i in event_list]
return HookExecutionResult(hook_name='a_hook')
@event_list_hook(name='a_named_hook')
def named_hook(context, _):
called[context.hook_def.name] = context.solid.name
return HookExecutionResult(hook_name='a_hook')
@solid
def a_solid(_):
pass
a_pipeline = PipelineDefinition(
solid_defs=[a_solid],
dependencies={
SolidInvocation('a_solid',
'a_solid_with_hook',
hook_defs={a_hook, named_hook}): {}
},
)
result = execute_pipeline(a_pipeline)
assert result.success
assert called.get('a_hook') == 'a_solid_with_hook'
assert called.get('a_named_hook') == 'a_solid_with_hook'
assert set([
event.event_type_value for event in called['step_event_list']
]) == set([event.event_type_value for event in result.step_event_list])
def test_mapper_errors():
@lambda_solid
def solid_a():
return 1
with pytest.raises(DagsterInvalidDefinitionError) as excinfo_1:
PipelineDefinition(
solid_defs=[solid_a],
dependencies={'solid_b': {'arg_a': DependencyDefinition('solid_a')}},
)
assert (
str(excinfo_1.value)
== 'Invalid dependencies: solid "solid_b" in dependency dictionary not found in solid list'
)
with pytest.raises(DagsterInvalidDefinitionError) as excinfo_2:
PipelineDefinition(
solid_defs=[solid_a],
dependencies={
SolidInvocation('solid_b', alias='solid_c'): {
'arg_a': DependencyDefinition('solid_a')
}
},
)
assert (
str(excinfo_2.value)
== 'Invalid dependencies: solid "solid_b" (aliased by "solid_c" in dependency dictionary) not found in solid list'
)
def define_test_notebook_dag_pipeline():
return PipelineDefinition(
name='test_notebook_dag',
solid_defs=[load_constant, add_two_numbers_pm_solid, mult_two_numbers_pm_solid],
dependencies={
SolidInvocation('load_constant', alias='load_a'): {},
SolidInvocation('load_constant', alias='load_b'): {},
SolidInvocation(name='add_two_numbers', alias='add_two'): {
'a': DependencyDefinition('load_a'),
'b': DependencyDefinition('load_b'),
},
SolidInvocation(name='mult_two_numbers', alias='mult_two'): {
'a': DependencyDefinition('add_two'),
'b': DependencyDefinition('load_b'),
},
},
)
def define_test_notebook_dag_pipeline():
return PipelineDefinition(
name="test_notebook_dag",
solid_defs=[load_constant, add_two_numbers_pm_solid, mult_two_numbers_pm_solid],
dependencies={
SolidInvocation("load_constant", alias="load_a"): {},
SolidInvocation("load_constant", alias="load_b"): {},
SolidInvocation(name="add_two_numbers", alias="add_two"): {
"a": DependencyDefinition("load_a"),
"b": DependencyDefinition("load_b"),
},
SolidInvocation(name="mult_two_numbers", alias="mult_two"): {
"a": DependencyDefinition("add_two"),
"b": DependencyDefinition("load_b"),
},
},
)
def define_spew_pipeline():
return PipelineDefinition(
name='log_spew',
solid_defs=[
nonce_solid('no_in_two_out', 0, 2),
nonce_solid('one_in_one_out', 1, 1),
nonce_solid('one_in_two_out', 1, 2),
nonce_solid('two_in_one_out', 2, 1),
nonce_solid('one_in_none_out', 1, 0),
],
dependencies={
SolidInvocation('no_in_two_out', alias='solid_a'): {},
SolidInvocation('one_in_one_out', alias='solid_b'): {
'input_0': DependencyDefinition('solid_a', 'output_0')
},
SolidInvocation('one_in_two_out', alias='solid_c'): {
'input_0': DependencyDefinition('solid_a', 'output_1')
},
SolidInvocation('two_in_one_out', alias='solid_d'): {
'input_0': DependencyDefinition('solid_b', 'output_0'),
'input_1': DependencyDefinition('solid_c', 'output_0'),
},
SolidInvocation('one_in_one_out', alias='solid_e'): {
'input_0': DependencyDefinition('solid_c', 'output_0')
},
SolidInvocation('two_in_one_out', alias='solid_f'): {
'input_0': DependencyDefinition('solid_d', 'output_0'),
'input_1': DependencyDefinition('solid_e', 'output_0'),
},
SolidInvocation('one_in_none_out', alias='solid_g'): {
'input_0': DependencyDefinition('solid_f', 'output_0')
},
},
mode_definitions=[ModeDefinition()],
)
def test_with_resource_mapper_fn_without_required_keys():
resources_initted = {}
@resource
def resource_a(_):
resources_initted['a'] = True
yield 'A'
@resource
def resource_c_alias(_):
resources_initted['c'] = True
yield 'C'
@solid(required_resource_keys={'a'})
def consumes_resource_a(context):
assert context.resources.a == 'A'
return context.resources.a
@solid
def consumes_resource_c(context):
assert context.resources.c == 'C'
return context.resources.c
def _resource_mapper_fn(resources, _resource_deps):
return {'a': resources['a'], 'c': resources['c_alias']}
pipeline_with_resource_mapper = PipelineDefinition(
solid_defs=[consumes_resource_a, consumes_resource_c],
mode_defs=[
ModeDefinition(resource_defs={
'a': resource_a,
'c_alias': resource_c_alias
})
],
dependencies={
SolidInvocation('consumes_resource_a'): {},
SolidInvocation(
'consumes_resource_c',
alias='consumes_resource_c',
resource_mapper_fn=_resource_mapper_fn,
): {},
},
)
assert execute_pipeline(pipeline_with_resource_mapper).success
def test_cycle_detect():
@lambda_solid
def return_one():
return 1
@lambda_solid
def add(a, b):
return a + b
with pytest.raises(DagsterInvalidDefinitionError,
match="Circular dependencies exist"):
PipelineDefinition(
solid_defs=[return_one, add],
name="test",
dependencies={
SolidInvocation("add", alias="first"): {
"a": DependencyDefinition("return_one"),
"b": DependencyDefinition("second"),
},
SolidInvocation("add", alias="second"): {
"a": DependencyDefinition("first"),
"b": DependencyDefinition("return_one"),
},
},
)
with pytest.raises(DagsterInvalidDefinitionError,
match="Circular dependencies exist"):
CompositeSolidDefinition(
name="circletron",
solid_defs=[return_one, add],
dependencies={
SolidInvocation("add", alias="first"): {
"a": DependencyDefinition("return_one"),
"b": DependencyDefinition("second"),
},
SolidInvocation("add", alias="second"): {
"a": DependencyDefinition("first"),
"b": DependencyDefinition("return_one"),
},
},
)
def test_cycle_detect():
@lambda_solid
def return_one():
return 1
@lambda_solid
def add(a, b):
return a + b
with pytest.raises(DagsterInvalidDefinitionError,
match="Circular dependencies exist"):
PipelineDefinition(
solid_defs=[return_one, add],
dependencies={
SolidInvocation('add', alias='first'): {
'a': DependencyDefinition('return_one'),
'b': DependencyDefinition('second'),
},
SolidInvocation('add', alias='second'): {
'a': DependencyDefinition('first'),
'b': DependencyDefinition('return_one'),
},
},
)
with pytest.raises(DagsterInvalidDefinitionError,
match="Circular dependencies exist"):
CompositeSolidDefinition(
name='circletron',
solid_defs=[return_one, add],
dependencies={
SolidInvocation('add', alias='first'): {
'a': DependencyDefinition('return_one'),
'b': DependencyDefinition('second'),
},
SolidInvocation('add', alias='second'): {
'a': DependencyDefinition('first'),
'b': DependencyDefinition('return_one'),
},
},
)
