def test_pipeline_streaming_multiple_outputs():
events = []
@solid(
outputs=[OutputDefinition(Int, 'one'),
OutputDefinition(Int, 'two')])
def push_one_two(_context):
events.append(1)
yield Output(1, 'one')
events.append(2)
yield Output(2, 'two')
pipeline_def = PipelineDefinition(
name='test_streaming_iterator_multiple_outputs',
solid_defs=[push_one_two])
step_event_iterator = step_output_event_filter(
execute_pipeline_iterator(pipeline_def))
one_output_step_event = next(step_event_iterator)
assert one_output_step_event.is_successful_output
assert one_output_step_event.step_output_data.value_repr == '1'
assert one_output_step_event.step_output_data.output_name == 'one'
assert events == [1]
two_output_step_event = next(step_event_iterator)
assert two_output_step_event.is_successful_output
assert two_output_step_event.step_output_data.value_repr == '2'
assert two_output_step_event.step_output_data.output_name == 'two'
assert events == [1, 2]
def test_pipeline_streaming_iterator():
events = []
@lambda_solid
def push_one():
events.append(1)
return 1
@lambda_solid
def add_one(num):
events.append(num + 1)
return num + 1
pipeline_def = PipelineDefinition(
name='test_streaming_iterator',
solid_defs=[push_one, add_one],
dependencies={'add_one': {
'num': DependencyDefinition('push_one')
}},
)
step_event_iterator = step_output_event_filter(
execute_pipeline_iterator(pipeline_def))
push_one_step_event = next(step_event_iterator)
assert push_one_step_event.is_successful_output
assert push_one_step_event.step_output_data.value_repr == '1'
assert events == [1]
add_one_step_event = next(step_event_iterator)
assert add_one_step_event.is_successful_output
assert add_one_step_event.step_output_data.value_repr == '2'
assert events == [1, 2]
def test_pipeline_streaming_iterator():
events = []
@lambda_solid
def push_one():
events.append(1)
return 1
@lambda_solid
def add_one(num):
events.append(num + 1)
return num + 1
@pipeline
def test_streaming_iterator():
add_one(push_one())
step_event_iterator = step_output_event_filter(
execute_pipeline_iterator(test_streaming_iterator))
push_one_step_event = next(step_event_iterator)
assert push_one_step_event.is_successful_output
assert events == [1]
add_one_step_event = next(step_event_iterator)
assert add_one_step_event.is_successful_output
assert events == [1, 2]
def test_pipeline_subset():
@lambda_solid
def return_one():
return 1
@lambda_solid
def add_one(num):
return num + 1
pipeline_def = PipelineDefinition(
solid_defs=[return_one, add_one],
dependencies={'add_one': {'num': DependencyDefinition('return_one')}},
)
pipeline_result = execute_pipeline(pipeline_def)
assert pipeline_result.success
assert pipeline_result.result_for_solid('add_one').output_value() == 2
env_config = {'solids': {'add_one': {'inputs': {'num': {'value': 3}}}}}
subset_result = execute_pipeline(
pipeline_def.build_sub_pipeline(['add_one']), environment_dict=env_config
)
assert subset_result.success
assert len(subset_result.solid_result_list) == 1
assert subset_result.result_for_solid('add_one').output_value() == 4
events = execute_pipeline_iterator(
pipeline_def.build_sub_pipeline(['add_one']), environment_dict=env_config
)
for step_event in step_output_event_filter(events):
assert step_event.is_step_success
def test_pipeline_subset_with_multi_dependency():
@lambda_solid
def return_one():
return 1
@lambda_solid
def return_two():
return 2
@lambda_solid(inputs=[InputDefinition('dep', Nothing)])
def noop():
return 3
pipeline_def = PipelineDefinition(
solid_defs=[return_one, return_two, noop],
dependencies={
'noop': {
'dep':
MultiDependencyDefinition([
DependencyDefinition('return_one'),
DependencyDefinition('return_two')
])
}
},
)
pipeline_result = execute_pipeline(pipeline_def)
assert pipeline_result.success
assert pipeline_result.result_for_solid('noop').result_value() == 3
subset_result = execute_pipeline(pipeline_def.build_sub_pipeline(['noop']))
assert subset_result.success
assert len(subset_result.solid_result_list) == 1
assert pipeline_result.result_for_solid('noop').result_value() == 3
events = execute_pipeline_iterator(
pipeline_def.build_sub_pipeline(['noop']))
for step_event in step_output_event_filter(events):
assert step_event.is_step_success
subset_result = execute_pipeline(
pipeline_def.build_sub_pipeline(['return_one', 'return_two', 'noop']))
assert subset_result.success
assert len(subset_result.solid_result_list) == 3
assert pipeline_result.result_for_solid('noop').result_value() == 3
def test_pipeline_name_threaded_through_context():
name = 'foobar'
@solid()
def assert_name_transform(context):
assert context.pipeline_def.name == name
result = execute_pipeline(
PipelineDefinition(name="foobar", solid_defs=[assert_name_transform]))
assert result.success
for step_event in step_output_event_filter(
execute_pipeline_iterator(
PipelineDefinition(name="foobar",
solid_defs=[assert_name_transform]), {})):
assert step_event.is_step_success
