def test_pipeline_streaming_multiple_outputs(): events = [] @solid( outputs=[OutputDefinition(Int, 'one'), OutputDefinition(Int, 'two')]) def push_one_two(_context): events.append(1) yield Output(1, 'one') events.append(2) yield Output(2, 'two') pipeline_def = PipelineDefinition( name='test_streaming_iterator_multiple_outputs', solid_defs=[push_one_two]) step_event_iterator = step_output_event_filter( execute_pipeline_iterator(pipeline_def)) one_output_step_event = next(step_event_iterator) assert one_output_step_event.is_successful_output assert one_output_step_event.step_output_data.value_repr == '1' assert one_output_step_event.step_output_data.output_name == 'one' assert events == [1] two_output_step_event = next(step_event_iterator) assert two_output_step_event.is_successful_output assert two_output_step_event.step_output_data.value_repr == '2' assert two_output_step_event.step_output_data.output_name == 'two' assert events == [1, 2]
def test_pipeline_streaming_iterator(): events = [] @lambda_solid def push_one(): events.append(1) return 1 @lambda_solid def add_one(num): events.append(num + 1) return num + 1 pipeline_def = PipelineDefinition( name='test_streaming_iterator', solid_defs=[push_one, add_one], dependencies={'add_one': { 'num': DependencyDefinition('push_one') }}, ) step_event_iterator = step_output_event_filter( execute_pipeline_iterator(pipeline_def)) push_one_step_event = next(step_event_iterator) assert push_one_step_event.is_successful_output assert push_one_step_event.step_output_data.value_repr == '1' assert events == [1] add_one_step_event = next(step_event_iterator) assert add_one_step_event.is_successful_output assert add_one_step_event.step_output_data.value_repr == '2' assert events == [1, 2]
def test_pipeline_streaming_iterator(): events = [] @lambda_solid def push_one(): events.append(1) return 1 @lambda_solid def add_one(num): events.append(num + 1) return num + 1 @pipeline def test_streaming_iterator(): add_one(push_one()) step_event_iterator = step_output_event_filter( execute_pipeline_iterator(test_streaming_iterator)) push_one_step_event = next(step_event_iterator) assert push_one_step_event.is_successful_output assert events == [1] add_one_step_event = next(step_event_iterator) assert add_one_step_event.is_successful_output assert events == [1, 2]
def test_pipeline_subset(): @lambda_solid def return_one(): return 1 @lambda_solid def add_one(num): return num + 1 pipeline_def = PipelineDefinition( solid_defs=[return_one, add_one], dependencies={'add_one': {'num': DependencyDefinition('return_one')}}, ) pipeline_result = execute_pipeline(pipeline_def) assert pipeline_result.success assert pipeline_result.result_for_solid('add_one').output_value() == 2 env_config = {'solids': {'add_one': {'inputs': {'num': {'value': 3}}}}} subset_result = execute_pipeline( pipeline_def.build_sub_pipeline(['add_one']), environment_dict=env_config ) assert subset_result.success assert len(subset_result.solid_result_list) == 1 assert subset_result.result_for_solid('add_one').output_value() == 4 events = execute_pipeline_iterator( pipeline_def.build_sub_pipeline(['add_one']), environment_dict=env_config ) for step_event in step_output_event_filter(events): assert step_event.is_step_success
def test_pipeline_subset_with_multi_dependency(): @lambda_solid def return_one(): return 1 @lambda_solid def return_two(): return 2 @lambda_solid(inputs=[InputDefinition('dep', Nothing)]) def noop(): return 3 pipeline_def = PipelineDefinition( solid_defs=[return_one, return_two, noop], dependencies={ 'noop': { 'dep': MultiDependencyDefinition([ DependencyDefinition('return_one'), DependencyDefinition('return_two') ]) } }, ) pipeline_result = execute_pipeline(pipeline_def) assert pipeline_result.success assert pipeline_result.result_for_solid('noop').result_value() == 3 subset_result = execute_pipeline(pipeline_def.build_sub_pipeline(['noop'])) assert subset_result.success assert len(subset_result.solid_result_list) == 1 assert pipeline_result.result_for_solid('noop').result_value() == 3 events = execute_pipeline_iterator( pipeline_def.build_sub_pipeline(['noop'])) for step_event in step_output_event_filter(events): assert step_event.is_step_success subset_result = execute_pipeline( pipeline_def.build_sub_pipeline(['return_one', 'return_two', 'noop'])) assert subset_result.success assert len(subset_result.solid_result_list) == 3 assert pipeline_result.result_for_solid('noop').result_value() == 3
def test_pipeline_name_threaded_through_context(): name = 'foobar' @solid() def assert_name_transform(context): assert context.pipeline_def.name == name result = execute_pipeline( PipelineDefinition(name="foobar", solid_defs=[assert_name_transform])) assert result.success for step_event in step_output_event_filter( execute_pipeline_iterator( PipelineDefinition(name="foobar", solid_defs=[assert_name_transform]), {})): assert step_event.is_step_success