def test_a_plus_b_final_subdag(): pipeline = PipelineDefinition.create_sub_pipeline( define_part_fourteen_step_one(), ['a_plus_b', 'final'], ['final'], injected_solids={ 'a_plus_b': { 'num1': define_stub_solid('stub_a', 2), 'num2': define_stub_solid('stub_b', 4), }, 'final': { 'num2': define_stub_solid('stub_c_plus_d', 6), } }, ) pipeline_result = execute_pipeline(pipeline) assert pipeline_result.result_for_solid( 'a_plus_b').transformed_value() == 6 assert pipeline_result.result_for_solid('final').transformed_value() == 36
def test_pandas_output_intermediate_csv_files(): with get_temp_file_names(2) as temp_tuple: sum_file, mult_file = temp_tuple # pylint: disable=E0632 write_sum_table = dagster_pd.to_csv_solid('write_sum_table') write_mult_table = dagster_pd.to_csv_solid('write_mult_table') pipeline = create_diamond_pipeline( extra_solids=[write_sum_table, write_mult_table], extra_dependencies={ write_sum_table.name: { 'df': DependencyDefinition('sum_table'), }, write_mult_table.name: { 'df': DependencyDefinition('mult_table'), } }) environment = get_num_csv_environment({ 'load_csv': config.Solid({ 'path': script_relative_path('num.csv'), }), write_sum_table.name: config.Solid({'path': sum_file}), write_mult_table.name: config.Solid({'path': mult_file}), }) subgraph_one_result = execute_pipeline(pipeline, environment=environment) assert len(subgraph_one_result.result_list) == 5 expected_sum = { 'num1': [1, 3], 'num2': [2, 4], 'sum': [3, 7], } assert pd.read_csv(sum_file).to_dict('list') == expected_sum sum_table_result = subgraph_one_result.result_for_solid('sum_table') assert sum_table_result.transformed_value().to_dict( 'list') == expected_sum expected_mult = { 'num1': [1, 3], 'num2': [2, 4], 'mult': [2, 12], } assert pd.read_csv(mult_file).to_dict('list') == expected_mult mult_table_result = subgraph_one_result.result_for_solid('mult_table') assert mult_table_result.transformed_value().to_dict( 'list') == expected_mult injected_solids = { 'sum_mult_table': { 'sum_table': dagster_pd.load_csv_solid('load_sum_table'), 'mult_table': dagster_pd.load_csv_solid('load_mult_table'), } } pipeline_result = execute_pipeline( PipelineDefinition.create_sub_pipeline( pipeline, ['sum_mult_table'], ['sum_mult_table'], injected_solids, ), environment=config.Environment(solids={ 'load_sum_table': config.Solid({'path': sum_file}, ), 'load_mult_table': config.Solid({'path': mult_file}, ), }, ), ) assert pipeline_result.success subgraph_two_result_list = pipeline_result.result_list assert len(subgraph_two_result_list) == 3 output_df = pipeline_result.result_for_solid( 'sum_mult_table').transformed_value() assert output_df.to_dict('list') == { 'num1': [1, 3], 'num2': [2, 4], 'sum': [3, 7], 'mult': [2, 12], 'sum_mult': [6, 84], }