def test_a_plus_b_final_subdag():
    pipeline = PipelineDefinition.create_sub_pipeline(
        define_part_fourteen_step_one(),
        ['a_plus_b', 'final'],
        ['final'],
        injected_solids={
            'a_plus_b': {
                'num1': define_stub_solid('stub_a', 2),
                'num2': define_stub_solid('stub_b', 4),
            },
            'final': {
                'num2': define_stub_solid('stub_c_plus_d', 6),
            }
        },
    )

    pipeline_result = execute_pipeline(pipeline)

    assert pipeline_result.result_for_solid(
        'a_plus_b').transformed_value() == 6
    assert pipeline_result.result_for_solid('final').transformed_value() == 36
Esempio n. 2
0
def test_pandas_output_intermediate_csv_files():

    with get_temp_file_names(2) as temp_tuple:
        sum_file, mult_file = temp_tuple  # pylint: disable=E0632

        write_sum_table = dagster_pd.to_csv_solid('write_sum_table')
        write_mult_table = dagster_pd.to_csv_solid('write_mult_table')

        pipeline = create_diamond_pipeline(
            extra_solids=[write_sum_table, write_mult_table],
            extra_dependencies={
                write_sum_table.name: {
                    'df': DependencyDefinition('sum_table'),
                },
                write_mult_table.name: {
                    'df': DependencyDefinition('mult_table'),
                }
            })

        environment = get_num_csv_environment({
            'load_csv':
            config.Solid({
                'path': script_relative_path('num.csv'),
            }),
            write_sum_table.name:
            config.Solid({'path': sum_file}),
            write_mult_table.name:
            config.Solid({'path': mult_file}),
        })

        subgraph_one_result = execute_pipeline(pipeline,
                                               environment=environment)

        assert len(subgraph_one_result.result_list) == 5

        expected_sum = {
            'num1': [1, 3],
            'num2': [2, 4],
            'sum': [3, 7],
        }

        assert pd.read_csv(sum_file).to_dict('list') == expected_sum
        sum_table_result = subgraph_one_result.result_for_solid('sum_table')
        assert sum_table_result.transformed_value().to_dict(
            'list') == expected_sum

        expected_mult = {
            'num1': [1, 3],
            'num2': [2, 4],
            'mult': [2, 12],
        }
        assert pd.read_csv(mult_file).to_dict('list') == expected_mult
        mult_table_result = subgraph_one_result.result_for_solid('mult_table')
        assert mult_table_result.transformed_value().to_dict(
            'list') == expected_mult

        injected_solids = {
            'sum_mult_table': {
                'sum_table': dagster_pd.load_csv_solid('load_sum_table'),
                'mult_table': dagster_pd.load_csv_solid('load_mult_table'),
            }
        }

        pipeline_result = execute_pipeline(
            PipelineDefinition.create_sub_pipeline(
                pipeline,
                ['sum_mult_table'],
                ['sum_mult_table'],
                injected_solids,
            ),
            environment=config.Environment(solids={
                'load_sum_table':
                config.Solid({'path': sum_file}, ),
                'load_mult_table':
                config.Solid({'path': mult_file}, ),
            }, ),
        )

        assert pipeline_result.success

        subgraph_two_result_list = pipeline_result.result_list

        assert len(subgraph_two_result_list) == 3
        output_df = pipeline_result.result_for_solid(
            'sum_mult_table').transformed_value()
        assert output_df.to_dict('list') == {
            'num1': [1, 3],
            'num2': [2, 4],
            'sum': [3, 7],
            'mult': [2, 12],
            'sum_mult': [6, 84],
        }