def test_a_plus_b_final_subdag():
pipeline = PipelineDefinition.create_sub_pipeline(
define_part_fourteen_step_one(),
['a_plus_b', 'final'],
['final'],
injected_solids={
'a_plus_b': {
'num1': define_stub_solid('stub_a', 2),
'num2': define_stub_solid('stub_b', 4),
},
'final': {
'num2': define_stub_solid('stub_c_plus_d', 6),
}
},
)
pipeline_result = execute_pipeline(pipeline)
assert pipeline_result.result_for_solid(
'a_plus_b').transformed_value() == 6
assert pipeline_result.result_for_solid('final').transformed_value() == 36
def test_pandas_output_intermediate_csv_files():
with get_temp_file_names(2) as temp_tuple:
sum_file, mult_file = temp_tuple # pylint: disable=E0632
write_sum_table = dagster_pd.to_csv_solid('write_sum_table')
write_mult_table = dagster_pd.to_csv_solid('write_mult_table')
pipeline = create_diamond_pipeline(
extra_solids=[write_sum_table, write_mult_table],
extra_dependencies={
write_sum_table.name: {
'df': DependencyDefinition('sum_table'),
},
write_mult_table.name: {
'df': DependencyDefinition('mult_table'),
}
})
environment = get_num_csv_environment({
'load_csv':
config.Solid({
'path': script_relative_path('num.csv'),
}),
write_sum_table.name:
config.Solid({'path': sum_file}),
write_mult_table.name:
config.Solid({'path': mult_file}),
})
subgraph_one_result = execute_pipeline(pipeline,
environment=environment)
assert len(subgraph_one_result.result_list) == 5
expected_sum = {
'num1': [1, 3],
'num2': [2, 4],
'sum': [3, 7],
}
assert pd.read_csv(sum_file).to_dict('list') == expected_sum
sum_table_result = subgraph_one_result.result_for_solid('sum_table')
assert sum_table_result.transformed_value().to_dict(
'list') == expected_sum
expected_mult = {
'num1': [1, 3],
'num2': [2, 4],
'mult': [2, 12],
}
assert pd.read_csv(mult_file).to_dict('list') == expected_mult
mult_table_result = subgraph_one_result.result_for_solid('mult_table')
assert mult_table_result.transformed_value().to_dict(
'list') == expected_mult
injected_solids = {
'sum_mult_table': {
'sum_table': dagster_pd.load_csv_solid('load_sum_table'),
'mult_table': dagster_pd.load_csv_solid('load_mult_table'),
}
}
pipeline_result = execute_pipeline(
PipelineDefinition.create_sub_pipeline(
pipeline,
['sum_mult_table'],
['sum_mult_table'],
injected_solids,
),
environment=config.Environment(solids={
'load_sum_table':
config.Solid({'path': sum_file}, ),
'load_mult_table':
config.Solid({'path': mult_file}, ),
}, ),
)
assert pipeline_result.success
subgraph_two_result_list = pipeline_result.result_list
assert len(subgraph_two_result_list) == 3
output_df = pipeline_result.result_for_solid(
'sum_mult_table').transformed_value()
assert output_df.to_dict('list') == {
'num1': [1, 3],
'num2': [2, 4],
'sum': [3, 7],
'mult': [2, 12],
'sum_mult': [6, 84],
}
