def test_create_single_solid_pipeline_with_alias(): a_source = define_stub_solid('A_source', [input_set('A_input')]) stub_solid = define_stub_solid('stub', [{'a_key': 'stubbed_thing'}]) single_solid_pipeline = PipelineDefinition.create_single_solid_pipeline( PipelineDefinition( solids=[a_source, create_root_solid('A')], dependencies={ SolidInstance('A', alias='aliased'): { 'A_input': DependencyDefinition(a_source.name) }, }, ), 'aliased', { 'aliased': { 'A_input': stub_solid, }, }, ) result = execute_pipeline(single_solid_pipeline) assert result.success expected = [{'a_key': 'stubbed_thing'}, {'A': 'transform_called'}] assert result.result_for_solid('aliased').transformed_value() == expected
def test_two_root_solid_pipeline_with_empty_dependency_definition(): stub_solid_a = define_stub_solid('stub_a', [{'a key': 'a value'}]) stub_solid_b = define_stub_solid('stub_b', [{'a key': 'a value'}]) single_solid_pipeline = PipelineDefinition(solids=[stub_solid_a, stub_solid_b], dependencies={}) result = execute_pipeline(single_solid_pipeline) assert result.success
def test_two_root_solid_pipeline_with_partial_dependency_definition(): stub_solid_a = define_stub_solid("stub_a", [{"a key": "a value"}]) stub_solid_b = define_stub_solid("stub_b", [{"a key": "a value"}]) single_dep_pipe = PipelineDefinition( solid_defs=[stub_solid_a, stub_solid_b], dependencies={"stub_a": {}}) assert execute_pipeline(single_dep_pipe).success
def test_two_root_solid_pipeline_with_partial_dependency_definition(): stub_solid_a = define_stub_solid('stub_a', [{'a key': 'a value'}]) stub_solid_b = define_stub_solid('stub_b', [{'a key': 'a value'}]) single_dep_pipe = PipelineDefinition( solid_defs=[stub_solid_a, stub_solid_b], dependencies={'stub_a': {}}) assert execute_pipeline(single_dep_pipe).success
def test_two_root_solid_pipeline_with_empty_dependency_definition(): stub_solid_a = define_stub_solid('stub_a', [{'a key': 'a value'}]) stub_solid_b = define_stub_solid('stub_b', [{'a key': 'a value'}]) @pipeline def pipe(): stub_solid_a() stub_solid_b() assert execute_pipeline(pipe).success
def test_two_root_solid_pipeline_with_empty_dependency_definition(): stub_solid_a = define_stub_solid("stub_a", [{"a key": "a value"}]) stub_solid_b = define_stub_solid("stub_b", [{"a key": "a value"}]) @pipeline def pipe(): stub_solid_a() stub_solid_b() assert execute_pipeline(pipe).success
def create_diamond_solids(): a_source = define_stub_solid("A_source", [input_set("A_input")]) node_a = create_root_solid("A") node_b = create_solid_with_deps("B", node_a) node_c = create_solid_with_deps("C", node_a) node_d = create_solid_with_deps("D", node_b, node_c) return [node_d, node_c, node_b, node_a, a_source]
def create_sum_sq_pipeline(context, expr, extra_solids=None, extra_deps=None): check.inst_param(expr, 'expr', DagsterSqlTableExpression) expr_solid = define_stub_solid('expr', expr) sum_solid = create_sum_table_solid() sum_sq_solid = create_sql_solid( name='sum_sq_table', inputs=[InputDefinition(sum_solid.name)], sql_text='SELECT num1, num2, sum, sum * sum as sum_sq from {sum_table}', ) dependencies = { sum_solid.name: { 'num_table': DependencyDefinition('expr') }, sum_sq_solid.name: { sum_solid.name: DependencyDefinition(sum_solid.name) }, } if extra_deps: dependencies.update(extra_deps) return pipeline_test_def( solids=[expr_solid, sum_solid, sum_sq_solid] + (extra_solids if extra_solids else []), context=context, dependencies=dependencies, )
def test_composite_with_no_output_mappings(): a_source = define_stub_solid('A_source', [input_set('A_input')]) node_a = create_root_solid('A') node_b = create_solid_with_deps('B', node_a) node_c = create_solid_with_deps('C', node_a) node_d = create_solid_with_deps('D', node_b, node_c) @composite_solid def diamond_composite(): a = node_a(a_source()) node_d(B=node_b(a), C=node_c(a)) res = execute_solid(diamond_composite) assert res.success assert res.output_values == {} with pytest.raises( DagsterInvariantViolationError, match=re.escape( 'Output \'result\' not defined in composite solid \'diamond_composite\': no output ' 'mappings were defined. If you were expecting this output to be present, you may be ' 'missing an output_mapping from an inner solid to its enclosing composite solid.' ), ): _ = res.output_value() assert len(res.solid_result_list) == 5
def create_diamond_solids(): a_source = define_stub_solid('A_source', [input_set('A_input')]) node_a = create_root_solid('A') node_b = create_solid_with_deps('B', node_a) node_c = create_solid_with_deps('C', node_a) node_d = create_solid_with_deps('D', node_b, node_c) return [node_d, node_c, node_b, node_a, a_source]
def test_execute_aliased_solid_in_diamond(): a_source = define_stub_solid('A_source', [input_set('A_input')]) pipeline_def = PipelineDefinition( name='aliased_pipeline', solids=[a_source, create_root_solid('A')], dependencies={ SolidInstance('A', alias='aliased'): { 'A_input': DependencyDefinition(a_source.name) } }, ) solid_result = execute_solid(pipeline_def, 'aliased', inputs={'A_input': [{ 'a key': 'a value' }]}) assert solid_result.success assert solid_result.transformed_value() == [ { 'a key': 'a value' }, { 'aliased': 'transform_called' }, ]
def test_singleton_pipeline(): stub_solid = define_stub_solid('stub', [{'a key': 'a value'}]) single_solid_pipeline = PipelineDefinition(solid_defs=[stub_solid], dependencies={}) result = execute_pipeline(single_solid_pipeline) assert result.success
def test_pipeline_init_failure(): stub_solid = define_stub_solid('stub', None) env_config = {} def failing_resource_fn(*args, **kwargs): raise Exception() pipeline_def = PipelineDefinition( [stub_solid], 'failing_init_pipeline', mode_definitions=[ ModeDefinition( resources={ 'failing': ResourceDefinition( resource_fn=failing_resource_fn) }) ], ) result = execute_pipeline( pipeline_def, environment_dict=env_config, run_config=RunConfig(executor_config=InProcessExecutorConfig( raise_on_error=False)), ) assert result.success is False assert len(result.event_list) == 1 event = result.event_list[0] assert event.event_type_value == 'PIPELINE_INIT_FAILURE' assert event.pipeline_init_failure_data
def define_pandas_input_transform_test_pipeline(): in_df = pd.DataFrame({'num': [3, 5, 7]}) return PipelineDefinition( name='input_transform_test_pipeline', solids=[define_stub_solid('load_df', in_df), define_pandas_input_transform_test_solid()], dependencies={'pandas_input_transform_test': {'df': DependencyDefinition('load_df')}}, )
def test_pipeline_init_failure(): stub_solid = define_stub_solid('stub', None) env_config = {} def failing_resource_fn(*args, **kwargs): raise Exception() @pipeline(mode_defs=[ ModeDefinition( resource_defs={ 'failing': ResourceDefinition(resource_fn=failing_resource_fn) }) ]) def failing_init_pipeline(): stub_solid() result = execute_pipeline(failing_init_pipeline, environment_dict=dict(env_config), raise_on_error=False) assert result.success is False assert len(result.event_list) == 1 event = result.event_list[0] assert event.event_type_value == 'PIPELINE_INIT_FAILURE' assert event.pipeline_init_failure_data
def test_singleton_pipeline(): stub_solid = define_stub_solid('stub', [{'a key': 'a value'}]) @pipeline def single_solid_pipeline(): stub_solid() assert execute_pipeline(single_solid_pipeline).success
def test_create_pipeline_with_bad_solids_list(): with pytest.raises( DagsterInvalidDefinitionError, match='"solids" arg to pipeline "a_pipeline" is not a list. Got', ): PipelineDefinition( name='a_pipeline', solid_defs=define_stub_solid('stub', [{'a key': 'a value'}]) )
def test_singleton_pipeline(): stub_solid = define_stub_solid("stub", [{"a key": "a value"}]) @pipeline def single_solid_pipeline(): stub_solid() assert execute_pipeline(single_solid_pipeline).success
def test_create_pipeline_with_bad_solids_list(): with pytest.raises( DagsterInvalidDefinitionError, match='"solids" arg to pipeline "a_pipeline" is not a list. Got', ): PipelineDefinition(name="a_pipeline", solid_defs=define_stub_solid( "stub", [{ "a key": "a value" }]))
def define_pandas_source_test_pipeline(): return PipelineDefinition( name='input_transform_test_pipeline', solids=[ define_stub_solid( 'load_num_csv', pd.read_csv(script_relative_path('data/num_prod.csv')) ), define_pandas_source_test_solid(), ], dependencies={'pandas_source_test': {'df': DependencyDefinition('load_num_csv')}}, )
def test_composite_io_mapping(): a_source = define_stub_solid('A_source', [input_set('A_input')]) node_a = create_root_solid('A') node_b = create_solid_with_deps('B', node_a) node_c = create_solid_with_deps('C', node_b) comp_a_inner = CompositeSolidDefinition( name='comp_a_inner', solid_defs=[a_source, node_a], dependencies={'A': { 'A_input': DependencyDefinition('A_source') }}, output_mappings=[OutputDefinition().mapping_from('A')], ) comp_a_outer = CompositeSolidDefinition( name='comp_a_outer', solid_defs=[comp_a_inner], output_mappings=[OutputDefinition().mapping_from('comp_a_inner')], ) comp_bc_inner = CompositeSolidDefinition( name='comp_bc_inner', solid_defs=[node_b, node_c], dependencies={'C': { 'B': DependencyDefinition('B') }}, input_mappings=[ InputDefinition(name='inner_B_in').mapping_to(solid_name='B', input_name='A') ], ) comp_bc_outer = CompositeSolidDefinition( name='comp_bc_outer', solid_defs=[comp_bc_inner], dependencies={}, input_mappings=[ InputDefinition(name='outer_B_in').mapping_to( solid_name='comp_bc_inner', input_name='inner_B_in') ], ) result = execute_pipeline( PipelineDefinition( name='wrapped_io', solid_defs=[comp_a_outer, comp_bc_outer], dependencies={ 'comp_bc_outer': { 'outer_B_in': DependencyDefinition('comp_a_outer') } }, )) assert result.success
def test_composite_basic_execution(): a_source = define_stub_solid('A_source', [input_set('A_input')]) node_a = create_root_solid('A') node_b = create_solid_with_deps('B', node_a) node_c = create_solid_with_deps('C', node_a) node_d = create_solid_with_deps('D', node_b, node_c) @composite_solid def diamond_composite(): a = node_a(a_source()) node_d(B=node_b(a), C=node_c(a)) @pipeline def test_pipeline_single(): diamond_composite() result = execute_pipeline(test_pipeline_single) assert result.success @pipeline def test_pipeline_double(): diamond_composite.alias('D1')() diamond_composite.alias('D2')() result = execute_pipeline(test_pipeline_double) assert result.success @composite_solid def wrapped_composite(): diamond_composite() @pipeline def test_pipeline_mixed(): diamond_composite() wrapped_composite() result = execute_pipeline(test_pipeline_mixed) assert result.success @composite_solid def empty(): pass @pipeline def test_pipeline_empty(): empty() result = execute_pipeline(test_pipeline_empty) assert result.success nested_pipeline = nesting_composite_pipeline(6, 2) result = execute_pipeline(nested_pipeline) assert result.success
def test_singleton_pipeline(): stub_solid = define_stub_solid("stub", [{"a key": "a value"}]) # will fail if any warning is emitted with warnings.catch_warnings(): warnings.simplefilter("error") @pipeline def single_solid_pipeline(): stub_solid() assert execute_pipeline(single_solid_pipeline).success
def test_wrong_input_value(): @lambda_solid(input_defs=[InputDefinition('foo', dagster_pd.DataFrame)]) def test_wrong_input(foo): return foo pass_solid = define_stub_solid('pass_solid', 'not a dataframe') @pipeline def test_pipeline(): test_wrong_input(pass_solid()) with pytest.raises(DagsterTypeCheckError): execute_pipeline(test_pipeline)
def test_execute_aliased_solid_in_diamond(): a_source = define_stub_solid('A_source', [input_set('A_input')]) @pipeline def aliased_pipeline(): create_root_solid('A').alias('aliased')(a_source()) solid_result = execute_solid_within_pipeline( aliased_pipeline, 'aliased', inputs={'A_input': [{'a key': 'a value'}]} ) assert solid_result.success assert solid_result.output_value() == [{'a key': 'a value'}, {'aliased': 'compute_called'}]
def test_wrong_input_value(): @lambda_solid(input_defs=[InputDefinition("foo", dagster_pd.DataFrame)]) def test_wrong_input(foo): return foo pass_solid = define_stub_solid("pass_solid", "not a dataframe") @pipeline def test_pipeline(): test_wrong_input(pass_solid()) with pytest.raises(DagsterTypeCheckDidNotPass): execute_pipeline(test_pipeline)
def test_execute_aliased_solid_in_diamond(): a_source = define_stub_solid("A_source", [input_set("A_input")]) @pipeline def aliased_pipeline(): create_root_solid("A").alias("aliased")(a_source()) solid_result = execute_solid_within_pipeline( aliased_pipeline, "aliased", inputs={"A_input": [{"a key": "a value"}]} ) assert solid_result.success assert solid_result.output_value() == [{"a key": "a value"}, {"aliased": "compute_called"}]
def test_composite_basic_execution(): a_source = define_stub_solid('A_source', [input_set('A_input')]) node_a = create_root_solid('A') node_b = create_solid_with_deps('B', node_a) node_c = create_solid_with_deps('C', node_a) node_d = create_solid_with_deps('D', node_b, node_c) diamond_composite = CompositeSolidDefinition( name='diamond_composite', solid_defs=[a_source, node_a, node_b, node_c, node_d], dependencies={ 'A': { 'A_input': DependencyDefinition('A_source') }, 'B': { 'A': DependencyDefinition('A') }, 'C': { 'A': DependencyDefinition('A') }, 'D': { 'B': DependencyDefinition('B'), 'C': DependencyDefinition('C') }, }, ) result = execute_pipeline( PipelineDefinition(solid_defs=[diamond_composite])) assert result.success result = execute_pipeline( PipelineDefinition( solid_defs=[diamond_composite], dependencies={ SolidInvocation('diamond_composite', alias='D1'): {}, SolidInvocation('diamond_composite', alias='D2'): {}, }, )) assert result.success wrapped_composite = CompositeSolidDefinition( name='wrapped_composite', solid_defs=[diamond_composite]) result = execute_pipeline( PipelineDefinition(solid_defs=[diamond_composite, wrapped_composite])) assert result.success empty_composite = CompositeSolidDefinition(name='empty', solid_defs=[]) result = execute_pipeline(PipelineDefinition(solid_defs=[empty_composite])) assert result.success
def test_composite_basic_execution(composition_decorator): a_source = define_stub_solid("A_source", [input_set("A_input")]) node_a = create_root_solid("A") node_b = create_solid_with_deps("B", node_a) node_c = create_solid_with_deps("C", node_a) node_d = create_solid_with_deps("D", node_b, node_c) @composition_decorator def diamond_composite(): a = node_a(a_source()) node_d(B=node_b(a), C=node_c(a)) res = execute_solid(diamond_composite) assert res.success @pipeline def test_pipeline_double(): diamond_composite.alias("D1")() diamond_composite.alias("D2")() result = execute_pipeline(test_pipeline_double) assert result.success @composition_decorator def wrapped_composite(): diamond_composite() @pipeline def test_pipeline_mixed(): diamond_composite() wrapped_composite() result = execute_pipeline(test_pipeline_mixed) assert result.success @composition_decorator def empty(): pass @pipeline def test_pipeline_empty(): empty() result = execute_pipeline(test_pipeline_empty) assert result.success nested_pipeline = nesting_composite_pipeline(6, 2) result = execute_pipeline(nested_pipeline) assert result.success
def test_wrong_input_value(): @lambda_solid(name="test_wrong_input", inputs=[InputDefinition('foo', dagster_pd.DataFrame)]) def df_solid(foo): return foo pass_solid = define_stub_solid('pass_solid', 'not a dataframe') pipeline = PipelineDefinition( solids=[pass_solid, df_solid], dependencies={'test_wrong_input': {'foo': DependencyDefinition('pass_solid')}}, ) with pytest.raises(DagsterTypeError): execute_pipeline(pipeline)