def test_execute_dep_solid_different_input_name(): pass_to_first = define_pass_value_solid('pass_to_first') first_solid = single_output_solid( 'first_solid', input_defs=[InputDefinition(name='a_thing')], compute_fn=lambda context, inputs: inputs['a_thing'] + inputs['a_thing'], output_def=OutputDefinition(), ) second_solid = single_output_solid( 'second_solid', input_defs=[InputDefinition(name='an_input')], compute_fn=lambda context, inputs: inputs['an_input'] + inputs['an_input'], output_def=OutputDefinition(), ) @pipeline def pipe(): second_solid(first_solid(pass_to_first())) result = execute_pipeline( pipe, environment_dict={'solids': {'pass_to_first': {'config': {'value': 'bar'}}}} ) assert result.success assert len(result.solid_result_list) == 3 assert result.result_for_solid('pass_to_first').output_value() == 'bar' assert result.result_for_solid('first_solid').output_value() == 'barbar' assert result.result_for_solid('second_solid').output_value() == 'barbarbarbar'
def test_execute_solid_with_dep_only_inputs_no_api(): did_run_dict = {} step_one_solid = single_output_solid( name='step_one_solid', input_defs=[], compute_fn=lambda context, args: _set_key_value( did_run_dict, 'step_one', True), output_def=OutputDefinition(), ) step_two_solid = single_output_solid( name='step_two_solid', input_defs=[InputDefinition('step_one_solid')], compute_fn=lambda context, args: _set_key_value( did_run_dict, 'step_two', True), output_def=OutputDefinition(), ) @pipeline def pipe(): step_two_solid(step_one_solid()) pipeline_result = execute_pipeline(pipe) assert pipeline_result.success for result in pipeline_result.solid_result_list: assert result.success assert did_run_dict['step_one'] is True assert did_run_dict['step_two'] is True
def test_execute_two_solids_with_same_input_name(): input_def = InputDefinition(name='a_thing') solid_one = single_output_solid( 'solid_one', input_defs=[input_def], compute_fn=lambda context, inputs: inputs['a_thing'] + inputs['a_thing'], output_def=OutputDefinition(), ) solid_two = single_output_solid( 'solid_two', input_defs=[input_def], compute_fn=lambda context, inputs: inputs['a_thing'] + inputs['a_thing'], output_def=OutputDefinition(), ) @pipeline def pipe(): solid_one(define_pass_value_solid('pass_to_one')()) solid_two(define_pass_value_solid('pass_to_two')()) result = execute_pipeline( pipe, environment_dict={ 'solids': { 'pass_to_one': {'config': {'value': 'foo'}}, 'pass_to_two': {'config': {'value': 'bar'}}, } }, ) assert result.success assert result.result_for_solid('solid_one').output_value() == 'foofoo' assert result.result_for_solid('solid_two').output_value() == 'barbar'
def test_failure_midstream(): ''' A \\ C (fails) = D (skipped) // B ''' solid_a = create_root_success_solid('A') solid_b = create_root_success_solid('B') def fail_fn(_context, inputs): check.failed('user error') return [inputs['A'], inputs['B'], {'C': 'compute_called'}] def success_fn(_context, inputs): return [inputs['C'], {'D': 'compute_called'}] solid_c = single_output_solid( name='C', input_defs=[InputDefinition(name='A'), InputDefinition(name='B')], compute_fn=fail_fn, output_def=OutputDefinition(), ) solid_d = single_output_solid( name='D', input_defs=[InputDefinition(name='C')], compute_fn=success_fn, output_def=OutputDefinition(), ) pipeline_def = PipelineDefinition( solid_defs=[solid_a, solid_b, solid_c, solid_d], dependencies={ 'C': { 'A': DependencyDefinition(solid_a.name), 'B': DependencyDefinition(solid_b.name) }, 'D': { 'C': DependencyDefinition(solid_c.name) }, }, ) pipeline_result = execute_pipeline( pipeline_def, run_config=RunConfig.nonthrowing_in_process()) assert pipeline_result.result_for_solid('A').success assert pipeline_result.result_for_solid('B').success assert not pipeline_result.result_for_solid('C').success assert pipeline_result.result_for_solid( 'C').failure_data.error.cls_name == 'CheckError' assert not pipeline_result.result_for_solid('D').success assert pipeline_result.result_for_solid('D').skipped
def test_failure_midstream(): """ A \\ C (fails) = D (skipped) // B """ solid_a = create_root_success_solid("A") solid_b = create_root_success_solid("B") def fail_fn(_context, inputs): check.failed("user error") return [inputs["A"], inputs["B"], {"C": "compute_called"}] def success_fn(_context, inputs): return [inputs["C"], {"D": "compute_called"}] solid_c = single_output_solid( name="C", input_defs=[InputDefinition(name="A"), InputDefinition(name="B")], compute_fn=fail_fn, output_def=OutputDefinition(), ) solid_d = single_output_solid( name="D", input_defs=[InputDefinition(name="C")], compute_fn=success_fn, output_def=OutputDefinition(), ) pipeline_def = PipelineDefinition( solid_defs=[solid_a, solid_b, solid_c, solid_d], dependencies={ "C": { "A": DependencyDefinition(solid_a.name), "B": DependencyDefinition(solid_b.name) }, "D": { "C": DependencyDefinition(solid_c.name) }, }, ) pipeline_result = execute_pipeline(pipeline_def, raise_on_error=False) assert pipeline_result.result_for_solid("A").success assert pipeline_result.result_for_solid("B").success assert not pipeline_result.result_for_solid("C").success assert pipeline_result.result_for_solid( "C").failure_data.error.cls_name == "CheckError" assert not pipeline_result.result_for_solid("D").success assert pipeline_result.result_for_solid("D").skipped
def test_execute_solid_with_input_same_name(): a_thing_solid = single_output_solid( 'a_thing', input_defs=[InputDefinition(name='a_thing')], compute_fn=lambda context, inputs: inputs['a_thing'] + inputs['a_thing' ], output_def=OutputDefinition(), ) @pipeline def pipe(): pass_value = define_pass_value_solid('pass_value') a_thing_solid(pass_value()) result = execute_pipeline(pipe, environment_dict={ 'solids': { 'pass_value': { 'config': { 'value': 'foo' } } } }) assert result.result_for_solid('a_thing').output_value() == 'foofoo'
def _dataframe_solid(name, input_defs, compute_fn): return single_output_solid( name=name, input_defs=input_defs, compute_fn=compute_fn, output_def=OutputDefinition(DataFrame), )
def create_root_fn_failure_solid(name): def failed_fn(**_kwargs): raise Exception('Compute failed') return single_output_solid( name=name, input_defs=[], compute_fn=failed_fn, output_def=OutputDefinition() )
def test_execute_solid_with_input_same_name(): a_thing_solid = single_output_solid( "a_thing", input_defs=[InputDefinition(name="a_thing")], compute_fn=lambda context, inputs: inputs["a_thing"] + inputs["a_thing" ], output_def=OutputDefinition(), ) @pipeline def pipe(): pass_value = define_pass_value_solid("pass_value") a_thing_solid(pass_value()) result = execute_pipeline( pipe, run_config={"solids": { "pass_value": { "config": { "value": "foo" } } }}) assert result.result_for_solid("a_thing").output_value() == "foofoo"
def test_execute_two_solids_with_same_input_name(): input_def = InputDefinition(name="a_thing") solid_one = single_output_solid( "solid_one", input_defs=[input_def], compute_fn=lambda context, inputs: inputs["a_thing"] + inputs["a_thing" ], output_def=OutputDefinition(), ) solid_two = single_output_solid( "solid_two", input_defs=[input_def], compute_fn=lambda context, inputs: inputs["a_thing"] + inputs["a_thing" ], output_def=OutputDefinition(), ) @pipeline def pipe(): solid_one(define_pass_value_solid("pass_to_one")()) solid_two(define_pass_value_solid("pass_to_two")()) result = execute_pipeline( pipe, run_config={ "solids": { "pass_to_one": { "config": { "value": "foo" } }, "pass_to_two": { "config": { "value": "bar" } }, } }, ) assert result.success assert result.result_for_solid("solid_one").output_value() == "foofoo" assert result.result_for_solid("solid_two").output_value() == "barbar"
def create_root_success_solid(name): def root_fn(_context, _args): passed_rows = [] passed_rows.append({name: 'compute_called'}) return passed_rows return single_output_solid( name=name, input_defs=[], compute_fn=root_fn, output_def=OutputDefinition() )
def test_single_compute_fn_returning_result(): solid_inst = single_output_solid( 'test_return_result', input_defs=[], compute_fn=lambda *_args, **_kwargs: Output(None), output_def=OutputDefinition(), ) with pytest.raises(DagsterInvariantViolationError): execute_solid(solid_inst)
def test_execute_dep_solid_different_input_name(): pass_to_first = define_pass_value_solid("pass_to_first") first_solid = single_output_solid( "first_solid", input_defs=[InputDefinition(name="a_thing")], compute_fn=lambda context, inputs: inputs["a_thing"] + inputs["a_thing" ], output_def=OutputDefinition(), ) second_solid = single_output_solid( "second_solid", input_defs=[InputDefinition(name="an_input")], compute_fn=lambda context, inputs: inputs["an_input"] + inputs[ "an_input"], output_def=OutputDefinition(), ) @pipeline def pipe(): second_solid(first_solid(pass_to_first())) result = execute_pipeline( pipe, run_config={"solids": { "pass_to_first": { "config": { "value": "bar" } } }}) assert result.success assert len(result.solid_result_list) == 3 assert result.result_for_solid("pass_to_first").output_value() == "bar" assert result.result_for_solid("first_solid").output_value() == "barbar" assert result.result_for_solid( "second_solid").output_value() == "barbarbarbar"
def create_definition_based_solid(): table_input = InputDefinition('num_csv', DataFrame) def compute_fn(_context, inputs): num_csv = inputs['num_csv'] num_csv['sum'] = num_csv['num1'] + num_csv['num2'] return num_csv # supports CSV and PARQUET by default hello_world = single_output_solid( name='hello_world', input_defs=[table_input], compute_fn=compute_fn, output_def=OutputDefinition(DataFrame), ) return hello_world
def create_definition_based_solid(): table_input = InputDefinition("num_csv", DataFrame) def compute_fn(_context, inputs): num_csv = inputs["num_csv"] num_csv["sum"] = num_csv["num1"] + num_csv["num2"] return num_csv # supports CSV and PARQUET by default hello_world = single_output_solid( name="hello_world", input_defs=[table_input], compute_fn=compute_fn, output_def=OutputDefinition(DataFrame), ) return hello_world
def test_failure_propagation(): ''' B =========== C // \\ A F (skipped) \\ // D (fails) == E (skipped) ''' solid_a = create_root_success_solid('A') def fail_fn(_context, inputs): check.failed('user error') return inputs def success_fn(_context, inputs): return inputs solid_b = single_output_solid( name='B', input_defs=[InputDefinition(name='A')], compute_fn=success_fn, output_def=OutputDefinition(), ) solid_c = single_output_solid( name='C', input_defs=[InputDefinition(name='B')], compute_fn=success_fn, output_def=OutputDefinition(), ) solid_d = single_output_solid( name='D', input_defs=[InputDefinition(name='A')], compute_fn=fail_fn, output_def=OutputDefinition(), ) solid_e = single_output_solid( name='E', input_defs=[InputDefinition(name='D')], compute_fn=success_fn, output_def=OutputDefinition(), ) solid_f = single_output_solid( name='F', input_defs=[InputDefinition(name='C'), InputDefinition(name='E')], compute_fn=success_fn, output_def=OutputDefinition(), ) pipeline_def = PipelineDefinition( solid_defs=[solid_a, solid_b, solid_c, solid_d, solid_e, solid_f], dependencies={ 'B': { 'A': DependencyDefinition(solid_a.name) }, 'D': { 'A': DependencyDefinition(solid_a.name) }, 'C': { 'B': DependencyDefinition(solid_b.name) }, 'E': { 'D': DependencyDefinition(solid_d.name) }, 'F': { 'C': DependencyDefinition(solid_c.name), 'E': DependencyDefinition(solid_e.name) }, }, ) pipeline_result = execute_pipeline(pipeline_def, raise_on_error=False) assert pipeline_result.result_for_solid('A').success assert pipeline_result.result_for_solid('B').success assert pipeline_result.result_for_solid('C').success assert not pipeline_result.result_for_solid('D').success assert pipeline_result.result_for_solid( 'D').failure_data.error.cls_name == 'CheckError' assert not pipeline_result.result_for_solid('E').success assert pipeline_result.result_for_solid('E').skipped assert not pipeline_result.result_for_solid('F').success assert pipeline_result.result_for_solid('F').skipped
def test_failure_propagation(): """ B =========== C // \\ A F (skipped) \\ // D (fails) == E (skipped) """ solid_a = create_root_success_solid("A") def fail_fn(_context, inputs): check.failed("user error") return inputs def success_fn(_context, inputs): return inputs solid_b = single_output_solid( name="B", input_defs=[InputDefinition(name="A")], compute_fn=success_fn, output_def=OutputDefinition(), ) solid_c = single_output_solid( name="C", input_defs=[InputDefinition(name="B")], compute_fn=success_fn, output_def=OutputDefinition(), ) solid_d = single_output_solid( name="D", input_defs=[InputDefinition(name="A")], compute_fn=fail_fn, output_def=OutputDefinition(), ) solid_e = single_output_solid( name="E", input_defs=[InputDefinition(name="D")], compute_fn=success_fn, output_def=OutputDefinition(), ) solid_f = single_output_solid( name="F", input_defs=[InputDefinition(name="C"), InputDefinition(name="E")], compute_fn=success_fn, output_def=OutputDefinition(), ) pipeline_def = PipelineDefinition( solid_defs=[solid_a, solid_b, solid_c, solid_d, solid_e, solid_f], dependencies={ "B": { "A": DependencyDefinition(solid_a.name) }, "D": { "A": DependencyDefinition(solid_a.name) }, "C": { "B": DependencyDefinition(solid_b.name) }, "E": { "D": DependencyDefinition(solid_d.name) }, "F": { "C": DependencyDefinition(solid_c.name), "E": DependencyDefinition(solid_e.name) }, }, ) pipeline_result = execute_pipeline(pipeline_def, raise_on_error=False) assert pipeline_result.result_for_solid("A").success assert pipeline_result.result_for_solid("B").success assert pipeline_result.result_for_solid("C").success assert not pipeline_result.result_for_solid("D").success assert pipeline_result.result_for_solid( "D").failure_data.error.cls_name == "CheckError" assert not pipeline_result.result_for_solid("E").success assert pipeline_result.result_for_solid("E").skipped assert not pipeline_result.result_for_solid("F").success assert pipeline_result.result_for_solid("F").skipped