def test_running(): handle = ExecutionTargetHandle.for_pipeline_python_file( __file__, 'passing_pipeline') environment_dict = { 'solids': { 'sum_solid': { 'inputs': { 'num': file_relative_path(__file__, 'data/num.csv') } } } } instance = DagsterInstance.local_temp() pipeline_run = instance.create_run_for_pipeline( pipeline_def=passing_pipeline, environment_dict=environment_dict, ) execution_manager = SubprocessExecutionManager(instance) execution_manager.execute_pipeline(handle, passing_pipeline, pipeline_run, instance) execution_manager.join() assert instance.get_run_by_id( pipeline_run.run_id).status == PipelineRunStatus.SUCCESS events = instance.all_logs(pipeline_run.run_id) assert events engine_events = get_events_of_type(events, DagsterEventType.ENGINE_EVENT) assert (len([ ev for ev in engine_events if 'SubprocessExecutionManager' in ev.message ]) == 3) # starting, started, exit
def test_execution_crash(): handle = ExecutionTargetHandle.for_pipeline_python_file( __file__, 'crashy_pipeline') environment_dict = { 'solids': { 'sum_solid': { 'inputs': { 'num': file_relative_path(__file__, 'data/num.csv') } } } } instance = DagsterInstance.local_temp() pipeline_run = instance.create_run_for_pipeline( pipeline_def=crashy_pipeline, environment_dict=environment_dict, ) execution_manager = SubprocessExecutionManager(instance) execution_manager.execute_pipeline(handle, crashy_pipeline, pipeline_run, instance) execution_manager.join() assert instance.get_run_by_id( pipeline_run.run_id).status == PipelineRunStatus.FAILURE crash_log = instance.all_logs(pipeline_run.run_id)[ -2] # last message is pipeline failure, second to last is... assert crash_log.message.startswith( '[SubprocessExecutionManager] Pipeline execution process for {run_id} unexpectedly exited' .format(run_id=pipeline_run.run_id))
def test_failing(): run_id = make_new_run_id() handle = ExecutionTargetHandle.for_pipeline_python_file( __file__, 'failing_pipeline') environment_dict = { 'solids': { 'sum_solid': { 'inputs': { 'num': file_relative_path(__file__, 'data/num.csv') } } } } selector = ExecutionSelector('csv_hello_world') instance = DagsterInstance.local_temp() pipeline_run = instance.create_run( PipelineRun( pipeline_name=failing_pipeline.name, run_id=run_id, selector=selector, environment_dict=environment_dict, mode='default', step_keys_to_execute=None, tags=None, status=PipelineRunStatus.NOT_STARTED, )) execution_manager = SubprocessExecutionManager(instance) execution_manager.execute_pipeline(handle, failing_pipeline, pipeline_run, instance) execution_manager.join() assert instance.get_run_by_id(run_id).status == PipelineRunStatus.FAILURE assert instance.all_logs(run_id)
def test_failing(): handle = ExecutionTargetHandle.for_pipeline_python_file( __file__, 'failing_pipeline') environment_dict = { 'solids': { 'sum_solid': { 'inputs': { 'num': file_relative_path(__file__, 'data/num.csv') } } } } instance = DagsterInstance.local_temp() pipeline_run = instance.create_run_for_pipeline( pipeline_def=failing_pipeline, environment_dict=environment_dict, ) execution_manager = SubprocessExecutionManager(instance) execution_manager.execute_pipeline(handle, failing_pipeline, pipeline_run, instance) execution_manager.join() assert instance.get_run_by_id( pipeline_run.run_id).status == PipelineRunStatus.FAILURE assert instance.all_logs(pipeline_run.run_id)
def test_execution_crash(): run_id = make_new_run_id() handle = ExecutionTargetHandle.for_pipeline_python_file(__file__, 'crashy_pipeline') environment_dict = { 'solids': {'sum_solid': {'inputs': {'num': file_relative_path(__file__, 'data/num.csv')}}} } selector = ExecutionSelector('csv_hello_world') instance = DagsterInstance.local_temp() pipeline_run = instance.create_run( PipelineRun( pipeline_name=crashy_pipeline.name, run_id=run_id, selector=selector, environment_dict=environment_dict, mode='default', reexecution_config=None, step_keys_to_execute=None, tags=None, status=PipelineRunStatus.NOT_STARTED, ) ) execution_manager = SubprocessExecutionManager(instance) execution_manager.execute_pipeline(handle, crashy_pipeline, pipeline_run, instance) execution_manager.join() assert instance.get_run_by_id(run_id).status == PipelineRunStatus.FAILURE last_log = instance.all_logs(run_id)[-1] assert last_log.message.startswith( 'Exception: Pipeline execution process for {run_id} unexpectedly exited\n'.format( run_id=run_id ) )
def test_multiprocessing_execution_for_composite_solid(): environment_dict = { 'solids': { 'composite_with_nested_config_solid': { 'solids': {'node_a': {'config': {'foo': 'baz'}}, 'node_b': {'config': {'bar': 3}}} } } } instance = DagsterInstance.local_temp() pipeline_run = instance.create_run_for_pipeline( pipeline_def=composite_pipeline, environment_dict=environment_dict, ) execution_manager = SubprocessExecutionManager(instance) execution_manager.execute_pipeline(reconstructable(composite_pipeline), pipeline_run, instance) execution_manager.join() assert instance.get_run_by_id(pipeline_run.run_id).status == PipelineRunStatus.SUCCESS environment_dict = { 'solids': { 'composite_with_nested_config_solid': { 'solids': {'node_a': {'config': {'foo': 'baz'}}, 'node_b': {'config': {'bar': 3}}} } }, 'execution': {'multiprocess': {}}, 'storage': {'filesystem': {}}, } pipeline_run = instance.create_run_for_pipeline( pipeline_def=composite_pipeline, environment_dict=environment_dict, ) execution_manager = SubprocessExecutionManager(instance) execution_manager.execute_pipeline(reconstructable(composite_pipeline), pipeline_run, instance) execution_manager.join()
def test_multiprocessing_execution_for_composite_solid(): environment_dict = { 'solids': { 'composite_with_nested_config_solid': { 'solids': {'node_a': {'config': {'foo': 'baz'}}, 'node_b': {'config': {'bar': 3}}} } } } run_id = make_new_run_id() handle = ExecutionTargetHandle.for_pipeline_python_file(__file__, 'composite_pipeline') instance = DagsterInstance.local_temp() pipeline_run = instance.create_run( PipelineRun( pipeline_name=composite_pipeline.name, run_id=run_id, selector=ExecutionSelector('nonce'), environment_dict=environment_dict, mode='default', reexecution_config=None, step_keys_to_execute=None, tags=None, status=PipelineRunStatus.NOT_STARTED, ) ) execution_manager = SubprocessExecutionManager(instance) execution_manager.execute_pipeline(handle, composite_pipeline, pipeline_run, instance) execution_manager.join() assert instance.get_run_by_id(run_id).status == PipelineRunStatus.SUCCESS environment_dict = { 'solids': { 'composite_with_nested_config_solid': { 'solids': {'node_a': {'config': {'foo': 'baz'}}, 'node_b': {'config': {'bar': 3}}} } }, 'execution': {'multiprocess': {}}, 'storage': {'filesystem': {}}, } run_id = make_new_run_id() pipeline_run = instance.create_run( PipelineRun( pipeline_name=composite_pipeline.name, run_id=run_id, selector=ExecutionSelector('nonce'), environment_dict=environment_dict, mode='default', reexecution_config=None, step_keys_to_execute=None, tags=None, status=PipelineRunStatus.NOT_STARTED, ) ) execution_manager = SubprocessExecutionManager(instance) execution_manager.execute_pipeline(handle, composite_pipeline, pipeline_run, instance) execution_manager.join()
def test_multiprocessing_execution_for_composite_solid_with_config_mapping(): environment_dict = { 'solids': { 'composite_with_nested_config_solid_and_config_mapping': { 'config': { 'foo': 'baz', 'bar': 3 } } } } handle = ExecutionTargetHandle.for_pipeline_python_file( __file__, 'composite_pipeline_with_config_mapping') instance = DagsterInstance.local_temp() pipeline_run = instance.create_run_for_pipeline( pipeline_def=composite_pipeline_with_config_mapping, environment_dict=environment_dict, ) execution_manager = SubprocessExecutionManager(instance) execution_manager.execute_pipeline(handle, composite_pipeline_with_config_mapping, pipeline_run, instance) execution_manager.join() assert instance.get_run_by_id( pipeline_run.run_id).status == PipelineRunStatus.SUCCESS environment_dict = { 'solids': { 'composite_with_nested_config_solid_and_config_mapping': { 'config': { 'foo': 'baz', 'bar': 3 } } }, 'execution': { 'multiprocess': {} }, 'storage': { 'filesystem': {} }, } pipeline_run = instance.create_run_for_pipeline( pipeline_def=composite_pipeline_with_config_mapping, environment_dict=environment_dict, ) execution_manager = SubprocessExecutionManager(instance) execution_manager.execute_pipeline(handle, composite_pipeline_with_config_mapping, pipeline_run, instance) execution_manager.join() assert instance.get_run_by_id( pipeline_run.run_id).status == PipelineRunStatus.SUCCESS
def test_running(): run_id = make_new_run_id() handle = ExecutionTargetHandle.for_pipeline_python_file( __file__, 'passing_pipeline') environment_dict = { 'solids': { 'sum_solid': { 'inputs': { 'num': script_relative_path('data/num.csv') } } } } selector = ExecutionSelector('csv_hello_world') instance = DagsterInstance.local_temp() pipeline_run = instance.create_run( PipelineRun( pipeline_name=passing_pipeline.name, run_id=run_id, selector=selector, environment_dict=environment_dict, mode='default', reexecution_config=None, step_keys_to_execute=None, tags=None, status=PipelineRunStatus.NOT_STARTED, )) execution_manager = SubprocessExecutionManager(instance) execution_manager.execute_pipeline(handle, passing_pipeline, pipeline_run, instance, raise_on_error=False) execution_manager.join() assert instance.get_run(run_id).status == PipelineRunStatus.SUCCESS events = instance.all_logs(run_id) assert events process_start_events = get_events_of_type( events, DagsterEventType.PIPELINE_PROCESS_START) assert len(process_start_events) == 1 process_started_events = get_events_of_type( events, DagsterEventType.PIPELINE_PROCESS_STARTED) assert len(process_started_events) == 1 process_exited_events = get_events_of_type( events, DagsterEventType.PIPELINE_PROCESS_EXITED) assert len(process_exited_events) == 1
def test_running(): run_id = make_new_run_id() handle = ExecutionTargetHandle.for_pipeline_python_file( __file__, 'passing_pipeline') environment_dict = { 'solids': { 'sum_solid': { 'inputs': { 'num': file_relative_path(__file__, 'data/num.csv') } } } } selector = ExecutionSelector('csv_hello_world') instance = DagsterInstance.local_temp() pipeline_run = instance.create_run( PipelineRun( pipeline_name=passing_pipeline.name, run_id=run_id, selector=selector, environment_dict=environment_dict, mode='default', step_keys_to_execute=None, tags=None, status=PipelineRunStatus.NOT_STARTED, )) execution_manager = SubprocessExecutionManager(instance) execution_manager.execute_pipeline(handle, passing_pipeline, pipeline_run, instance) execution_manager.join() assert instance.get_run_by_id(run_id).status == PipelineRunStatus.SUCCESS events = instance.all_logs(run_id) assert events engine_events = get_events_of_type(events, DagsterEventType.ENGINE_EVENT) assert (len([ ev for ev in engine_events if 'SubprocessExecutionManager' in ev.message ]) == 3) # starting, started, exit