def test_engine_events(get_external_pipeline): # pylint: disable=redefined-outer-name with instance_for_test() as instance: pipeline_run = instance.create_run_for_pipeline( pipeline_def=math_diamond, run_config=None) run_id = pipeline_run.run_id assert instance.get_run_by_id( run_id).status == PipelineRunStatus.NOT_STARTED with get_external_pipeline( pipeline_run.pipeline_name) as external_pipeline: launcher = instance.run_launcher launcher.launch_run(instance, pipeline_run, external_pipeline) finished_pipeline_run = poll_for_finished_run(instance, run_id) assert finished_pipeline_run assert finished_pipeline_run.run_id == run_id assert finished_pipeline_run.status == PipelineRunStatus.SUCCESS poll_for_event(instance, run_id, event_type="ENGINE_EVENT", message="Process for pipeline exited") event_records = instance.all_logs(run_id) (started_process, executing_steps, finished_steps, process_exited) = tuple(_get_engine_events(event_records)) assert "Started process for pipeline" in started_process.message assert "Executing steps in process" in executing_steps.message assert "Finished steps in process" in finished_steps.message assert "Process for pipeline exited" in process_exited.message
def test_launch_run_with_unloadable_pipeline_grpc(): with instance_for_test() as instance: with get_foo_pipeline_handle() as pipeline_handle: api_client = pipeline_handle.repository_handle.repository_location.client pipeline_run = instance.create_run( pipeline_name="foo", run_id=None, run_config={}, mode="default", solids_to_execute=None, step_keys_to_execute=None, status=None, tags=None, root_run_id=None, parent_run_id=None, pipeline_snapshot=None, execution_plan_snapshot=None, parent_pipeline_snapshot=None, ) run_id = pipeline_run.run_id original_origin = pipeline_handle.get_external_origin() # point the api to a pipeline that cannot be loaded res = deserialize_json_to_dagster_namedtuple( api_client.start_run( ExecuteExternalPipelineArgs( pipeline_origin=original_origin._replace( pipeline_name="i_am_fake_pipeline"), pipeline_run_id=run_id, instance_ref=instance.get_ref(), ))) assert res.success finished_pipeline_run = poll_for_finished_run(instance, run_id) assert finished_pipeline_run assert finished_pipeline_run.run_id == run_id assert finished_pipeline_run.status == PipelineRunStatus.FAILURE poll_for_event(instance, run_id, event_type="ENGINE_EVENT", message="Process for pipeline exited") event_records = instance.all_logs(run_id) _check_event_log_contains( event_records, [ ("ENGINE_EVENT", "Started process for pipeline"), ("ENGINE_EVENT", "Could not load pipeline definition"), ( "PIPELINE_FAILURE", "This pipeline run has been marked as failed from outside the execution context", ), ("ENGINE_EVENT", "Process for pipeline exited"), ], )
def test_terminated_run(get_external_pipeline, run_config): # pylint: disable=redefined-outer-name with instance_for_test() as instance: pipeline_run = instance.create_run_for_pipeline( pipeline_def=sleepy_pipeline, run_config=run_config, ) with get_external_pipeline(pipeline_run.pipeline_name) as external_pipeline: run_id = pipeline_run.run_id assert instance.get_run_by_id(run_id).status == PipelineRunStatus.NOT_STARTED instance.launch_run(pipeline_run.run_id, external_pipeline) poll_for_step_start(instance, run_id) launcher = instance.run_launcher assert launcher.can_terminate(run_id) assert launcher.terminate(run_id) terminated_pipeline_run = poll_for_finished_run(instance, run_id, timeout=30) terminated_pipeline_run = instance.get_run_by_id(run_id) assert terminated_pipeline_run.status == PipelineRunStatus.CANCELED poll_for_event( instance, run_id, event_type="ENGINE_EVENT", message="Process for pipeline exited", ) run_logs = instance.all_logs(run_id) if _is_multiprocess(run_config): _check_event_log_contains( run_logs, [ ("PIPELINE_CANCELING", "Sending pipeline termination request."), ( "ENGINE_EVENT", "Multiprocess executor: received termination signal - forwarding to active child process", ), ( "ENGINE_EVENT", "Multiprocess executor: interrupted all active child processes", ), ("STEP_FAILURE", 'Execution of step "sleepy_solid" failed.'), ("PIPELINE_CANCELED", 'Execution of pipeline "sleepy_pipeline" canceled.',), ("ENGINE_EVENT", "Process for pipeline exited"), ], ) else: _check_event_log_contains( run_logs, [ ("PIPELINE_CANCELING", "Sending pipeline termination request."), ("STEP_FAILURE", 'Execution of step "sleepy_solid" failed.'), ("PIPELINE_CANCELED", 'Execution of pipeline "sleepy_pipeline" canceled.',), ("ENGINE_EVENT", "Pipeline execution terminated by interrupt"), ("ENGINE_EVENT", "Process for pipeline exited"), ], )
def test_engine_events(get_external_pipeline, run_config): # pylint: disable=redefined-outer-name with instance_for_test() as instance: pipeline_run = instance.create_run_for_pipeline( pipeline_def=math_diamond, run_config=run_config) run_id = pipeline_run.run_id assert instance.get_run_by_id( run_id).status == PipelineRunStatus.NOT_STARTED with get_external_pipeline( pipeline_run.pipeline_name) as external_pipeline: launcher = instance.run_launcher launcher.launch_run(instance, pipeline_run, external_pipeline) finished_pipeline_run = poll_for_finished_run(instance, run_id) assert finished_pipeline_run assert finished_pipeline_run.run_id == run_id assert finished_pipeline_run.status == PipelineRunStatus.SUCCESS poll_for_event(instance, run_id, event_type="ENGINE_EVENT", message="Process for pipeline exited") event_records = instance.all_logs(run_id) engine_events = _get_engine_events(event_records) if _is_multiprocess(run_config): messages = [ "Started process for pipeline", "Executing steps using multiprocess executor", "Launching subprocess for return_one.compute", "Executing step return_one.compute in subprocess", # multiply_by_2 and multiply_by_3 launch and execute in non-deterministic order "", "", "", "", "Launching subprocess for add.compute", "Executing step add.compute in subprocess", "Multiprocess executor: parent process exiting", "Process for pipeline exited", ] else: messages = [ "Started process for pipeline", "Executing steps in process", "Finished steps in process", "Process for pipeline exited", ] events_iter = iter(engine_events) assert len(engine_events) == len(messages) for message in messages: next_log = next(events_iter) assert message in next_log.message
def test_terminated_run(get_external_pipeline, in_process): # pylint: disable=redefined-outer-name with instance_for_test() as instance: pipeline_run = instance.create_run_for_pipeline( pipeline_def=sleepy_pipeline, run_config=None) with get_external_pipeline( pipeline_run.pipeline_name) as external_pipeline: run_id = pipeline_run.run_id assert instance.get_run_by_id( run_id).status == PipelineRunStatus.NOT_STARTED launcher = instance.run_launcher launcher.launch_run(instance, pipeline_run, external_pipeline) poll_for_step_start(instance, run_id) assert launcher.can_terminate(run_id) assert launcher.terminate(run_id) terminated_pipeline_run = poll_for_finished_run(instance, run_id, timeout=30) terminated_pipeline_run = instance.get_run_by_id(run_id) assert terminated_pipeline_run.status == PipelineRunStatus.FAILURE poll_for_event(instance, run_id, event_type="ENGINE_EVENT", message="Process for pipeline exited") run_logs = instance.all_logs(run_id) event_types = [ event.dagster_event.event_type_value for event in run_logs ] if in_process: assert event_types == [ "ENGINE_EVENT", "ENGINE_EVENT", "PIPELINE_START", "ENGINE_EVENT", "STEP_START", "STEP_FAILURE", "PIPELINE_FAILURE", "ENGINE_EVENT", ] else: assert event_types == [ "ENGINE_EVENT", "PIPELINE_START", "ENGINE_EVENT", "STEP_START", "STEP_FAILURE", "PIPELINE_FAILURE", "ENGINE_EVENT", "ENGINE_EVENT", ]
def test_launch_run_grpc(): with instance_for_test() as instance: with get_bar_repo_repository_location(instance) as repository_location: pipeline_handle = PipelineHandle( "foo", repository_location.get_repository("bar_repo").handle) api_client = repository_location.client pipeline_run = instance.create_run( pipeline_name="foo", run_id=None, run_config={}, mode="default", solids_to_execute=None, step_keys_to_execute=None, status=None, tags=None, root_run_id=None, parent_run_id=None, pipeline_snapshot=None, execution_plan_snapshot=None, parent_pipeline_snapshot=None, ) run_id = pipeline_run.run_id res = deserialize_json_to_dagster_namedtuple( api_client.start_run( ExecuteExternalPipelineArgs( pipeline_origin=pipeline_handle.get_external_origin(), pipeline_run_id=run_id, instance_ref=instance.get_ref(), ))) assert res.success finished_pipeline_run = poll_for_finished_run(instance, run_id) assert finished_pipeline_run assert finished_pipeline_run.run_id == run_id assert finished_pipeline_run.status == PipelineRunStatus.SUCCESS poll_for_event(instance, run_id, event_type="ENGINE_EVENT", message="Process for run exited") event_records = instance.all_logs(run_id) _check_event_log_contains( event_records, [("ENGINE_EVENT", msg) for msg in [ "Started process for run", "Executing steps in process", "Finished steps in process", "Process for run exited", ]], )
def test_launch_run_grpc(): with instance_for_test() as instance: with get_foo_grpc_pipeline_handle() as pipeline_handle: api_client = pipeline_handle.repository_handle.repository_location_handle.client pipeline_run = instance.create_run( pipeline_name="foo", run_id=None, run_config={}, mode="default", solids_to_execute=None, step_keys_to_execute=None, status=None, tags=None, root_run_id=None, parent_run_id=None, pipeline_snapshot=None, execution_plan_snapshot=None, parent_pipeline_snapshot=None, ) run_id = pipeline_run.run_id res = api_client.start_run( ExecuteRunArgs( pipeline_origin=pipeline_handle.get_origin(), pipeline_run_id=run_id, instance_ref=instance.get_ref(), )) assert res.success finished_pipeline_run = poll_for_finished_run(instance, run_id) assert finished_pipeline_run assert finished_pipeline_run.run_id == run_id assert finished_pipeline_run.status == PipelineRunStatus.SUCCESS poll_for_event(instance, run_id, event_type="ENGINE_EVENT", message="Process for pipeline exited") event_records = instance.all_logs(run_id) (started_process, executing_steps, finished_steps, process_exited) = tuple(_get_engine_events(event_records)) assert "Started process for pipeline" in started_process.message assert "Executing steps in process" in executing_steps.message assert "Finished steps in process" in finished_steps.message assert "Process for pipeline exited" in process_exited.message
def test_terminated_run(get_external_pipeline, run_config): # pylint: disable=redefined-outer-name with instance_for_test() as instance: pipeline_run = instance.create_run_for_pipeline( pipeline_def=sleepy_pipeline, run_config=run_config, ) with get_external_pipeline( pipeline_run.pipeline_name) as external_pipeline: run_id = pipeline_run.run_id assert instance.get_run_by_id( run_id).status == PipelineRunStatus.NOT_STARTED instance.launch_run(pipeline_run.run_id, external_pipeline) poll_for_step_start(instance, run_id) launcher = instance.run_launcher assert launcher.can_terminate(run_id) assert launcher.terminate(run_id) terminated_pipeline_run = poll_for_finished_run(instance, run_id, timeout=30) terminated_pipeline_run = instance.get_run_by_id(run_id) assert terminated_pipeline_run.status == PipelineRunStatus.FAILURE poll_for_event( instance, run_id, event_type="ENGINE_EVENT", message="Process for pipeline exited", ) run_logs = instance.all_logs(run_id) if _is_multiprocess(run_config): _check_event_log( run_logs, [ ("ENGINE_EVENT", "Started process for pipeline"), ("PIPELINE_START", 'Started execution of pipeline "sleepy_pipeline".'), ("ENGINE_EVENT", "Executing steps using multiprocess executor"), ("ENGINE_EVENT", "Launching subprocess for sleepy_solid.compute"), ("ENGINE_EVENT", "Executing step sleepy_solid.compute in subprocess"), ("STEP_START", 'Started execution of step "sleepy_solid.compute".'), ("ENGINE_EVENT", "Received pipeline termination request"), ( "ENGINE_EVENT", "Multiprocess executor: received termination signal - forwarding to active child process", ), ("STEP_FAILURE", 'Execution of step "sleepy_solid.compute" failed.'), ( "PIPELINE_FAILURE", 'Execution of pipeline "sleepy_pipeline" failed. An exception was thrown during execution.', ), ("ENGINE_EVENT", "Process for pipeline exited"), ], ) else: _check_event_log( run_logs, [ ("ENGINE_EVENT", "Started process for pipeline"), ("PIPELINE_START", 'Started execution of pipeline "sleepy_pipeline".'), ("ENGINE_EVENT", "Executing steps in process"), ("STEP_START", 'Started execution of step "sleepy_solid.compute".'), ("ENGINE_EVENT", "Received pipeline termination request"), ("STEP_FAILURE", 'Execution of step "sleepy_solid.compute" failed.'), ("PIPELINE_FAILURE", 'Execution of pipeline "sleepy_pipeline" failed.'), ("ENGINE_EVENT", "Pipeline execution terminated by interrupt"), ("ENGINE_EVENT", "Process for pipeline exited"), ], )
def test_engine_events(get_workspace, run_config): # pylint: disable=redefined-outer-name with instance_for_test() as instance: with get_workspace(instance) as workspace: external_pipeline = ( workspace.get_repository_location("test") .get_repository("nope") .get_full_external_pipeline("math_diamond") ) pipeline_run = instance.create_run_for_pipeline( pipeline_def=math_diamond, run_config=run_config, external_pipeline_origin=external_pipeline.get_external_origin(), pipeline_code_origin=external_pipeline.get_python_origin(), ) run_id = pipeline_run.run_id assert instance.get_run_by_id(run_id).status == PipelineRunStatus.NOT_STARTED instance.launch_run(pipeline_run.run_id, workspace) finished_pipeline_run = poll_for_finished_run(instance, run_id) assert finished_pipeline_run assert finished_pipeline_run.run_id == run_id assert finished_pipeline_run.status == PipelineRunStatus.SUCCESS poll_for_event( instance, run_id, event_type="ENGINE_EVENT", message="Process for pipeline exited" ) event_records = instance.all_logs(run_id) engine_events = _get_engine_events(event_records) if _is_multiprocess(run_config): messages = [ "Started process for pipeline", "Executing steps using multiprocess executor", "Launching subprocess for return_one", "Executing step return_one in subprocess", "Starting initialization of resources", "Finished initialization of resources", # multiply_by_2 and multiply_by_3 launch and execute in non-deterministic order "", "", "", "", "", "", "", "", "Launching subprocess for add", "Executing step add in subprocess", "Starting initialization of resources", "Finished initialization of resources", "Multiprocess executor: parent process exiting", "Process for pipeline exited", ] else: messages = [ "Started process for pipeline", "Executing steps in process", "Starting initialization of resources", "Finished initialization of resources", "Finished steps in process", "Process for pipeline exited", ] events_iter = iter(engine_events) assert len(engine_events) == len(messages) for message in messages: next_log = next(events_iter) assert message in next_log.message