def test_engine_events(get_external_pipeline): # pylint: disable=redefined-outer-name
with instance_for_test() as instance:
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=math_diamond, run_config=None)
run_id = pipeline_run.run_id
assert instance.get_run_by_id(
run_id).status == PipelineRunStatus.NOT_STARTED
with get_external_pipeline(
pipeline_run.pipeline_name) as external_pipeline:
launcher = instance.run_launcher
launcher.launch_run(instance, pipeline_run, external_pipeline)
finished_pipeline_run = poll_for_finished_run(instance, run_id)
assert finished_pipeline_run
assert finished_pipeline_run.run_id == run_id
assert finished_pipeline_run.status == PipelineRunStatus.SUCCESS
poll_for_event(instance,
run_id,
event_type="ENGINE_EVENT",
message="Process for pipeline exited")
event_records = instance.all_logs(run_id)
(started_process, executing_steps, finished_steps,
process_exited) = tuple(_get_engine_events(event_records))
assert "Started process for pipeline" in started_process.message
assert "Executing steps in process" in executing_steps.message
assert "Finished steps in process" in finished_steps.message
assert "Process for pipeline exited" in process_exited.message
def test_launch_run_with_unloadable_pipeline_grpc():
with instance_for_test() as instance:
with get_foo_pipeline_handle() as pipeline_handle:
api_client = pipeline_handle.repository_handle.repository_location.client
pipeline_run = instance.create_run(
pipeline_name="foo",
run_id=None,
run_config={},
mode="default",
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=None,
execution_plan_snapshot=None,
parent_pipeline_snapshot=None,
)
run_id = pipeline_run.run_id
original_origin = pipeline_handle.get_external_origin()
# point the api to a pipeline that cannot be loaded
res = deserialize_json_to_dagster_namedtuple(
api_client.start_run(
ExecuteExternalPipelineArgs(
pipeline_origin=original_origin._replace(
pipeline_name="i_am_fake_pipeline"),
pipeline_run_id=run_id,
instance_ref=instance.get_ref(),
)))
assert res.success
finished_pipeline_run = poll_for_finished_run(instance, run_id)
assert finished_pipeline_run
assert finished_pipeline_run.run_id == run_id
assert finished_pipeline_run.status == PipelineRunStatus.FAILURE
poll_for_event(instance,
run_id,
event_type="ENGINE_EVENT",
message="Process for pipeline exited")
event_records = instance.all_logs(run_id)
_check_event_log_contains(
event_records,
[
("ENGINE_EVENT", "Started process for pipeline"),
("ENGINE_EVENT", "Could not load pipeline definition"),
(
"PIPELINE_FAILURE",
"This pipeline run has been marked as failed from outside the execution context",
),
("ENGINE_EVENT", "Process for pipeline exited"),
],
)
def test_terminated_run(get_external_pipeline, run_config): # pylint: disable=redefined-outer-name
with instance_for_test() as instance:
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=sleepy_pipeline, run_config=run_config,
)
with get_external_pipeline(pipeline_run.pipeline_name) as external_pipeline:
run_id = pipeline_run.run_id
assert instance.get_run_by_id(run_id).status == PipelineRunStatus.NOT_STARTED
instance.launch_run(pipeline_run.run_id, external_pipeline)
poll_for_step_start(instance, run_id)
launcher = instance.run_launcher
assert launcher.can_terminate(run_id)
assert launcher.terminate(run_id)
terminated_pipeline_run = poll_for_finished_run(instance, run_id, timeout=30)
terminated_pipeline_run = instance.get_run_by_id(run_id)
assert terminated_pipeline_run.status == PipelineRunStatus.CANCELED
poll_for_event(
instance, run_id, event_type="ENGINE_EVENT", message="Process for pipeline exited",
)
run_logs = instance.all_logs(run_id)
if _is_multiprocess(run_config):
_check_event_log_contains(
run_logs,
[
("PIPELINE_CANCELING", "Sending pipeline termination request."),
(
"ENGINE_EVENT",
"Multiprocess executor: received termination signal - forwarding to active child process",
),
(
"ENGINE_EVENT",
"Multiprocess executor: interrupted all active child processes",
),
("STEP_FAILURE", 'Execution of step "sleepy_solid" failed.'),
("PIPELINE_CANCELED", 'Execution of pipeline "sleepy_pipeline" canceled.',),
("ENGINE_EVENT", "Process for pipeline exited"),
],
)
else:
_check_event_log_contains(
run_logs,
[
("PIPELINE_CANCELING", "Sending pipeline termination request."),
("STEP_FAILURE", 'Execution of step "sleepy_solid" failed.'),
("PIPELINE_CANCELED", 'Execution of pipeline "sleepy_pipeline" canceled.',),
("ENGINE_EVENT", "Pipeline execution terminated by interrupt"),
("ENGINE_EVENT", "Process for pipeline exited"),
],
)
def test_engine_events(get_external_pipeline, run_config): # pylint: disable=redefined-outer-name
with instance_for_test() as instance:
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=math_diamond, run_config=run_config)
run_id = pipeline_run.run_id
assert instance.get_run_by_id(
run_id).status == PipelineRunStatus.NOT_STARTED
with get_external_pipeline(
pipeline_run.pipeline_name) as external_pipeline:
launcher = instance.run_launcher
launcher.launch_run(instance, pipeline_run, external_pipeline)
finished_pipeline_run = poll_for_finished_run(instance, run_id)
assert finished_pipeline_run
assert finished_pipeline_run.run_id == run_id
assert finished_pipeline_run.status == PipelineRunStatus.SUCCESS
poll_for_event(instance,
run_id,
event_type="ENGINE_EVENT",
message="Process for pipeline exited")
event_records = instance.all_logs(run_id)
engine_events = _get_engine_events(event_records)
if _is_multiprocess(run_config):
messages = [
"Started process for pipeline",
"Executing steps using multiprocess executor",
"Launching subprocess for return_one.compute",
"Executing step return_one.compute in subprocess",
# multiply_by_2 and multiply_by_3 launch and execute in non-deterministic order
"",
"",
"",
"",
"Launching subprocess for add.compute",
"Executing step add.compute in subprocess",
"Multiprocess executor: parent process exiting",
"Process for pipeline exited",
]
else:
messages = [
"Started process for pipeline",
"Executing steps in process",
"Finished steps in process",
"Process for pipeline exited",
]
events_iter = iter(engine_events)
assert len(engine_events) == len(messages)
for message in messages:
next_log = next(events_iter)
assert message in next_log.message
def test_terminated_run(get_external_pipeline, in_process): # pylint: disable=redefined-outer-name
with instance_for_test() as instance:
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=sleepy_pipeline, run_config=None)
with get_external_pipeline(
pipeline_run.pipeline_name) as external_pipeline:
run_id = pipeline_run.run_id
assert instance.get_run_by_id(
run_id).status == PipelineRunStatus.NOT_STARTED
launcher = instance.run_launcher
launcher.launch_run(instance, pipeline_run, external_pipeline)
poll_for_step_start(instance, run_id)
assert launcher.can_terminate(run_id)
assert launcher.terminate(run_id)
terminated_pipeline_run = poll_for_finished_run(instance,
run_id,
timeout=30)
terminated_pipeline_run = instance.get_run_by_id(run_id)
assert terminated_pipeline_run.status == PipelineRunStatus.FAILURE
poll_for_event(instance,
run_id,
event_type="ENGINE_EVENT",
message="Process for pipeline exited")
run_logs = instance.all_logs(run_id)
event_types = [
event.dagster_event.event_type_value for event in run_logs
]
if in_process:
assert event_types == [
"ENGINE_EVENT",
"ENGINE_EVENT",
"PIPELINE_START",
"ENGINE_EVENT",
"STEP_START",
"STEP_FAILURE",
"PIPELINE_FAILURE",
"ENGINE_EVENT",
]
else:
assert event_types == [
"ENGINE_EVENT",
"PIPELINE_START",
"ENGINE_EVENT",
"STEP_START",
"STEP_FAILURE",
"PIPELINE_FAILURE",
"ENGINE_EVENT",
"ENGINE_EVENT",
]
def test_launch_run_grpc():
with instance_for_test() as instance:
with get_bar_repo_repository_location(instance) as repository_location:
pipeline_handle = PipelineHandle(
"foo",
repository_location.get_repository("bar_repo").handle)
api_client = repository_location.client
pipeline_run = instance.create_run(
pipeline_name="foo",
run_id=None,
run_config={},
mode="default",
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=None,
execution_plan_snapshot=None,
parent_pipeline_snapshot=None,
)
run_id = pipeline_run.run_id
res = deserialize_json_to_dagster_namedtuple(
api_client.start_run(
ExecuteExternalPipelineArgs(
pipeline_origin=pipeline_handle.get_external_origin(),
pipeline_run_id=run_id,
instance_ref=instance.get_ref(),
)))
assert res.success
finished_pipeline_run = poll_for_finished_run(instance, run_id)
assert finished_pipeline_run
assert finished_pipeline_run.run_id == run_id
assert finished_pipeline_run.status == PipelineRunStatus.SUCCESS
poll_for_event(instance,
run_id,
event_type="ENGINE_EVENT",
message="Process for run exited")
event_records = instance.all_logs(run_id)
_check_event_log_contains(
event_records,
[("ENGINE_EVENT", msg) for msg in [
"Started process for run",
"Executing steps in process",
"Finished steps in process",
"Process for run exited",
]],
)
def test_launch_run_grpc():
with instance_for_test() as instance:
with get_foo_grpc_pipeline_handle() as pipeline_handle:
api_client = pipeline_handle.repository_handle.repository_location_handle.client
pipeline_run = instance.create_run(
pipeline_name="foo",
run_id=None,
run_config={},
mode="default",
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=None,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=None,
execution_plan_snapshot=None,
parent_pipeline_snapshot=None,
)
run_id = pipeline_run.run_id
res = api_client.start_run(
ExecuteRunArgs(
pipeline_origin=pipeline_handle.get_origin(),
pipeline_run_id=run_id,
instance_ref=instance.get_ref(),
))
assert res.success
finished_pipeline_run = poll_for_finished_run(instance, run_id)
assert finished_pipeline_run
assert finished_pipeline_run.run_id == run_id
assert finished_pipeline_run.status == PipelineRunStatus.SUCCESS
poll_for_event(instance,
run_id,
event_type="ENGINE_EVENT",
message="Process for pipeline exited")
event_records = instance.all_logs(run_id)
(started_process, executing_steps, finished_steps,
process_exited) = tuple(_get_engine_events(event_records))
assert "Started process for pipeline" in started_process.message
assert "Executing steps in process" in executing_steps.message
assert "Finished steps in process" in finished_steps.message
assert "Process for pipeline exited" in process_exited.message
def test_terminated_run(get_external_pipeline, run_config): # pylint: disable=redefined-outer-name
with instance_for_test() as instance:
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=sleepy_pipeline,
run_config=run_config,
)
with get_external_pipeline(
pipeline_run.pipeline_name) as external_pipeline:
run_id = pipeline_run.run_id
assert instance.get_run_by_id(
run_id).status == PipelineRunStatus.NOT_STARTED
instance.launch_run(pipeline_run.run_id, external_pipeline)
poll_for_step_start(instance, run_id)
launcher = instance.run_launcher
assert launcher.can_terminate(run_id)
assert launcher.terminate(run_id)
terminated_pipeline_run = poll_for_finished_run(instance,
run_id,
timeout=30)
terminated_pipeline_run = instance.get_run_by_id(run_id)
assert terminated_pipeline_run.status == PipelineRunStatus.FAILURE
poll_for_event(
instance,
run_id,
event_type="ENGINE_EVENT",
message="Process for pipeline exited",
)
run_logs = instance.all_logs(run_id)
if _is_multiprocess(run_config):
_check_event_log(
run_logs,
[
("ENGINE_EVENT", "Started process for pipeline"),
("PIPELINE_START",
'Started execution of pipeline "sleepy_pipeline".'),
("ENGINE_EVENT",
"Executing steps using multiprocess executor"),
("ENGINE_EVENT",
"Launching subprocess for sleepy_solid.compute"),
("ENGINE_EVENT",
"Executing step sleepy_solid.compute in subprocess"),
("STEP_START",
'Started execution of step "sleepy_solid.compute".'),
("ENGINE_EVENT",
"Received pipeline termination request"),
(
"ENGINE_EVENT",
"Multiprocess executor: received termination signal - forwarding to active child process",
),
("STEP_FAILURE",
'Execution of step "sleepy_solid.compute" failed.'),
(
"PIPELINE_FAILURE",
'Execution of pipeline "sleepy_pipeline" failed. An exception was thrown during execution.',
),
("ENGINE_EVENT", "Process for pipeline exited"),
],
)
else:
_check_event_log(
run_logs,
[
("ENGINE_EVENT", "Started process for pipeline"),
("PIPELINE_START",
'Started execution of pipeline "sleepy_pipeline".'),
("ENGINE_EVENT", "Executing steps in process"),
("STEP_START",
'Started execution of step "sleepy_solid.compute".'),
("ENGINE_EVENT",
"Received pipeline termination request"),
("STEP_FAILURE",
'Execution of step "sleepy_solid.compute" failed.'),
("PIPELINE_FAILURE",
'Execution of pipeline "sleepy_pipeline" failed.'),
("ENGINE_EVENT",
"Pipeline execution terminated by interrupt"),
("ENGINE_EVENT", "Process for pipeline exited"),
],
)
def test_engine_events(get_workspace, run_config): # pylint: disable=redefined-outer-name
with instance_for_test() as instance:
with get_workspace(instance) as workspace:
external_pipeline = (
workspace.get_repository_location("test")
.get_repository("nope")
.get_full_external_pipeline("math_diamond")
)
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=math_diamond,
run_config=run_config,
external_pipeline_origin=external_pipeline.get_external_origin(),
pipeline_code_origin=external_pipeline.get_python_origin(),
)
run_id = pipeline_run.run_id
assert instance.get_run_by_id(run_id).status == PipelineRunStatus.NOT_STARTED
instance.launch_run(pipeline_run.run_id, workspace)
finished_pipeline_run = poll_for_finished_run(instance, run_id)
assert finished_pipeline_run
assert finished_pipeline_run.run_id == run_id
assert finished_pipeline_run.status == PipelineRunStatus.SUCCESS
poll_for_event(
instance, run_id, event_type="ENGINE_EVENT", message="Process for pipeline exited"
)
event_records = instance.all_logs(run_id)
engine_events = _get_engine_events(event_records)
if _is_multiprocess(run_config):
messages = [
"Started process for pipeline",
"Executing steps using multiprocess executor",
"Launching subprocess for return_one",
"Executing step return_one in subprocess",
"Starting initialization of resources",
"Finished initialization of resources",
# multiply_by_2 and multiply_by_3 launch and execute in non-deterministic order
"",
"",
"",
"",
"",
"",
"",
"",
"Launching subprocess for add",
"Executing step add in subprocess",
"Starting initialization of resources",
"Finished initialization of resources",
"Multiprocess executor: parent process exiting",
"Process for pipeline exited",
]
else:
messages = [
"Started process for pipeline",
"Executing steps in process",
"Starting initialization of resources",
"Finished initialization of resources",
"Finished steps in process",
"Process for pipeline exited",
]
events_iter = iter(engine_events)
assert len(engine_events) == len(messages)
for message in messages:
next_log = next(events_iter)
assert message in next_log.message
