def _report_run_failed_if_not_finished(
instance: DagsterInstance,
pipeline_run_id: str) -> Generator[DagsterEvent, None, None]:
check.inst_param(instance, "instance", DagsterInstance)
pipeline_run = instance.get_run_by_id(pipeline_run_id)
if pipeline_run and (not pipeline_run.is_finished):
yield instance.report_run_failed(pipeline_run)
def _resume_run_command_body(
recon_pipeline: ReconstructablePipeline,
pipeline_run_id: Optional[str],
instance: DagsterInstance,
write_stream_fn: Callable[[DagsterEvent], Any],
set_exit_code_on_failure: bool,
):
if instance.should_start_background_run_thread:
cancellation_thread, cancellation_thread_shutdown_event = start_run_cancellation_thread(
instance, pipeline_run_id)
pipeline_run = instance.get_run_by_id(pipeline_run_id)
check.inst(
pipeline_run,
PipelineRun,
"Pipeline run with id '{}' not found for run execution.".format(
pipeline_run_id),
)
pid = os.getpid()
instance.report_engine_event(
"Started process for resuming pipeline (pid: {pid}).".format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"),
)
run_worker_failed = False
try:
for event in core_execute_run(
recon_pipeline,
pipeline_run,
instance,
resume_from_failure=True,
):
write_stream_fn(event)
if event.event_type == DagsterEventType.PIPELINE_FAILURE:
run_worker_failed = True
except:
# relies on core_execute_run writing failures to the event log before raising
run_worker_failed = True
finally:
if instance.should_start_background_run_thread:
cancellation_thread_shutdown_event.set()
if cancellation_thread.is_alive():
cancellation_thread.join(timeout=15)
if cancellation_thread.is_alive():
instance.report_engine_event(
"Cancellation thread did not shutdown gracefully",
pipeline_run,
)
instance.report_engine_event(
"Process for pipeline exited (pid: {pid}).".format(pid=pid),
pipeline_run,
)
return 1 if (run_worker_failed and set_exit_code_on_failure) else 0
