def _report_run_failed_if_not_finished( instance: DagsterInstance, pipeline_run_id: str) -> Generator[DagsterEvent, None, None]: check.inst_param(instance, "instance", DagsterInstance) pipeline_run = instance.get_run_by_id(pipeline_run_id) if pipeline_run and (not pipeline_run.is_finished): yield instance.report_run_failed(pipeline_run)
def _resume_run_command_body( recon_pipeline: ReconstructablePipeline, pipeline_run_id: Optional[str], instance: DagsterInstance, write_stream_fn: Callable[[DagsterEvent], Any], set_exit_code_on_failure: bool, ): if instance.should_start_background_run_thread: cancellation_thread, cancellation_thread_shutdown_event = start_run_cancellation_thread( instance, pipeline_run_id) pipeline_run = instance.get_run_by_id(pipeline_run_id) check.inst( pipeline_run, PipelineRun, "Pipeline run with id '{}' not found for run execution.".format( pipeline_run_id), ) pid = os.getpid() instance.report_engine_event( "Started process for resuming pipeline (pid: {pid}).".format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"), ) run_worker_failed = False try: for event in core_execute_run( recon_pipeline, pipeline_run, instance, resume_from_failure=True, ): write_stream_fn(event) if event.event_type == DagsterEventType.PIPELINE_FAILURE: run_worker_failed = True except: # relies on core_execute_run writing failures to the event log before raising run_worker_failed = True finally: if instance.should_start_background_run_thread: cancellation_thread_shutdown_event.set() if cancellation_thread.is_alive(): cancellation_thread.join(timeout=15) if cancellation_thread.is_alive(): instance.report_engine_event( "Cancellation thread did not shutdown gracefully", pipeline_run, ) instance.report_engine_event( "Process for pipeline exited (pid: {pid}).".format(pid=pid), pipeline_run, ) return 1 if (run_worker_failed and set_exit_code_on_failure) else 0