def _execute_run_command_body(recon_pipeline, pipeline_run_id, instance, write_stream_fn): if instance.should_start_background_run_thread: cancellation_thread, cancellation_thread_shutdown_event = start_run_cancellation_thread( instance, pipeline_run_id) pipeline_run = instance.get_run_by_id(pipeline_run_id) pid = os.getpid() instance.report_engine_event( "Started process for run (pid: {pid}).".format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"), ) try: for event in core_execute_run(recon_pipeline, pipeline_run, instance): write_stream_fn(event) finally: if instance.should_start_background_run_thread: cancellation_thread_shutdown_event.set() if cancellation_thread.is_alive(): cancellation_thread.join(timeout=15) if cancellation_thread.is_alive(): instance.report_engine_event( "Cancellation thread did not shutdown gracefully", pipeline_run, ) instance.report_engine_event( "Process for run exited (pid: {pid}).".format(pid=pid), pipeline_run, )
def _execute_run_command_body(recon_pipeline, pipeline_run_id, instance, write_stream_fn): # we need to send but the fact that we have loaded the args so the calling # process knows it is safe to clean up the temp input file write_stream_fn(ExecuteRunArgsLoadComplete()) pipeline_run = instance.get_run_by_id(pipeline_run_id) pid = os.getpid() instance.report_engine_event( "Started process for pipeline (pid: {pid}).".format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"), ) # Perform setup so that termination of the execution will unwind and report to the # instance correctly setup_windows_interrupt_support() try: for event in core_execute_run(recon_pipeline, pipeline_run, instance): write_stream_fn(event) finally: instance.report_engine_event( "Process for pipeline exited (pid: {pid}).".format(pid=pid), pipeline_run, )
def _resume_run_command_body( recon_pipeline: ReconstructablePipeline, pipeline_run_id: Optional[str], instance: DagsterInstance, write_stream_fn: Callable[[DagsterEvent], Any], set_exit_code_on_failure: bool, ): if instance.should_start_background_run_thread: cancellation_thread, cancellation_thread_shutdown_event = start_run_cancellation_thread( instance, pipeline_run_id) pipeline_run = instance.get_run_by_id(pipeline_run_id) check.inst( pipeline_run, PipelineRun, "Pipeline run with id '{}' not found for run execution.".format( pipeline_run_id), ) pid = os.getpid() instance.report_engine_event( "Started process for resuming pipeline (pid: {pid}).".format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"), ) run_worker_failed = False try: for event in core_execute_run( recon_pipeline, pipeline_run, instance, resume_from_failure=True, ): write_stream_fn(event) if event.event_type == DagsterEventType.PIPELINE_FAILURE: run_worker_failed = True except: # relies on core_execute_run writing failures to the event log before raising run_worker_failed = True finally: if instance.should_start_background_run_thread: cancellation_thread_shutdown_event.set() if cancellation_thread.is_alive(): cancellation_thread.join(timeout=15) if cancellation_thread.is_alive(): instance.report_engine_event( "Cancellation thread did not shutdown gracefully", pipeline_run, ) instance.report_engine_event( "Process for pipeline exited (pid: {pid}).".format(pid=pid), pipeline_run, ) return 1 if (run_worker_failed and set_exit_code_on_failure) else 0
def _execute_run_command_body(recon_pipeline, pipeline_run_id, instance, write_stream_fn, set_exit_code_on_failure): if instance.should_start_background_run_thread: cancellation_thread, cancellation_thread_shutdown_event = start_run_cancellation_thread( instance, pipeline_run_id) pipeline_run = instance.get_run_by_id(pipeline_run_id) pid = os.getpid() instance.report_engine_event( "Started process for run (pid: {pid}).".format(pid=pid), pipeline_run, EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"), ) run_worker_failed = 0 try: for event in core_execute_run( recon_pipeline, pipeline_run, instance, ): write_stream_fn(event) if event.event_type == DagsterEventType.PIPELINE_FAILURE: run_worker_failed = True except: # relies on core_execute_run writing failures to the event log before raising run_worker_failed = True finally: if instance.should_start_background_run_thread: cancellation_thread_shutdown_event.set() if cancellation_thread.is_alive(): cancellation_thread.join(timeout=15) if cancellation_thread.is_alive(): instance.report_engine_event( "Cancellation thread did not shutdown gracefully", pipeline_run, ) instance.report_engine_event( "Process for run exited (pid: {pid}).".format(pid=pid), pipeline_run, ) return 1 if (run_worker_failed and set_exit_code_on_failure) else 0