def _execute_run_command_body(recon_pipeline, pipeline_run_id, instance,
write_stream_fn):
if instance.should_start_background_run_thread:
cancellation_thread, cancellation_thread_shutdown_event = start_run_cancellation_thread(
instance, pipeline_run_id)
pipeline_run = instance.get_run_by_id(pipeline_run_id)
pid = os.getpid()
instance.report_engine_event(
"Started process for run (pid: {pid}).".format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"),
)
try:
for event in core_execute_run(recon_pipeline, pipeline_run, instance):
write_stream_fn(event)
finally:
if instance.should_start_background_run_thread:
cancellation_thread_shutdown_event.set()
if cancellation_thread.is_alive():
cancellation_thread.join(timeout=15)
if cancellation_thread.is_alive():
instance.report_engine_event(
"Cancellation thread did not shutdown gracefully",
pipeline_run,
)
instance.report_engine_event(
"Process for run exited (pid: {pid}).".format(pid=pid),
pipeline_run,
)
def _execute_run_command_body(recon_pipeline, pipeline_run_id, instance,
write_stream_fn):
# we need to send but the fact that we have loaded the args so the calling
# process knows it is safe to clean up the temp input file
write_stream_fn(ExecuteRunArgsLoadComplete())
pipeline_run = instance.get_run_by_id(pipeline_run_id)
pid = os.getpid()
instance.report_engine_event(
"Started process for pipeline (pid: {pid}).".format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"),
)
# Perform setup so that termination of the execution will unwind and report to the
# instance correctly
setup_windows_interrupt_support()
try:
for event in core_execute_run(recon_pipeline, pipeline_run, instance):
write_stream_fn(event)
finally:
instance.report_engine_event(
"Process for pipeline exited (pid: {pid}).".format(pid=pid),
pipeline_run,
)
def _resume_run_command_body(
recon_pipeline: ReconstructablePipeline,
pipeline_run_id: Optional[str],
instance: DagsterInstance,
write_stream_fn: Callable[[DagsterEvent], Any],
set_exit_code_on_failure: bool,
):
if instance.should_start_background_run_thread:
cancellation_thread, cancellation_thread_shutdown_event = start_run_cancellation_thread(
instance, pipeline_run_id)
pipeline_run = instance.get_run_by_id(pipeline_run_id)
check.inst(
pipeline_run,
PipelineRun,
"Pipeline run with id '{}' not found for run execution.".format(
pipeline_run_id),
)
pid = os.getpid()
instance.report_engine_event(
"Started process for resuming pipeline (pid: {pid}).".format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"),
)
run_worker_failed = False
try:
for event in core_execute_run(
recon_pipeline,
pipeline_run,
instance,
resume_from_failure=True,
):
write_stream_fn(event)
if event.event_type == DagsterEventType.PIPELINE_FAILURE:
run_worker_failed = True
except:
# relies on core_execute_run writing failures to the event log before raising
run_worker_failed = True
finally:
if instance.should_start_background_run_thread:
cancellation_thread_shutdown_event.set()
if cancellation_thread.is_alive():
cancellation_thread.join(timeout=15)
if cancellation_thread.is_alive():
instance.report_engine_event(
"Cancellation thread did not shutdown gracefully",
pipeline_run,
)
instance.report_engine_event(
"Process for pipeline exited (pid: {pid}).".format(pid=pid),
pipeline_run,
)
return 1 if (run_worker_failed and set_exit_code_on_failure) else 0
def _execute_run_command_body(recon_pipeline, pipeline_run_id, instance,
write_stream_fn, set_exit_code_on_failure):
if instance.should_start_background_run_thread:
cancellation_thread, cancellation_thread_shutdown_event = start_run_cancellation_thread(
instance, pipeline_run_id)
pipeline_run = instance.get_run_by_id(pipeline_run_id)
pid = os.getpid()
instance.report_engine_event(
"Started process for run (pid: {pid}).".format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"),
)
run_worker_failed = 0
try:
for event in core_execute_run(
recon_pipeline,
pipeline_run,
instance,
):
write_stream_fn(event)
if event.event_type == DagsterEventType.PIPELINE_FAILURE:
run_worker_failed = True
except:
# relies on core_execute_run writing failures to the event log before raising
run_worker_failed = True
finally:
if instance.should_start_background_run_thread:
cancellation_thread_shutdown_event.set()
if cancellation_thread.is_alive():
cancellation_thread.join(timeout=15)
if cancellation_thread.is_alive():
instance.report_engine_event(
"Cancellation thread did not shutdown gracefully",
pipeline_run,
)
instance.report_engine_event(
"Process for run exited (pid: {pid}).".format(pid=pid),
pipeline_run,
)
return 1 if (run_worker_failed and set_exit_code_on_failure) else 0
