def execute(self, pipeline_context, execution_plan):
check.inst_param(pipeline_context, 'pipeline_context', SystemPipelineExecutionContext)
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
step_keys_to_execute = execution_plan.step_keys_to_execute
yield DagsterEvent.engine_event(
pipeline_context,
'Executing steps in process (pid: {pid})'.format(pid=os.getpid()),
event_specific_data=EngineEventData.in_process(os.getpid(), step_keys_to_execute),
)
with time_execution_scope() as timer_result:
for event in inner_plan_execution_iterator(
pipeline_context, execution_plan, self.retries
):
yield event
yield DagsterEvent.engine_event(
pipeline_context,
'Finished steps in process (pid: {pid}) in {duration_ms}'.format(
pid=os.getpid(), duration_ms=format_duration(timer_result.millis)
),
event_specific_data=EngineEventData.in_process(os.getpid(), step_keys_to_execute),
)
def execute(self, pipeline_context, execution_plan):
check.inst_param(pipeline_context, "pipeline_context",
SystemPipelineExecutionContext)
check.inst_param(execution_plan, "execution_plan", ExecutionPlan)
step_keys_to_execute = execution_plan.step_keys_to_execute
yield DagsterEvent.engine_event(
pipeline_context,
"Executing steps in process (pid: {pid})".format(pid=os.getpid()),
event_specific_data=EngineEventData.in_process(
os.getpid(), step_keys_to_execute),
)
with time_execution_scope() as timer_result:
yield from inner_plan_execution_iterator(pipeline_context,
execution_plan)
yield DagsterEvent.engine_event(
pipeline_context,
"Finished steps in process (pid: {pid}) in {duration_ms}".format(
pid=os.getpid(),
duration_ms=format_duration(timer_result.millis)),
event_specific_data=EngineEventData.in_process(
os.getpid(), step_keys_to_execute),
)
def _execute_run_command_body(recon_pipeline, pipeline_run_id, instance,
write_stream_fn):
if instance.should_start_background_run_thread:
cancellation_thread, cancellation_thread_shutdown_event = start_run_cancellation_thread(
instance, pipeline_run_id)
pipeline_run = instance.get_run_by_id(pipeline_run_id)
pid = os.getpid()
instance.report_engine_event(
"Started process for run (pid: {pid}).".format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"),
)
try:
for event in core_execute_run(recon_pipeline, pipeline_run, instance):
write_stream_fn(event)
finally:
if instance.should_start_background_run_thread:
cancellation_thread_shutdown_event.set()
if cancellation_thread.is_alive():
cancellation_thread.join(timeout=15)
if cancellation_thread.is_alive():
instance.report_engine_event(
"Cancellation thread did not shutdown gracefully",
pipeline_run,
)
instance.report_engine_event(
"Process for run exited (pid: {pid}).".format(pid=pid),
pipeline_run,
)
def test_correct_timezone(conn_string):
event_log_storage = PostgresEventLogStorage.create_clean_storage(
conn_string)
curr_time = time.time()
event = DagsterEventRecord(
None,
"Message2",
"debug",
"",
"foo",
curr_time,
dagster_event=DagsterEvent(
DagsterEventType.PIPELINE_START.value,
"nonce",
event_specific_data=EngineEventData.in_process(999),
),
)
event_log_storage.store_event(event)
logs = event_log_storage.get_logs_for_run("foo")
assert len(logs) == 1
log = logs[0]
stats = event_log_storage.get_stats_for_run("foo")
assert int(log.timestamp) == int(stats.start_time)
assert int(log.timestamp) == int(curr_time)
def _execute_run_command_body(recon_pipeline, pipeline_run_id, instance,
write_stream_fn):
# we need to send but the fact that we have loaded the args so the calling
# process knows it is safe to clean up the temp input file
write_stream_fn(ExecuteRunArgsLoadComplete())
pipeline_run = instance.get_run_by_id(pipeline_run_id)
pid = os.getpid()
instance.report_engine_event(
"Started process for pipeline (pid: {pid}).".format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"),
)
# Perform setup so that termination of the execution will unwind and report to the
# instance correctly
setup_windows_interrupt_support()
try:
for event in core_execute_run(recon_pipeline, pipeline_run, instance):
write_stream_fn(event)
finally:
instance.report_engine_event(
"Process for pipeline exited (pid: {pid}).".format(pid=pid),
pipeline_run,
)
def test_correct_timezone(self, storage):
curr_time = time.time()
event = EventRecord(
None,
"Message2",
"debug",
"",
"foo",
curr_time,
dagster_event=DagsterEvent(
DagsterEventType.PIPELINE_START.value,
"nonce",
event_specific_data=EngineEventData.in_process(999),
),
)
storage.store_event(event)
logs = storage.get_logs_for_run("foo")
assert len(logs) == 1
log = logs[0]
stats = storage.get_stats_for_run("foo")
assert int(log.timestamp) == int(stats.start_time)
assert int(log.timestamp) == int(curr_time)
def _resume_run_command_body(
recon_pipeline: ReconstructablePipeline,
pipeline_run_id: Optional[str],
instance: DagsterInstance,
write_stream_fn: Callable[[DagsterEvent], Any],
set_exit_code_on_failure: bool,
):
if instance.should_start_background_run_thread:
cancellation_thread, cancellation_thread_shutdown_event = start_run_cancellation_thread(
instance, pipeline_run_id)
pipeline_run = instance.get_run_by_id(pipeline_run_id)
check.inst(
pipeline_run,
PipelineRun,
"Pipeline run with id '{}' not found for run execution.".format(
pipeline_run_id),
)
pid = os.getpid()
instance.report_engine_event(
"Started process for resuming pipeline (pid: {pid}).".format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"),
)
run_worker_failed = False
try:
for event in core_execute_run(
recon_pipeline,
pipeline_run,
instance,
resume_from_failure=True,
):
write_stream_fn(event)
if event.event_type == DagsterEventType.PIPELINE_FAILURE:
run_worker_failed = True
except:
# relies on core_execute_run writing failures to the event log before raising
run_worker_failed = True
finally:
if instance.should_start_background_run_thread:
cancellation_thread_shutdown_event.set()
if cancellation_thread.is_alive():
cancellation_thread.join(timeout=15)
if cancellation_thread.is_alive():
instance.report_engine_event(
"Cancellation thread did not shutdown gracefully",
pipeline_run,
)
instance.report_engine_event(
"Process for pipeline exited (pid: {pid}).".format(pid=pid),
pipeline_run,
)
return 1 if (run_worker_failed and set_exit_code_on_failure) else 0
def _execute_run_command_body(
output_file,
recon_repo,
pipeline_run_id,
instance_ref_json,
):
with ipc_write_stream(output_file) as stream:
instance = _get_instance(stream, instance_ref_json)
if not instance:
return
pipeline_run = instance.get_run_by_id(pipeline_run_id)
pid = os.getpid()
instance.report_engine_event(
'Started process for pipeline (pid: {pid}).'.format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid,
marker_end='cli_api_subprocess_init'),
)
recon_pipeline = _recon_pipeline(stream, recon_repo, pipeline_run)
# Perform setup so that termination of the execution will unwind and report to the
# instance correctly
setup_interrupt_support()
try:
for event in execute_run_iterator(recon_pipeline, pipeline_run,
instance):
stream.send(event)
except DagsterSubprocessError as err:
if not all([
err_info.cls_name == 'KeyboardInterrupt'
for err_info in err.subprocess_error_infos
]):
instance.report_engine_event(
'An exception was thrown during execution that is likely a framework error, '
'rather than an error in user code.',
pipeline_run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())),
)
except Exception: # pylint: disable=broad-except
instance.report_engine_event(
'An exception was thrown during execution that is likely a framework error, '
'rather than an error in user code.',
pipeline_run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())),
)
finally:
instance.report_engine_event(
'Process for pipeline exited (pid: {pid}).'.format(pid=pid),
pipeline_run,
)
def _execute_run_command_body(recon_pipeline, pipeline_run_id, instance,
write_stream_fn):
# we need to send but the fact that we have loaded the args so the calling
# process knows it is safe to clean up the temp input file
write_stream_fn(ExecuteRunArgsLoadComplete())
pipeline_run = instance.get_run_by_id(pipeline_run_id)
pid = os.getpid()
instance.report_engine_event(
"Started process for pipeline (pid: {pid}).".format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"),
)
# Perform setup so that termination of the execution will unwind and report to the
# instance correctly
setup_windows_interrupt_support()
try:
for event in execute_run_iterator(recon_pipeline, pipeline_run,
instance):
write_stream_fn(event)
except KeyboardInterrupt:
instance.report_engine_event(
message="Pipeline execution terminated by interrupt",
pipeline_run=pipeline_run,
)
except DagsterSubprocessError as err:
if not all([
err_info.cls_name == "KeyboardInterrupt"
for err_info in err.subprocess_error_infos
]):
instance.report_engine_event(
"An exception was thrown during execution that is likely a framework error, "
"rather than an error in user code.",
pipeline_run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())),
)
except Exception: # pylint: disable=broad-except
instance.report_engine_event(
"An exception was thrown during execution that is likely a framework error, "
"rather than an error in user code.",
pipeline_run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())),
)
finally:
instance.report_engine_event(
"Process for pipeline exited (pid: {pid}).".format(pid=pid),
pipeline_run,
)
def _execute_run_command_body(output_file, recon_pipeline, pipeline_run_id,
instance_ref):
with ipc_write_stream(output_file) as stream:
# we need to send but the fact that we have loaded the args so the calling
# process knows it is safe to clean up the temp input file
stream.send(ExecuteRunArgsLoadComplete())
instance = DagsterInstance.from_ref(instance_ref)
pipeline_run = instance.get_run_by_id(pipeline_run_id)
pid = os.getpid()
instance.report_engine_event(
'Started process for pipeline (pid: {pid}).'.format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid,
marker_end='cli_api_subprocess_init'),
)
# Perform setup so that termination of the execution will unwind and report to the
# instance correctly
setup_interrupt_support()
try:
for event in execute_run_iterator(recon_pipeline, pipeline_run,
instance):
stream.send(event)
except DagsterSubprocessError as err:
if not all([
err_info.cls_name == 'KeyboardInterrupt'
for err_info in err.subprocess_error_infos
]):
instance.report_engine_event(
'An exception was thrown during execution that is likely a framework error, '
'rather than an error in user code.',
pipeline_run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())),
)
except Exception: # pylint: disable=broad-except
instance.report_engine_event(
'An exception was thrown during execution that is likely a framework error, '
'rather than an error in user code.',
pipeline_run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())),
)
finally:
instance.report_engine_event(
'Process for pipeline exited (pid: {pid}).'.format(pid=pid),
pipeline_run,
)
def execute(self, pipeline_context, execution_plan):
check.inst_param(pipeline_context, "pipeline_context",
PlanOrchestrationContext)
check.inst_param(execution_plan, "execution_plan", ExecutionPlan)
step_keys_to_execute = execution_plan.step_keys_to_execute
yield DagsterEvent.engine_event(
pipeline_context,
"Executing steps in process (pid: {pid})".format(pid=os.getpid()),
event_specific_data=EngineEventData.in_process(
os.getpid(), step_keys_to_execute),
)
with time_execution_scope() as timer_result:
yield from iter(
ExecuteRunWithPlanIterable(
execution_plan=pipeline_context.execution_plan,
iterator=inner_plan_execution_iterator,
execution_context_manager=PlanExecutionContextManager(
pipeline=pipeline_context.pipeline,
retry_mode=pipeline_context.retry_mode,
execution_plan=pipeline_context.execution_plan,
run_config=pipeline_context.run_config,
pipeline_run=pipeline_context.pipeline_run,
instance=pipeline_context.instance,
raise_on_error=pipeline_context.raise_on_error,
output_capture=pipeline_context.output_capture,
),
))
yield DagsterEvent.engine_event(
pipeline_context,
"Finished steps in process (pid: {pid}) in {duration_ms}".format(
pid=os.getpid(),
duration_ms=format_duration(timer_result.millis)),
event_specific_data=EngineEventData.in_process(
os.getpid(), step_keys_to_execute),
)
def create_event(count: int, run_id: str = RUN_ID):
return EventLogEntry(
error_info=None,
user_message=str(count),
level="debug",
run_id=run_id,
timestamp=time.time(),
dagster_event=DagsterEvent(
DagsterEventType.ENGINE_EVENT.value,
"nonce",
event_specific_data=EngineEventData.in_process(999),
),
)
def create_test_event_log_record(message: str, run_id: str = DEFAULT_RUN_ID):
return EventRecord(
None,
message,
"debug",
"",
run_id,
time.time(),
dagster_event=DagsterEvent(
DagsterEventType.ENGINE_EVENT.value,
"nonce",
event_specific_data=EngineEventData.in_process(999),
),
)
def create_event(count: int, run_id: str = RUN_ID):
return EventLogEntry(
None,
str(count),
"debug",
"",
run_id,
time.time(),
dagster_event=DagsterEvent(
DagsterEventType.ENGINE_EVENT.value,
"nonce",
event_specific_data=EngineEventData.in_process(999),
),
)
def evt(name):
return DagsterEventRecord(
None,
name,
'debug',
'',
'foo',
time.time(),
dagster_event=DagsterEvent(
DagsterEventType.ENGINE_EVENT.value,
'nonce',
event_specific_data=EngineEventData.in_process(999),
),
)
def evt(name):
return EventRecord(
None,
name,
"debug",
"",
"foo",
time.time(),
dagster_event=DagsterEvent(
DagsterEventType.ENGINE_EVENT.value,
"nonce",
event_specific_data=EngineEventData.in_process(999),
),
)
def _execute_run_command_body(recon_pipeline, pipeline_run_id, instance,
write_stream_fn, set_exit_code_on_failure):
if instance.should_start_background_run_thread:
cancellation_thread, cancellation_thread_shutdown_event = start_run_cancellation_thread(
instance, pipeline_run_id)
pipeline_run = instance.get_run_by_id(pipeline_run_id)
pid = os.getpid()
instance.report_engine_event(
"Started process for run (pid: {pid}).".format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid, marker_end="cli_api_subprocess_init"),
)
run_worker_failed = 0
try:
for event in core_execute_run(
recon_pipeline,
pipeline_run,
instance,
):
write_stream_fn(event)
if event.event_type == DagsterEventType.PIPELINE_FAILURE:
run_worker_failed = True
except:
# relies on core_execute_run writing failures to the event log before raising
run_worker_failed = True
finally:
if instance.should_start_background_run_thread:
cancellation_thread_shutdown_event.set()
if cancellation_thread.is_alive():
cancellation_thread.join(timeout=15)
if cancellation_thread.is_alive():
instance.report_engine_event(
"Cancellation thread did not shutdown gracefully",
pipeline_run,
)
instance.report_engine_event(
"Process for run exited (pid: {pid}).".format(pid=pid),
pipeline_run,
)
return 1 if (run_worker_failed and set_exit_code_on_failure) else 0
def _execute_run(request):
try:
execute_run_args = deserialize_json_to_dagster_namedtuple(
request.serialized_execute_run_args)
check.inst_param(execute_run_args, 'execute_run_args', ExecuteRunArgs)
recon_pipeline = recon_pipeline_from_origin(
execute_run_args.pipeline_origin)
instance = DagsterInstance.from_ref(execute_run_args.instance_ref)
pipeline_run = instance.get_run_by_id(execute_run_args.pipeline_run_id)
pid = os.getpid()
except: # pylint: disable=bare-except
yield IPCErrorMessage(
serializable_error_info=serializable_error_info_from_exc_info(
sys.exc_info()),
message='Error during RPC setup for ExecuteRun',
)
return
yield instance.report_engine_event(
'Started process for pipeline (pid: {pid}).'.format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid, marker_end='cli_api_subprocess_init'),
)
# This is so nasty but seemingly unavoidable
# https://amir.rachum.com/blog/2017/03/03/generator-cleanup/
closed = False
try:
for event in _core_execute_run(recon_pipeline, pipeline_run, instance):
yield event
except GeneratorExit:
closed = True
raise
finally:
if not closed:
yield instance.report_engine_event(
'Process for pipeline exited (pid: {pid}).'.format(pid=pid),
pipeline_run,
)
def test_event_log_storage_store_events_and_wipe(self, storage):
assert len(storage.get_logs_for_run(DEFAULT_RUN_ID)) == 0
storage.store_event(
EventRecord(
None,
"Message2",
"debug",
"",
DEFAULT_RUN_ID,
time.time(),
dagster_event=DagsterEvent(
DagsterEventType.ENGINE_EVENT.value,
"nonce",
event_specific_data=EngineEventData.in_process(999),
),
))
assert len(storage.get_logs_for_run(DEFAULT_RUN_ID)) == 1
assert storage.get_stats_for_run(DEFAULT_RUN_ID)
storage.wipe()
assert len(storage.get_logs_for_run(DEFAULT_RUN_ID)) == 0
def test_in_memory_event_log_storage_store_events_and_wipe():
storage = InMemoryEventLogStorage()
assert len(storage.get_logs_for_run('foo')) == 0
storage.store_event(
DagsterEventRecord(
None,
'Message2',
'debug',
'',
'foo',
time.time(),
dagster_event=DagsterEvent(
DagsterEventType.ENGINE_EVENT.value,
'nonce',
event_specific_data=EngineEventData.in_process(999),
),
))
assert len(storage.get_logs_for_run('foo')) == 1
storage.wipe()
assert len(storage.get_logs_for_run('foo')) == 0
def test_event_log_storage_store_events_and_wipe(event_storage_factory_cm_fn):
with event_storage_factory_cm_fn() as storage:
assert len(storage.get_logs_for_run('foo')) == 0
storage.store_event(
DagsterEventRecord(
None,
'Message2',
'debug',
'',
'foo',
time.time(),
dagster_event=DagsterEvent(
DagsterEventType.ENGINE_EVENT.value,
'nonce',
event_specific_data=EngineEventData.in_process(999),
),
))
assert len(storage.get_logs_for_run('foo')) == 1
assert storage.get_stats_for_run('foo')
storage.wipe()
assert len(storage.get_logs_for_run('foo')) == 0
def test_filesystem_event_log_storage_store_events_and_wipe():
with seven.TemporaryDirectory() as tmpdir_path:
storage = SqliteEventLogStorage(tmpdir_path)
assert len(storage.get_logs_for_run('foo')) == 0
storage.store_event(
DagsterEventRecord(
None,
'Message2',
'debug',
'',
'foo',
time.time(),
dagster_event=DagsterEvent(
DagsterEventType.ENGINE_EVENT.value,
'nonce',
event_specific_data=EngineEventData.in_process(999),
),
))
assert len(storage.get_logs_for_run('foo')) == 1
storage.wipe()
assert len(storage.get_logs_for_run('foo')) == 0
def test_event_log_delete(event_storage_factory_cm_fn):
with event_storage_factory_cm_fn() as storage:
assert len(storage.get_logs_for_run("foo")) == 0
storage.store_event(
EventRecord(
None,
"Message2",
"debug",
"",
"foo",
time.time(),
dagster_event=DagsterEvent(
DagsterEventType.ENGINE_EVENT.value,
"nonce",
event_specific_data=EngineEventData.in_process(999),
),
))
assert len(storage.get_logs_for_run("foo")) == 1
assert storage.get_stats_for_run("foo")
storage.delete_events("foo")
assert len(storage.get_logs_for_run("foo")) == 0
def execute(pipeline_context, execution_plan):
check.inst_param(pipeline_context, 'pipeline_context',
SystemPipelineExecutionContext)
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
yield DagsterEvent.engine_event(
pipeline_context,
'Executing steps in process (pid: {pid})'.format(pid=os.getpid()),
event_specific_data=EngineEventData.in_process(
os.getpid(), execution_plan.step_keys_to_execute),
)
with time_execution_scope() as timer_result:
check.param_invariant(
isinstance(pipeline_context.executor_config, ExecutorConfig),
'pipeline_context',
'Expected executor_config to be ExecutorConfig got {}'.format(
pipeline_context.executor_config),
)
for event in copy_required_intermediates_for_execution(
pipeline_context, execution_plan):
yield event
# It would be good to implement a reference tracking algorithm here to
# garbage collect results that are no longer needed by any steps
# https://github.com/dagster-io/dagster/issues/811
active_execution = execution_plan.start()
while not active_execution.is_complete:
steps = active_execution.get_steps_to_execute(limit=1)
check.invariant(
len(steps) == 1,
'Invariant Violation: expected step to be available to execute'
)
step = steps[0]
step_context = pipeline_context.for_step(step)
check.invariant(
all(
hasattr(step_context.resources, resource_key) for
resource_key in step_context.required_resource_keys),
'expected step context to have all required resources',
)
with mirror_step_io(step_context):
# capture all of the logs for this step
uncovered_inputs = pipeline_context.intermediates_manager.uncovered_inputs(
step_context, step)
if uncovered_inputs:
# In partial pipeline execution, we may end up here without having validated the
# missing dependent outputs were optional
_assert_missing_inputs_optional(
uncovered_inputs, execution_plan, step.key)
step_context.log.info((
'Not all inputs covered for {step}. Not executing. Output missing for '
'inputs: {uncovered_inputs}').format(
uncovered_inputs=uncovered_inputs,
step=step.key))
yield DagsterEvent.step_skipped_event(step_context)
active_execution.mark_skipped(step.key)
continue
for step_event in check.generator(
dagster_event_sequence_for_step(step_context)):
check.inst(step_event, DagsterEvent)
yield step_event
active_execution.handle_event(step_event)
active_execution.verify_complete(pipeline_context,
step.key)
# process skips from failures or uncovered inputs
for event in active_execution.skipped_step_events_iterator(
pipeline_context):
yield event
yield DagsterEvent.engine_event(
pipeline_context,
'Finished steps in process (pid: {pid}) in {duration_ms}'.format(
pid=os.getpid(),
duration_ms=format_duration(timer_result.millis)),
event_specific_data=EngineEventData.in_process(
os.getpid(), execution_plan.step_keys_to_execute),
)
def execute(self, pipeline_context, execution_plan):
check.inst_param(pipeline_context, "pipeline_context", SystemPipelineExecutionContext)
check.inst_param(execution_plan, "execution_plan", ExecutionPlan)
limit = self.max_concurrent
yield DagsterEvent.engine_event(
pipeline_context,
"Executing steps using multithread executor (pid: {pid})".format(pid=os.getpid()),
event_specific_data=EngineEventData.in_process(os.getpid(), execution_plan.step_keys_to_execute),
)
with time_execution_scope() as timer_result:
with execution_plan.start(retries=self.retries) as active_execution:
active_iters = {}
errors = {}
while not active_execution.is_complete or active_iters:
# start iterators
while len(active_iters) < limit:
steps = active_execution.get_steps_to_execute(limit=(limit - len(active_iters)))
if not steps:
break
for step in steps:
step_context = pipeline_context.for_step(step)
active_iters[step.key] = self.execute_step_in_thread(step.key, step_context, errors)
# process active iterators
empty_iters = []
for key, step_iter in active_iters.items():
try:
event_or_none = next(step_iter)
if event_or_none is None:
continue
yield event_or_none
active_execution.handle_event(event_or_none)
except ThreadCrashException:
serializable_error = serializable_error_info_from_exc_info(sys.exc_info())
yield DagsterEvent.engine_event(
pipeline_context,
f"Multithread executor: thread for step {key} exited unexpectedly",
EngineEventData.engine_error(serializable_error),
)
step_failure_event = DagsterEvent.step_failure_event(
step_context=pipeline_context.for_step(active_execution.get_step_by_key(key)),
step_failure_data=StepFailureData(error=serializable_error, user_failure_data=None),
)
active_execution.handle_event(step_failure_event)
yield step_failure_event
empty_iters.append(key)
except StopIteration:
empty_iters.append(key)
# clear and mark complete finished iterators
for key in empty_iters:
del active_iters[key]
active_execution.verify_complete(pipeline_context, key)
# process skipped and abandoned steps
for event in active_execution.plan_events_iterator(pipeline_context):
yield event
errs = {tid: err for tid, err in errors.items() if err}
if errs:
raise DagsterThreadError(
"During multithread execution errors occurred in threads:\n{error_list}".format(
error_list="\n".join(
[
"In thread {tid}: {err}".format(tid=tid, err=err.to_string())
for tid, err in errs.items()
]
)
),
thread_error_infos=list(errs.values()),
)
yield DagsterEvent.engine_event(
pipeline_context,
"Multithread executor: parent process exiting after {duration} (pid: {pid})".format(
duration=format_duration(timer_result.millis), pid=os.getpid()
),
event_specific_data=EngineEventData.multiprocess(os.getpid()),
)
def in_mp_process(cls, handle, pipeline_run, instance_ref, term_event):
"""
Execute pipeline using message queue as a transport
"""
run_id = pipeline_run.run_id
pipeline_name = pipeline_run.pipeline_name
instance = DagsterInstance.from_ref(instance_ref)
pid = os.getpid()
instance.report_engine_event(
'Started process for pipeline (pid: {pid}).'.format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid,
marker_end='dagit_subprocess_init'),
cls,
)
start_termination_thread(term_event)
try:
handle.build_repository_definition()
pipeline_def = handle.with_pipeline_name(
pipeline_name).build_pipeline_definition()
except Exception: # pylint: disable=broad-except
instance.report_engine_event(
'Failed attempting to load pipeline "{}"'.format(
pipeline_name),
pipeline_run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())),
cls,
)
return
try:
event_list = []
for event in execute_run_iterator(
pipeline_def.build_sub_pipeline(
pipeline_run.selector.solid_subset),
pipeline_run,
instance,
):
event_list.append(event)
return PipelineExecutionResult(pipeline_def, run_id, event_list,
lambda: None)
# Add a DagsterEvent for unexpected exceptions
# Explicitly ignore KeyboardInterrupts since they are used for termination
except DagsterSubprocessError as err:
if not all([
err_info.cls_name == 'KeyboardInterrupt'
for err_info in err.subprocess_error_infos
]):
instance.report_engine_event(
'An exception was thrown during execution that is likely a framework error, '
'rather than an error in user code.',
pipeline_run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())),
cls,
)
except Exception: # pylint: disable=broad-except
instance.report_engine_event(
'An exception was thrown during execution that is likely a framework error, '
'rather than an error in user code.',
pipeline_run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())),
cls,
)
finally:
instance.report_engine_event(
'Process for pipeline exited (pid: {pid}).'.format(pid=pid),
pipeline_run,
cls=cls,
)
def _execute_run(request):
try:
execute_run_args = deserialize_json_to_dagster_namedtuple(
request.serialized_execute_run_args)
check.inst_param(execute_run_args, 'execute_run_args', ExecuteRunArgs)
recon_pipeline = recon_pipeline_from_origin(
execute_run_args.pipeline_origin)
instance = DagsterInstance.from_ref(execute_run_args.instance_ref)
pipeline_run = instance.get_run_by_id(execute_run_args.pipeline_run_id)
pid = os.getpid()
except: # pylint: disable=bare-except
yield IPCErrorMessage(
serializable_error_info=serializable_error_info_from_exc_info(
sys.exc_info()),
message='Error during RPC setup for ExecuteRun',
)
return
yield instance.report_engine_event(
'Started process for pipeline (pid: {pid}).'.format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid, marker_end='cli_api_subprocess_init'),
)
# This is so nasty but seemingly unavoidable
# https://amir.rachum.com/blog/2017/03/03/generator-cleanup/
closed = False
try:
for event in execute_run_iterator(recon_pipeline, pipeline_run,
instance):
yield event
except DagsterSubprocessError as err:
if not all([
err_info.cls_name == 'KeyboardInterrupt'
for err_info in err.subprocess_error_infos
]):
yield instance.report_engine_event(
'An exception was thrown during execution that is likely a framework error, '
'rather than an error in user code.',
pipeline_run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())),
)
instance.report_run_failed(pipeline_run)
except GeneratorExit:
closed = True
raise
except Exception: # pylint: disable=broad-except
yield instance.report_engine_event(
'An exception was thrown during execution that is likely a framework error, '
'rather than an error in user code.',
pipeline_run,
EngineEventData.engine_error(
serializable_error_info_from_exc_info(sys.exc_info())),
)
instance.report_run_failed(pipeline_run)
finally:
if not closed:
yield instance.report_engine_event(
'Process for pipeline exited (pid: {pid}).'.format(pid=pid),
pipeline_run,
)
def execute(pipeline_context, execution_plan):
check.inst_param(pipeline_context, 'pipeline_context',
SystemPipelineExecutionContext)
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
step_levels = execution_plan.execution_step_levels()
step_key_set = set(step.key for step_level in step_levels
for step in step_level)
yield DagsterEvent.engine_event(
pipeline_context,
'Executing steps in process (pid: {pid})'.format(pid=os.getpid()),
event_specific_data=EngineEventData.in_process(
os.getpid(), step_key_set),
)
with time_execution_scope() as timer_result:
check.param_invariant(
isinstance(pipeline_context.executor_config, ExecutorConfig),
'pipeline_context',
'Expected executor_config to be ExecutorConfig got {}'.format(
pipeline_context.executor_config),
)
for event in copy_required_intermediates_for_execution(
pipeline_context, execution_plan):
yield event
failed_or_skipped_steps = set()
# It would be good to implement a reference tracking algorithm here to
# garbage collect results that are no longer needed by any steps
# https://github.com/dagster-io/dagster/issues/811
for step_level in step_levels:
for step in step_level:
step_context = pipeline_context.for_step(step)
with mirror_step_io(step_context):
# capture all of the logs for this step
failed_inputs = []
for step_input in step.step_inputs:
failed_inputs.extend(
failed_or_skipped_steps.intersection(
step_input.dependency_keys))
if failed_inputs:
step_context.log.info((
'Dependencies for step {step} failed: {failed_inputs}. Not executing.'
).format(step=step.key,
failed_inputs=failed_inputs))
failed_or_skipped_steps.add(step.key)
yield DagsterEvent.step_skipped_event(step_context)
continue
uncovered_inputs = pipeline_context.intermediates_manager.uncovered_inputs(
step_context, step)
if uncovered_inputs:
# In partial pipeline execution, we may end up here without having validated the
# missing dependent outputs were optional
_assert_missing_inputs_optional(
uncovered_inputs, execution_plan, step.key)
step_context.log.info((
'Not all inputs covered for {step}. Not executing. Output missing for '
'inputs: {uncovered_inputs}').format(
uncovered_inputs=uncovered_inputs,
step=step.key))
failed_or_skipped_steps.add(step.key)
yield DagsterEvent.step_skipped_event(step_context)
continue
for step_event in check.generator(
dagster_event_sequence_for_step(step_context)):
check.inst(step_event, DagsterEvent)
if step_event.is_step_failure:
failed_or_skipped_steps.add(step.key)
yield step_event
yield DagsterEvent.engine_event(
pipeline_context,
'Finished steps in process (pid: {pid}) in {duration_ms}'.format(
pid=os.getpid(),
duration_ms=format_duration(timer_result.millis)),
event_specific_data=EngineEventData.in_process(
os.getpid(), step_key_set),
)
def _run_in_subprocess(
serialized_execute_run_args,
recon_pipeline,
termination_event,
subprocess_status_handler,
run_event_handler,
):
start_termination_thread(termination_event)
try:
execute_run_args = deserialize_json_to_dagster_namedtuple(
serialized_execute_run_args)
check.inst_param(execute_run_args, 'execute_run_args', ExecuteRunArgs)
instance = DagsterInstance.from_ref(execute_run_args.instance_ref)
pipeline_run = instance.get_run_by_id(execute_run_args.pipeline_run_id)
pid = os.getpid()
except: # pylint: disable=bare-except
event = IPCErrorMessage(
serializable_error_info=serializable_error_info_from_exc_info(
sys.exc_info()),
message='Error during RPC setup for ExecuteRun',
)
subprocess_status_handler(event)
subprocess_status_handler(RunInSubprocessComplete())
return
subprocess_status_handler(StartRunInSubprocessSuccessful())
run_event_handler(
instance.report_engine_event(
'Started process for pipeline (pid: {pid}).'.format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid,
marker_end='cli_api_subprocess_init'),
))
# This is so nasty but seemingly unavoidable
# https://amir.rachum.com/blog/2017/03/03/generator-cleanup/
closed = False
try:
for event in _core_execute_run(recon_pipeline, pipeline_run, instance):
run_event_handler(event)
except KeyboardInterrupt:
run_event_handler(
instance.report_engine_event(
message='Pipeline execution terminated by interrupt',
pipeline_run=pipeline_run,
))
raise
except GeneratorExit:
closed = True
raise
finally:
if not closed:
run_event_handler(
instance.report_engine_event(
'Process for pipeline exited (pid: {pid}).'.format(
pid=pid),
pipeline_run,
))
subprocess_status_handler(RunInSubprocessComplete())
def _run_in_subprocess(
serialized_execute_run_args,
recon_pipeline,
termination_event,
subprocess_status_handler,
run_event_handler,
):
start_termination_thread(termination_event)
try:
execute_run_args = deserialize_json_to_dagster_namedtuple(
serialized_execute_run_args)
check.inst_param(execute_run_args, "execute_run_args",
ExecuteExternalPipelineArgs)
instance = DagsterInstance.from_ref(execute_run_args.instance_ref)
pipeline_run = instance.get_run_by_id(execute_run_args.pipeline_run_id)
if not pipeline_run:
raise DagsterRunNotFoundError(
"gRPC server could not load run {run_id} in order to execute it. Make sure that the gRPC server has access to your run storage."
.format(run_id=execute_run_args.pipeline_run_id),
invalid_run_id=execute_run_args.pipeline_run_id,
)
pid = os.getpid()
except: # pylint: disable=bare-except
serializable_error_info = serializable_error_info_from_exc_info(
sys.exc_info())
event = IPCErrorMessage(
serializable_error_info=serializable_error_info,
message="Error during RPC setup for executing run: {message}".
format(message=serializable_error_info.message),
)
subprocess_status_handler(event)
subprocess_status_handler(RunInSubprocessComplete())
if instance:
instance.dispose()
return
subprocess_status_handler(StartRunInSubprocessSuccessful())
run_event_handler(
instance.report_engine_event(
"Started process for pipeline (pid: {pid}).".format(pid=pid),
pipeline_run,
EngineEventData.in_process(pid,
marker_end="cli_api_subprocess_init"),
))
# This is so nasty but seemingly unavoidable
# https://amir.rachum.com/blog/2017/03/03/generator-cleanup/
closed = False
try:
for event in _core_execute_run(recon_pipeline, pipeline_run, instance):
run_event_handler(event)
except GeneratorExit:
closed = True
raise
finally:
if not closed:
run_event_handler(
instance.report_engine_event(
"Process for pipeline exited (pid: {pid}).".format(
pid=pid),
pipeline_run,
))
subprocess_status_handler(RunInSubprocessComplete())
instance.dispose()
