def _pipeline_execution_iterator(pipeline_context, execution_plan):
"""A complete execution of a pipeline. Yields pipeline start, success,
and failure events.
Args:
pipeline_context (SystemPipelineExecutionContext):
execution_plan (ExecutionPlan):
"""
check.inst_param(pipeline_context, "pipeline_context",
SystemPipelineExecutionContext)
check.inst_param(execution_plan, "execution_plan", ExecutionPlan)
yield DagsterEvent.pipeline_start(pipeline_context)
pipeline_exception_info = None
failed_steps = []
generator_closed = False
try:
for event in pipeline_context.executor.execute(pipeline_context,
execution_plan):
if event.is_step_failure:
failed_steps.append(event.step_key)
yield event
except GeneratorExit:
# Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed
# (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/).
generator_closed = True
pipeline_exception_info = serializable_error_info_from_exc_info(
sys.exc_info())
raise
except (Exception, KeyboardInterrupt): # pylint: disable=broad-except
pipeline_exception_info = serializable_error_info_from_exc_info(
sys.exc_info())
raise # finally block will run before this is re-raised
finally:
if pipeline_exception_info:
event = DagsterEvent.pipeline_failure(
pipeline_context,
"An exception was thrown during execution.",
pipeline_exception_info,
)
elif failed_steps:
event = DagsterEvent.pipeline_failure(
pipeline_context,
"Steps failed: {}.".format(failed_steps),
)
else:
event = DagsterEvent.pipeline_success(pipeline_context)
if not generator_closed:
yield event
def _pipeline_execution_iterator(pipeline_context, execution_plan,
pipeline_run):
'''A complete execution of a pipeline. Yields pipeline start, success,
and failure events. Defers to _steps_execution_iterator for step execution.
'''
if (isinstance(pipeline_context, DagsterEvent)
and pipeline_context.event_type # pylint: disable=no-member
== DagsterEventType.PIPELINE_INIT_FAILURE):
yield pipeline_context
return
yield DagsterEvent.pipeline_start(pipeline_context)
pipeline_success = True
try:
for event in _steps_execution_iterator(pipeline_context,
execution_plan=execution_plan,
pipeline_run=pipeline_run):
if event.is_step_failure:
pipeline_success = False
yield event
except (Exception, KeyboardInterrupt):
pipeline_success = False
raise # finally block will run before this is re-raised
finally:
if pipeline_success:
yield DagsterEvent.pipeline_success(pipeline_context)
else:
yield DagsterEvent.pipeline_failure(pipeline_context)
def _pipeline_execution_iterator(pipeline_context, execution_plan,
pipeline_run):
'''A complete execution of a pipeline. Yields pipeline start, success,
and failure events.
'''
check.inst_param(pipeline_context, 'pipeline_context',
SystemPipelineExecutionContext)
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
yield DagsterEvent.pipeline_start(pipeline_context)
pipeline_success = True
generator_closed = False
try:
for event in pipeline_context.executor_config.get_engine().execute(
pipeline_context, execution_plan):
if event.is_step_failure:
pipeline_success = False
yield event
except GeneratorExit:
# Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed
# (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/).
generator_closed = True
pipeline_success = False
raise
except (Exception, KeyboardInterrupt):
pipeline_success = False
raise # finally block will run before this is re-raised
finally:
if pipeline_success:
event = DagsterEvent.pipeline_success(pipeline_context)
else:
event = DagsterEvent.pipeline_failure(pipeline_context)
if not generator_closed:
yield event
def _execute_pipeline_iterator(pipeline_context, execution_plan, run_config,
step_keys_to_execute):
if (isinstance(pipeline_context, DagsterEvent)
and pipeline_context.event_type # pylint: disable=no-member
== DagsterEventType.PIPELINE_INIT_FAILURE):
yield pipeline_context
return
yield DagsterEvent.pipeline_start(pipeline_context)
pipeline_success = True
try:
for event in _execute_plan_iterator(
pipeline_context,
execution_plan=execution_plan,
run_config=run_config,
step_keys_to_execute=step_keys_to_execute,
):
if event.is_step_failure:
pipeline_success = False
yield event
finally:
if pipeline_success:
yield DagsterEvent.pipeline_success(pipeline_context)
else:
yield DagsterEvent.pipeline_failure(pipeline_context)
def _pipeline_execution_iterator(pipeline_context, execution_plan, run_config,
step_keys_to_execute):
'''A complete execution of a pipeline. Yields pipeline start, success,
and failure events. Defers to _steps_execution_iterator for step execution.
'''
if (isinstance(pipeline_context, DagsterEvent)
and pipeline_context.event_type # pylint: disable=no-member
== DagsterEventType.PIPELINE_INIT_FAILURE):
yield pipeline_context
return
yield DagsterEvent.pipeline_start(pipeline_context)
pipeline_success = True
try:
for event in _steps_execution_iterator(
pipeline_context,
execution_plan=execution_plan,
run_config=run_config,
step_keys_to_execute=step_keys_to_execute,
):
if event.is_step_failure:
pipeline_success = False
yield event
finally:
if pipeline_success:
yield DagsterEvent.pipeline_success(pipeline_context)
else:
yield DagsterEvent.pipeline_failure(pipeline_context)
def _pipeline_execution_iterator(pipeline_context, execution_plan,
pipeline_run):
'''A complete execution of a pipeline. Yields pipeline start, success,
and failure events. Defers to _steps_execution_iterator for step execution.
'''
check.inst_param(pipeline_context, 'pipeline_context',
SystemPipelineExecutionContext)
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
yield DagsterEvent.pipeline_start(pipeline_context)
pipeline_success = True
try:
for event in _steps_execution_iterator(pipeline_context,
execution_plan=execution_plan,
pipeline_run=pipeline_run):
if event.is_step_failure:
pipeline_success = False
yield event
except (Exception, KeyboardInterrupt):
pipeline_success = False
raise # finally block will run before this is re-raised
finally:
if pipeline_success:
yield DagsterEvent.pipeline_success(pipeline_context)
else:
yield DagsterEvent.pipeline_failure(pipeline_context)
def _pipeline_execution_iterator(pipeline_context,
execution_plan,
retries=None):
'''A complete execution of a pipeline. Yields pipeline start, success,
and failure events.
Args:
pipeline_context (SystemPipelineExecutionContext):
execution_plan (ExecutionPlan):
retries (None): Must be None. This is to align the signature of
`_pipeline_execution_iterator` with that of
`dagster.core.execution.plan.execute_plan.inner_plan_execution_iterator` so the same
machinery in _ExecuteRunWithPlanIterable can call them without unpleasant workarounds.
(Default: None)
'''
check.inst_param(pipeline_context, 'pipeline_context',
SystemPipelineExecutionContext)
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
check.invariant(
retries is None,
'Programming error: Retries not supported in _pipeline_execution_iterator'
)
yield DagsterEvent.pipeline_start(pipeline_context)
steps_started = set([])
pipeline_success = True
generator_closed = False
try:
for event in pipeline_context.executor.execute(pipeline_context,
execution_plan):
if event.is_step_start:
steps_started.add(event.step_key)
if event.is_step_success:
if event.step_key not in steps_started:
pipeline_success = False
else:
steps_started.remove(event.step_key)
if event.is_step_failure:
pipeline_success = False
yield event
except GeneratorExit:
# Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed
# (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/).
generator_closed = True
pipeline_success = False
raise
except (Exception, KeyboardInterrupt):
pipeline_success = False
raise # finally block will run before this is re-raised
finally:
if steps_started:
pipeline_success = False
if pipeline_success:
event = DagsterEvent.pipeline_success(pipeline_context)
else:
event = DagsterEvent.pipeline_failure(pipeline_context)
if not generator_closed:
yield event
def _execute_pipeline_iterator(context_or_failure_event):
# Due to use of context managers, if the user land code in context or resource init fails
# we can get either a pipeline_context or the failure event here.
if (isinstance(context_or_failure_event, DagsterEvent)
and context_or_failure_event.event_type
== DagsterEventType.PIPELINE_INIT_FAILURE):
yield context_or_failure_event
return
pipeline_context = context_or_failure_event
check.inst_param(pipeline_context, 'pipeline_context',
SystemPipelineExecutionContext)
yield DagsterEvent.pipeline_start(pipeline_context)
execution_plan = ExecutionPlan.build(
pipeline_context.pipeline_def,
pipeline_context.environment_config,
pipeline_context.mode_def,
)
steps = execution_plan.topological_steps()
if not steps:
pipeline_context.log.debug(
'Pipeline {pipeline} has no nodes and no execution will happen'.
format(pipeline=pipeline_context.pipeline_def.display_name))
yield DagsterEvent.pipeline_success(pipeline_context)
return
_setup_reexecution(pipeline_context.run_config, pipeline_context,
execution_plan)
pipeline_context.log.debug(
'About to execute the compute node graph in the following order {order}'
.format(order=[step.key for step in steps]))
check.invariant(
len([
step_input for step_input in steps[0].step_inputs
if step_input.is_from_output
]) == 0)
pipeline_success = True
try:
for event in invoke_executor_on_plan(
pipeline_context, execution_plan,
pipeline_context.run_config.step_keys_to_execute):
if event.is_step_failure:
pipeline_success = False
yield event
finally:
if pipeline_success:
yield DagsterEvent.pipeline_success(pipeline_context)
else:
yield DagsterEvent.pipeline_failure(pipeline_context)
def _pipeline_execution_iterator(pipeline_context, execution_plan):
"""A complete execution of a pipeline. Yields pipeline start, success,
and failure events.
Args:
pipeline_context (SystemPipelineExecutionContext):
execution_plan (ExecutionPlan):
"""
check.inst_param(pipeline_context, "pipeline_context",
SystemPipelineExecutionContext)
check.inst_param(execution_plan, "execution_plan", ExecutionPlan)
yield DagsterEvent.pipeline_start(pipeline_context)
steps_started = set([])
pipeline_success_ref = BoolRef(True)
generator_closed = False
try:
for event in _core_execution_iterator(pipeline_context, execution_plan,
steps_started,
pipeline_success_ref):
yield event
except GeneratorExit:
# Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed
# (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/).
generator_closed = True
pipeline_success_ref.value = False
raise
finally:
if steps_started:
pipeline_success_ref.value = False
if pipeline_success_ref.value:
event = DagsterEvent.pipeline_success(pipeline_context)
else:
event = DagsterEvent.pipeline_failure(pipeline_context)
if not generator_closed:
yield event
def pipeline_execution_iterator(
pipeline_context: PlanOrchestrationContext,
execution_plan: ExecutionPlan) -> Iterator[DagsterEvent]:
"""A complete execution of a pipeline. Yields pipeline start, success,
and failure events.
Args:
pipeline_context (PlanOrchestrationContext):
execution_plan (ExecutionPlan):
"""
yield DagsterEvent.pipeline_start(pipeline_context)
pipeline_exception_info = None
pipeline_canceled_info = None
failed_steps = []
generator_closed = False
try:
for event in pipeline_context.executor.execute(pipeline_context,
execution_plan):
if event.is_step_failure:
failed_steps.append(event.step_key)
yield event
except GeneratorExit:
# Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed
# (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/).
generator_closed = True
pipeline_exception_info = serializable_error_info_from_exc_info(
sys.exc_info())
if pipeline_context.raise_on_error:
raise
except (KeyboardInterrupt, DagsterExecutionInterruptedError):
pipeline_canceled_info = serializable_error_info_from_exc_info(
sys.exc_info())
if pipeline_context.raise_on_error:
raise
except Exception: # pylint: disable=broad-except
pipeline_exception_info = serializable_error_info_from_exc_info(
sys.exc_info())
if pipeline_context.raise_on_error:
raise # finally block will run before this is re-raised
finally:
if pipeline_canceled_info:
reloaded_run = pipeline_context.instance.get_run_by_id(
pipeline_context.run_id)
if reloaded_run and reloaded_run.status == PipelineRunStatus.CANCELING:
event = DagsterEvent.pipeline_canceled(pipeline_context,
pipeline_canceled_info)
else:
event = DagsterEvent.pipeline_failure(
pipeline_context,
"Execution was interrupted unexpectedly. "
"No user initiated termination request was found, treating as failure.",
pipeline_canceled_info,
)
elif pipeline_exception_info:
event = DagsterEvent.pipeline_failure(
pipeline_context,
"An exception was thrown during execution.",
pipeline_exception_info,
)
elif failed_steps:
event = DagsterEvent.pipeline_failure(
pipeline_context,
"Steps failed: {}.".format(failed_steps),
)
else:
event = DagsterEvent.pipeline_success(pipeline_context)
if not generator_closed:
yield event
def host_mode_execution_context_event_generator(
pipeline,
execution_plan,
run_config,
pipeline_run,
instance,
raise_on_error,
executor_defs,
output_capture,
resume_from_failure: bool = False,
):
check.inst_param(execution_plan, "execution_plan", ExecutionPlan)
check.inst_param(pipeline, "pipeline", ReconstructablePipeline)
check.dict_param(run_config, "run_config", key_type=str)
check.inst_param(pipeline_run, "pipeline_run", PipelineRun)
check.inst_param(instance, "instance", DagsterInstance)
executor_defs = check.list_param(executor_defs,
"executor_defs",
of_type=ExecutorDefinition)
check.bool_param(raise_on_error, "raise_on_error")
check.invariant(output_capture is None)
execution_context = None
loggers = []
for (logger_def, logger_config) in default_system_loggers():
loggers.append(
logger_def.logger_fn(
InitLoggerContext(
logger_config,
pipeline_def=None,
logger_def=logger_def,
run_id=pipeline_run.run_id,
)))
log_manager = DagsterLogManager.create(loggers=loggers,
pipeline_run=pipeline_run,
instance=instance)
try:
executor = _get_host_mode_executor(pipeline, run_config, executor_defs,
instance)
execution_context = PlanOrchestrationContext(
plan_data=PlanData(
pipeline=pipeline,
pipeline_run=pipeline_run,
instance=instance,
execution_plan=execution_plan,
raise_on_error=raise_on_error,
retry_mode=executor.retries,
),
log_manager=log_manager,
executor=executor,
output_capture=None,
resume_from_failure=resume_from_failure,
)
yield execution_context
except DagsterError as dagster_error:
if execution_context is None:
user_facing_exc_info = (
# pylint does not know original_exc_info exists is is_user_code_error is true
# pylint: disable=no-member
dagster_error.original_exc_info # type: ignore
if dagster_error.is_user_code_error else sys.exc_info())
error_info = serializable_error_info_from_exc_info(
user_facing_exc_info)
event = DagsterEvent.pipeline_failure(
pipeline_context_or_name=pipeline_run.pipeline_name,
context_msg=
(f'Pipeline failure during initialization for pipeline "{pipeline_run.pipeline_name}". '
"This may be due to a failure in initializing the executor or one of the loggers."
),
error_info=error_info,
)
log_manager.log_dagster_event(
level=logging.ERROR,
msg=event.message,
dagster_event=event # type: ignore
)
yield event
else:
# pipeline teardown failure
raise dagster_error
if raise_on_error:
raise dagster_error
def orchestration_context_event_generator(
pipeline: IPipeline,
execution_plan: ExecutionPlan,
run_config: Dict[str, Any],
pipeline_run: PipelineRun,
instance: DagsterInstance,
raise_on_error: bool,
executor_defs: Optional[List[ExecutorDefinition]],
output_capture: Optional[Dict["StepOutputHandle", Any]],
) -> Generator[Union[DagsterEvent, PlanOrchestrationContext], None, None]:
check.invariant(executor_defs is None)
context_creation_data = create_context_creation_data(
pipeline,
execution_plan,
run_config,
pipeline_run,
instance,
)
log_manager = create_log_manager(context_creation_data)
try:
executor = create_executor(context_creation_data)
execution_context = PlanOrchestrationContext(
plan_data=create_plan_data(context_creation_data, raise_on_error,
executor.retries),
log_manager=log_manager,
executor=executor,
output_capture=output_capture,
)
_validate_plan_with_context(execution_context, execution_plan)
yield execution_context
except DagsterError as dagster_error:
dagster_error = cast(DagsterUserCodeExecutionError, dagster_error)
user_facing_exc_info = (
# pylint does not know original_exc_info exists is is_user_code_error is true
# pylint: disable=no-member
dagster_error.original_exc_info
if dagster_error.is_user_code_error else sys.exc_info())
error_info = serializable_error_info_from_exc_info(
user_facing_exc_info)
event = DagsterEvent.pipeline_failure(
pipeline_context_or_name=pipeline_run.pipeline_name,
context_msg=
(f'Pipeline failure during initialization for pipeline "{pipeline_run.pipeline_name}". '
"This may be due to a failure in initializing the executor or one of the loggers."
),
error_info=error_info,
)
log_manager.error(
event.message,
dagster_event=event,
pipeline_name=pipeline_run.pipeline_name,
)
yield event
if raise_on_error:
raise dagster_error
