def execute(pipeline_context, execution_plan):
check.inst_param(pipeline_context, 'pipeline_context', SystemPipelineExecutionContext)
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
step_levels = execution_plan.execution_step_levels()
intermediates_manager = pipeline_context.intermediates_manager
limit = pipeline_context.executor_config.max_concurrent
step_key_set = set(step.key for step in execution_plan.execution_steps())
yield DagsterEvent.engine_event(
pipeline_context,
'Executing steps using multiprocess engine: parent process (pid: {pid})'.format(
pid=os.getpid()
),
event_specific_data=EngineEventData.multiprocess(
os.getpid(), step_keys_to_execute=step_key_set
),
)
# It would be good to implement a reference tracking algorithm here so we could
# garbage collection results that are no longer needed by any steps
# https://github.com/dagster-io/dagster/issues/811
with time_execution_scope() as timer_result:
for event in copy_required_intermediates_for_execution(
pipeline_context, execution_plan
):
yield event
for step_level in step_levels:
step_contexts_to_execute = []
for step in step_level:
step_context = pipeline_context.for_step(step)
if not intermediates_manager.all_inputs_covered(step_context, step):
uncovered_inputs = intermediates_manager.uncovered_inputs(
step_context, step
)
step_context.log.error(
(
'Not all inputs covered for {step}. Not executing.'
'Output missing for inputs: {uncovered_inputs}'
).format(uncovered_inputs=uncovered_inputs, step=step.key)
)
continue
step_contexts_to_execute.append(step_context)
for step_event in bounded_parallel_executor(step_contexts_to_execute, limit):
yield step_event
yield DagsterEvent.engine_event(
pipeline_context,
'Multiprocess engine: parent process exiting after {duration} (pid: {pid})'.format(
duration=format_duration(timer_result.millis), pid=os.getpid()
),
event_specific_data=EngineEventData.multiprocess(os.getpid()),
)
def _execute_steps_core_loop(step_context, inputs, intermediates_manager):
check.inst_param(step_context, 'step_context', SystemStepExecutionContext)
check.dict_param(inputs, 'inputs', key_type=str)
check.inst_param(intermediates_manager, 'intermediates_manager',
IntermediatesManager)
evaluated_inputs = {}
# do runtime type checks of inputs versus step inputs
for input_name, input_value in inputs.items():
evaluated_inputs[input_name] = _get_evaluated_input(
step_context.step, input_name, input_value)
yield DagsterEvent.step_start_event(step_context)
with time_execution_scope() as timer_result:
step_output_iterator = check.generator(
_iterate_step_outputs_within_boundary(step_context,
evaluated_inputs))
for step_output in check.generator(
_error_check_step_outputs(step_context, step_output_iterator)):
if isinstance(step_output, StepOutputValue):
yield _create_step_output_event(step_context, step_output,
intermediates_manager)
elif isinstance(step_output, Materialization):
yield DagsterEvent.step_materialization(step_context, step_output)
elif isinstance(step_output, ExpectationResult):
yield DagsterEvent.step_expectation_result(step_context,
step_output)
else:
check.failed(
'Unexpected step_output {step_output}, should have been caught earlier'
.format(step_output=step_output))
yield DagsterEvent.step_success_event(
step_context, StepSuccessData(duration_ms=timer_result.millis))
def execute(self, pipeline_context, execution_plan):
check.inst_param(pipeline_context, "pipeline_context",
SystemPipelineExecutionContext)
check.inst_param(execution_plan, "execution_plan", ExecutionPlan)
step_keys_to_execute = execution_plan.step_keys_to_execute
yield DagsterEvent.engine_event(
pipeline_context,
"Executing steps in process (pid: {pid})".format(pid=os.getpid()),
event_specific_data=EngineEventData.in_process(
os.getpid(), step_keys_to_execute),
)
with time_execution_scope() as timer_result:
yield from inner_plan_execution_iterator(pipeline_context,
execution_plan)
yield DagsterEvent.engine_event(
pipeline_context,
"Finished steps in process (pid: {pid}) in {duration_ms}".format(
pid=os.getpid(),
duration_ms=format_duration(timer_result.millis)),
event_specific_data=EngineEventData.in_process(
os.getpid(), step_keys_to_execute),
)
def _execution_step_error_boundary(context, step, msg, **kwargs):
'''
Wraps the execution of user-space code in an error boundary. This places a uniform
policy around an user code invoked by the framework. This ensures that all user
errors are wrapped in the SolidUserCodeExecutionError, and that the original stack
trace of the user error is preserved, so that it can be reported without confusing
framework code in the stack trace, if a tool author wishes to do so. This has
been especially help in a notebooking context.
'''
check.inst_param(context, 'context', RuntimeExecutionContext)
check.str_param(msg, 'msg')
context.events.execution_plan_step_start(step.key)
try:
with time_execution_scope() as timer_result:
yield
context.events.execution_plan_step_success(step.key,
timer_result.millis)
except Exception as e: # pylint: disable=W0703
context.events.execution_plan_step_failure(step.key, sys.exc_info())
stack_trace = get_formatted_stack_trace(e)
context.error(str(e), stack_trace=stack_trace)
if isinstance(e, DagsterError):
raise e
else:
raise_from(
DagsterUserCodeExecutionError(
msg.format(**kwargs),
user_exception=e,
original_exc_info=sys.exc_info()),
e,
)
def test_basic_usage():
with time_execution_scope() as timer_result:
pass
assert timer_result
assert isinstance(timer_result.millis, float)
assert isinstance(timer_result.seconds, float)
def execute(self, pipeline_context, execution_plan):
check.inst_param(pipeline_context, 'pipeline_context', SystemPipelineExecutionContext)
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
step_keys_to_execute = execution_plan.step_keys_to_execute
yield DagsterEvent.engine_event(
pipeline_context,
'Executing steps in process (pid: {pid})'.format(pid=os.getpid()),
event_specific_data=EngineEventData.in_process(os.getpid(), step_keys_to_execute),
)
with time_execution_scope() as timer_result:
for event in inner_plan_execution_iterator(
pipeline_context, execution_plan, self.retries
):
yield event
yield DagsterEvent.engine_event(
pipeline_context,
'Finished steps in process (pid: {pid}) in {duration_ms}'.format(
pid=os.getpid(), duration_ms=format_duration(timer_result.millis)
),
event_specific_data=EngineEventData.in_process(os.getpid(), step_keys_to_execute),
)
def single_resource_event_generator(context, resource_name, resource_def):
try:
msg_fn = lambda: "Error executing resource_fn on ResourceDefinition {name}".format(
name=resource_name)
with user_code_error_boundary(DagsterResourceFunctionError, msg_fn):
try:
with time_execution_scope() as timer_result:
resource_or_gen = resource_def.resource_fn(context)
gen = ensure_gen(resource_or_gen)
resource = next(gen)
resource = InitializedResource(
resource, format_duration(timer_result.millis))
except StopIteration:
check.failed(
"Resource generator {name} must yield one item.".format(
name=resource_name))
yield resource
except DagsterUserCodeExecutionError as dagster_user_error:
raise dagster_user_error
with user_code_error_boundary(DagsterResourceFunctionError, msg_fn):
try:
next(gen)
except StopIteration:
pass
else:
check.failed(
"Resource generator {name} yielded more than one item.".format(
name=resource_name))
def _core_dagster_event_sequence_for_step(step_context):
'''
Execute the step within the step_context argument given the in-memory
events. This function yields a sequence of DagsterEvents, but without
catching any exceptions that have bubbled up during the computation
of the step.
'''
check.inst_param(step_context, 'step_context', SystemStepExecutionContext)
yield DagsterEvent.step_start_event(step_context)
inputs = {}
for input_name, input_value in _input_values_from_intermediates_manager(
step_context).items():
if isinstance(input_value, ObjectStoreOperation):
yield DagsterEvent.object_store_operation(
step_context,
ObjectStoreOperation.serializable(input_value,
value_name=input_name))
inputs[input_name] = input_value.obj
else:
inputs[input_name] = input_value
for input_name, input_value in inputs.items():
for evt in check.generator(
_type_checked_event_sequence_for_input(step_context,
input_name,
input_value)):
yield evt
with time_execution_scope() as timer_result:
user_event_sequence = check.generator(
_user_event_sequence_for_step_compute_fn(step_context, inputs))
# It is important for this loop to be indented within the
# timer block above in order for time to be recorded accurately.
for user_event in check.generator(
_step_output_error_checked_user_event_sequence(
step_context, user_event_sequence)):
if isinstance(user_event, Output):
for evt in _create_step_events_for_output(
step_context, user_event):
yield evt
elif isinstance(user_event, Materialization):
yield DagsterEvent.step_materialization(
step_context, user_event)
elif isinstance(user_event, ExpectationResult):
yield DagsterEvent.step_expectation_result(
step_context, user_event)
else:
check.failed(
'Unexpected event {event}, should have been caught earlier'
.format(event=user_event))
yield DagsterEvent.step_success_event(
step_context, StepSuccessData(duration_ms=timer_result.millis))
def _execute_core_transform(context, compute_node, conf, inputs):
'''
Execute the user-specified transform for the solid. Wrap in an error boundary and do
all relevant logging and metrics tracking
'''
check.inst_param(context, 'context', ExecutionContext)
check.inst_param(compute_node, 'compute_node', ComputeNode)
check.dict_param(inputs, 'inputs', key_type=str)
error_str = 'Error occured during core transform'
solid = compute_node.solid
with context.values({
'solid': solid.name,
'solid_definition': solid.definition.name
}):
context.debug('Executing core transform for solid {solid}.'.format(
solid=solid.name))
with time_execution_scope() as timer_result, \
_user_code_error_boundary(context, error_str):
all_results = list(
_collect_result_list(context, compute_node, conf, inputs))
if len(all_results) != len(solid.definition.output_defs):
emitted_result_names = set([r.output_name for r in all_results])
solid_output_names = set([
output_def.name for output_def in solid.definition.output_defs
])
omitted_outputs = solid_output_names.difference(
emitted_result_names)
context.info('Solid {solid} did not fire outputs {outputs}'.format(
solid=solid.name,
outputs=repr(omitted_outputs),
))
context.debug(
'Finished executing transform for solid {solid}. Time elapsed: {millis:.3f} ms'
.format(
solid=compute_node.solid.name,
millis=timer_result.millis,
),
execution_time_ms=timer_result.millis,
)
for result in all_results:
yield result
def single_resource_event_generator(context, resource_name, resource_def):
try:
msg_fn = lambda: "Error executing resource_fn on ResourceDefinition {name}".format(
name=resource_name)
with user_code_error_boundary(DagsterResourceFunctionError,
msg_fn,
log_manager=context.log):
try:
with time_execution_scope() as timer_result:
resource_or_gen = (resource_def.resource_fn(context)
if is_context_provided(
get_function_params(
resource_def.resource_fn)) else
resource_def.resource_fn())
# Flag for whether resource is generator. This is used to ensure that teardown
# occurs when resources are initialized out of execution.
is_gen = inspect.isgenerator(
resource_or_gen) or isinstance(resource_or_gen,
ContextDecorator)
resource_iter = _wrapped_resource_iterator(resource_or_gen)
resource = next(resource_iter)
resource = InitializedResource(
resource, format_duration(timer_result.millis), is_gen)
except StopIteration:
check.failed(
"Resource generator {name} must yield one item.".format(
name=resource_name))
yield resource
except DagsterUserCodeExecutionError as dagster_user_error:
raise dagster_user_error
with user_code_error_boundary(DagsterResourceFunctionError,
msg_fn,
log_manager=context.log):
try:
next(resource_iter)
except StopIteration:
pass
else:
check.failed(
"Resource generator {name} yielded more than one item.".format(
name=resource_name))
def _core_dagster_event_sequence_for_step(step_context, inputs, intermediates_manager):
'''
Execute the step within the step_context argument given the in-memory
events. This function yields a sequence of DagsterEvents, but without
catching any exceptions that have bubbled up during the computation
of the step.
'''
check.inst_param(step_context, 'step_context', SystemStepExecutionContext)
check.dict_param(inputs, 'inputs', key_type=str)
check.inst_param(intermediates_manager, 'intermediates_manager', IntermediatesManager)
evaluated_inputs = {}
# do runtime type checks of inputs versus step inputs
for input_name, input_value in inputs.items():
evaluated_inputs[input_name] = _get_evaluated_input(
step_context.step, input_name, input_value
)
yield DagsterEvent.step_start_event(step_context)
with time_execution_scope() as timer_result:
event_sequence = check.generator(
_event_sequence_for_step_compute_fn(step_context, evaluated_inputs)
)
# It is important for this loop to be indented within the
# timer block above in order for time to be recorded accurately.
for event in check.generator(
_step_output_error_checked_event_sequence(step_context, event_sequence)
):
if isinstance(event, Result):
yield _create_step_output_event(step_context, event, intermediates_manager)
elif isinstance(event, Materialization):
yield DagsterEvent.step_materialization(step_context, event)
elif isinstance(event, ExpectationResult):
yield DagsterEvent.step_expectation_result(step_context, event)
else:
check.failed(
'Unexpected event {event}, should have been caught earlier'.format(event=event)
)
yield DagsterEvent.step_success_event(
step_context, StepSuccessData(duration_ms=timer_result.millis)
)
def execute(self, pipeline_context, execution_plan):
check.inst_param(pipeline_context, "pipeline_context",
PlanOrchestrationContext)
check.inst_param(execution_plan, "execution_plan", ExecutionPlan)
step_keys_to_execute = execution_plan.step_keys_to_execute
yield DagsterEvent.engine_event(
pipeline_context,
"Executing steps in process (pid: {pid})".format(pid=os.getpid()),
event_specific_data=EngineEventData.in_process(
os.getpid(), step_keys_to_execute),
)
with time_execution_scope() as timer_result:
yield from iter(
ExecuteRunWithPlanIterable(
execution_plan=pipeline_context.execution_plan,
iterator=inner_plan_execution_iterator,
execution_context_manager=PlanExecutionContextManager(
pipeline=pipeline_context.pipeline,
retry_mode=pipeline_context.retry_mode,
execution_plan=pipeline_context.execution_plan,
run_config=pipeline_context.run_config,
pipeline_run=pipeline_context.pipeline_run,
instance=pipeline_context.instance,
raise_on_error=pipeline_context.raise_on_error,
output_capture=pipeline_context.output_capture,
),
))
yield DagsterEvent.engine_event(
pipeline_context,
"Finished steps in process (pid: {pid}) in {duration_ms}".format(
pid=os.getpid(),
duration_ms=format_duration(timer_result.millis)),
event_specific_data=EngineEventData.in_process(
os.getpid(), step_keys_to_execute),
)
def core_dagster_event_sequence_for_step(step_context, prior_attempt_count):
"""
Execute the step within the step_context argument given the in-memory
events. This function yields a sequence of DagsterEvents, but without
catching any exceptions that have bubbled up during the computation
of the step.
"""
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.int_param(prior_attempt_count, "prior_attempt_count")
if prior_attempt_count > 0:
yield DagsterEvent.step_restarted_event(step_context,
prior_attempt_count)
else:
yield DagsterEvent.step_start_event(step_context)
inputs = {}
for input_name, input_value in _input_values_from_intermediate_storage(
step_context):
if isinstance(input_value, ObjectStoreOperation):
yield DagsterEvent.object_store_operation(
step_context,
ObjectStoreOperation.serializable(input_value,
value_name=input_name))
inputs[input_name] = input_value.obj
elif isinstance(input_value, MultipleStepOutputsListWrapper):
for op in input_value:
yield DagsterEvent.object_store_operation(
step_context,
ObjectStoreOperation.serializable(op,
value_name=input_name))
inputs[input_name] = [op.obj for op in input_value]
else:
inputs[input_name] = input_value
for input_name, input_value in inputs.items():
for evt in check.generator(
_type_checked_event_sequence_for_input(step_context,
input_name,
input_value)):
yield evt
with time_execution_scope() as timer_result:
user_event_sequence = check.generator(
_user_event_sequence_for_step_compute_fn(step_context, inputs))
# It is important for this loop to be indented within the
# timer block above in order for time to be recorded accurately.
for user_event in check.generator(
_step_output_error_checked_user_event_sequence(
step_context, user_event_sequence)):
if isinstance(user_event, Output):
for evt in _create_step_events_for_output(
step_context, user_event):
yield evt
elif isinstance(user_event,
(AssetMaterialization, Materialization)):
yield DagsterEvent.step_materialization(
step_context, user_event)
elif isinstance(user_event, ExpectationResult):
yield DagsterEvent.step_expectation_result(
step_context, user_event)
else:
check.failed(
"Unexpected event {event}, should have been caught earlier"
.format(event=user_event))
# We only want to log exactly one step success event or failure event if possible,
# so wait to handle any interrupts (that normally log a failure event) until the success
# event has finished
with delay_interrupts():
yield DagsterEvent.step_success_event(
step_context, StepSuccessData(duration_ms=timer_result.millis))
def core_dagster_event_sequence_for_step(
step_context: SystemStepExecutionContext,
prior_attempt_count: int) -> Iterator[DagsterEvent]:
"""
Execute the step within the step_context argument given the in-memory
events. This function yields a sequence of DagsterEvents, but without
catching any exceptions that have bubbled up during the computation
of the step.
"""
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.int_param(prior_attempt_count, "prior_attempt_count")
if prior_attempt_count > 0:
yield DagsterEvent.step_restarted_event(step_context,
prior_attempt_count)
else:
yield DagsterEvent.step_start_event(step_context)
inputs = {}
for step_input in step_context.step.step_inputs:
input_def = step_input.source.get_input_def(step_context.pipeline_def)
dagster_type = input_def.dagster_type
if dagster_type.kind == DagsterTypeKind.NOTHING:
continue
for event_or_input_value in ensure_gen(
step_input.source.load_input_object(step_context)):
if isinstance(event_or_input_value, DagsterEvent):
yield event_or_input_value
else:
check.invariant(step_input.name not in inputs)
inputs[step_input.name] = event_or_input_value
for input_name, input_value in inputs.items():
for evt in check.generator(
_type_checked_event_sequence_for_input(step_context,
input_name,
input_value)):
yield evt
with time_execution_scope() as timer_result:
user_event_sequence = check.generator(
_user_event_sequence_for_step_compute_fn(step_context, inputs))
# It is important for this loop to be indented within the
# timer block above in order for time to be recorded accurately.
for user_event in check.generator(
_step_output_error_checked_user_event_sequence(
step_context, user_event_sequence)):
if isinstance(user_event, (Output, DynamicOutput)):
for evt in _type_check_and_store_output(
step_context, user_event):
yield evt
elif isinstance(user_event,
(AssetMaterialization, Materialization)):
yield DagsterEvent.step_materialization(
step_context, user_event)
elif isinstance(user_event, ExpectationResult):
yield DagsterEvent.step_expectation_result(
step_context, user_event)
else:
check.failed(
"Unexpected event {event}, should have been caught earlier"
.format(event=user_event))
yield DagsterEvent.step_success_event(
step_context, StepSuccessData(duration_ms=timer_result.millis))
def execute(pipeline_context, execution_plan):
check.inst_param(pipeline_context, 'pipeline_context',
SystemPipelineExecutionContext)
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
intermediates_manager = pipeline_context.intermediates_manager
limit = pipeline_context.executor_config.max_concurrent
yield DagsterEvent.engine_event(
pipeline_context,
'Executing steps using multiprocess engine: parent process (pid: {pid})'
.format(pid=os.getpid()),
event_specific_data=EngineEventData.multiprocess(
os.getpid(),
step_keys_to_execute=execution_plan.step_keys_to_execute),
)
# It would be good to implement a reference tracking algorithm here so we could
# garbage collection results that are no longer needed by any steps
# https://github.com/dagster-io/dagster/issues/811
with time_execution_scope() as timer_result:
active_execution = execution_plan.start(
retries=pipeline_context.executor_config.retries)
active_iters = {}
errors = {}
term_events = {}
stopping = False
while (not stopping
and not active_execution.is_complete) or active_iters:
try:
# start iterators
while len(active_iters) < limit and not stopping:
steps = active_execution.get_steps_to_execute(
limit=(limit - len(active_iters)))
if not steps:
break
for step in steps:
step_context = pipeline_context.for_step(step)
term_events[
step.key] = get_multiprocessing_context(
).Event()
active_iters[
step.key] = execute_step_out_of_process(
step_context, step, errors, term_events)
# process active iterators
empty_iters = []
for key, step_iter in active_iters.items():
try:
event_or_none = next(step_iter)
if event_or_none is None:
continue
else:
yield event_or_none
active_execution.handle_event(event_or_none)
except StopIteration:
empty_iters.append(key)
# clear and mark complete finished iterators
for key in empty_iters:
del active_iters[key]
if term_events[key].is_set():
stopping = True
del term_events[key]
active_execution.verify_complete(pipeline_context, key)
# process skips from failures or uncovered inputs
for event in active_execution.skipped_step_events_iterator(
pipeline_context):
yield event
# In the very small chance that we get interrupted in this coordination section and not
# polling the subprocesses for events - try to clean up gracefully
except KeyboardInterrupt:
yield DagsterEvent.engine_event(
pipeline_context,
'Multiprocess engine: received KeyboardInterrupt - forwarding to active child processes',
EngineEventData.interrupted(list(term_events.keys())),
)
stopping = True
for event in term_events.values():
event.set()
errs = {pid: err for pid, err in errors.items() if err}
if errs:
raise DagsterSubprocessError(
'During multiprocess execution errors occurred in child processes:\n{error_list}'
.format(error_list='\n'.join([
'In process {pid}: {err}'.format(pid=pid,
err=err.to_string())
for pid, err in errs.items()
])),
subprocess_error_infos=list(errs.values()),
)
yield DagsterEvent.engine_event(
pipeline_context,
'Multiprocess engine: parent process exiting after {duration} (pid: {pid})'
.format(duration=format_duration(timer_result.millis),
pid=os.getpid()),
event_specific_data=EngineEventData.multiprocess(os.getpid()),
)
def core_dagster_event_sequence_for_step(step_context, prior_attempt_count):
"""
Execute the step within the step_context argument given the in-memory
events. This function yields a sequence of DagsterEvents, but without
catching any exceptions that have bubbled up during the computation
of the step.
"""
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.int_param(prior_attempt_count, "prior_attempt_count")
if prior_attempt_count > 0:
yield DagsterEvent.step_restarted_event(step_context,
prior_attempt_count)
else:
yield DagsterEvent.step_start_event(step_context)
inputs = {}
for input_name, input_value in _load_input_values(step_context):
# TODO yuhan retire ObjectStoreOperation https://github.com/dagster-io/dagster/issues/3043
if isinstance(input_value, ObjectStoreOperation):
yield DagsterEvent.object_store_operation(
step_context,
ObjectStoreOperation.serializable(input_value,
value_name=input_name))
inputs[input_name] = input_value.obj
elif isinstance(input_value, FanInStepInputValuesWrapper):
final_values = []
for inner_value in input_value:
# inner value is either a store interaction
# TODO yuhan retire ObjectStoreOperation https://github.com/dagster-io/dagster/issues/3043
if isinstance(inner_value, ObjectStoreOperation):
yield DagsterEvent.object_store_operation(
step_context,
ObjectStoreOperation.serializable(
inner_value, value_name=input_name),
)
final_values.append(inner_value.obj)
elif isinstance(inner_value, AssetStoreOperation):
yield DagsterEvent.asset_store_operation(
step_context,
AssetStoreOperation.serializable(inner_value))
final_values.append(inner_value.obj)
# or the value directly
else:
final_values.append(inner_value)
inputs[input_name] = final_values
elif isinstance(input_value, AssetStoreOperation):
yield DagsterEvent.asset_store_operation(
step_context, AssetStoreOperation.serializable(input_value))
inputs[input_name] = input_value.obj
else:
inputs[input_name] = input_value
for input_name, input_value in inputs.items():
for evt in check.generator(
_type_checked_event_sequence_for_input(step_context,
input_name,
input_value)):
yield evt
with time_execution_scope() as timer_result:
user_event_sequence = check.generator(
_user_event_sequence_for_step_compute_fn(step_context, inputs))
# It is important for this loop to be indented within the
# timer block above in order for time to be recorded accurately.
for user_event in check.generator(
_step_output_error_checked_user_event_sequence(
step_context, user_event_sequence)):
if isinstance(user_event, (Output, DynamicOutput)):
for evt in _create_step_events_for_output(
step_context, user_event):
yield evt
elif isinstance(user_event,
(AssetMaterialization, Materialization)):
yield DagsterEvent.step_materialization(
step_context, user_event)
elif isinstance(user_event, ExpectationResult):
yield DagsterEvent.step_expectation_result(
step_context, user_event)
else:
check.failed(
"Unexpected event {event}, should have been caught earlier"
.format(event=user_event))
yield DagsterEvent.step_success_event(
step_context, StepSuccessData(duration_ms=timer_result.millis))
def execute(pipeline_context, execution_plan):
check.inst_param(pipeline_context, 'pipeline_context',
SystemPipelineExecutionContext)
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
yield DagsterEvent.engine_event(
pipeline_context,
'Executing steps in process (pid: {pid})'.format(pid=os.getpid()),
event_specific_data=EngineEventData.in_process(
os.getpid(), execution_plan.step_keys_to_execute),
)
with time_execution_scope() as timer_result:
check.param_invariant(
isinstance(pipeline_context.executor_config, ExecutorConfig),
'pipeline_context',
'Expected executor_config to be ExecutorConfig got {}'.format(
pipeline_context.executor_config),
)
for event in copy_required_intermediates_for_execution(
pipeline_context, execution_plan):
yield event
# It would be good to implement a reference tracking algorithm here to
# garbage collect results that are no longer needed by any steps
# https://github.com/dagster-io/dagster/issues/811
active_execution = execution_plan.start()
while not active_execution.is_complete:
steps = active_execution.get_steps_to_execute(limit=1)
check.invariant(
len(steps) == 1,
'Invariant Violation: expected step to be available to execute'
)
step = steps[0]
step_context = pipeline_context.for_step(step)
check.invariant(
all(
hasattr(step_context.resources, resource_key) for
resource_key in step_context.required_resource_keys),
'expected step context to have all required resources',
)
with mirror_step_io(step_context):
# capture all of the logs for this step
uncovered_inputs = pipeline_context.intermediates_manager.uncovered_inputs(
step_context, step)
if uncovered_inputs:
# In partial pipeline execution, we may end up here without having validated the
# missing dependent outputs were optional
_assert_missing_inputs_optional(
uncovered_inputs, execution_plan, step.key)
step_context.log.info((
'Not all inputs covered for {step}. Not executing. Output missing for '
'inputs: {uncovered_inputs}').format(
uncovered_inputs=uncovered_inputs,
step=step.key))
yield DagsterEvent.step_skipped_event(step_context)
active_execution.mark_skipped(step.key)
continue
for step_event in check.generator(
dagster_event_sequence_for_step(step_context)):
check.inst(step_event, DagsterEvent)
yield step_event
active_execution.handle_event(step_event)
active_execution.verify_complete(pipeline_context,
step.key)
# process skips from failures or uncovered inputs
for event in active_execution.skipped_step_events_iterator(
pipeline_context):
yield event
yield DagsterEvent.engine_event(
pipeline_context,
'Finished steps in process (pid: {pid}) in {duration_ms}'.format(
pid=os.getpid(),
duration_ms=format_duration(timer_result.millis)),
event_specific_data=EngineEventData.in_process(
os.getpid(), execution_plan.step_keys_to_execute),
)
def core_dagster_event_sequence_for_step(
step_context: StepExecutionContext,
) -> Iterator[DagsterEvent]:
"""
Execute the step within the step_context argument given the in-memory
events. This function yields a sequence of DagsterEvents, but without
catching any exceptions that have bubbled up during the computation
of the step.
"""
check.inst_param(step_context, "step_context", StepExecutionContext)
if step_context.previous_attempt_count > 0:
yield DagsterEvent.step_restarted_event(step_context, step_context.previous_attempt_count)
else:
yield DagsterEvent.step_start_event(step_context)
inputs = {}
for step_input in step_context.step.step_inputs:
input_def = step_input.source.get_input_def(step_context.pipeline_def)
dagster_type = input_def.dagster_type
if dagster_type.kind == DagsterTypeKind.NOTHING:
continue
for event_or_input_value in ensure_gen(step_input.source.load_input_object(step_context)):
if isinstance(event_or_input_value, DagsterEvent):
yield event_or_input_value
else:
check.invariant(step_input.name not in inputs)
inputs[step_input.name] = event_or_input_value
for input_name, input_value in inputs.items():
for evt in check.generator(
_type_checked_event_sequence_for_input(step_context, input_name, input_value)
):
yield evt
input_lineage = step_context.get_input_lineage()
# The core execution loop expects a compute generator in a specific format: a generator that
# takes a context and dictionary of inputs as input, yields output events. If a solid definition
# was generated from the @solid or @lambda_solid decorator, then compute_fn needs to be coerced
# into this format. If the solid definition was created directly, then it is expected that the
# compute_fn is already in this format.
if isinstance(step_context.solid_def.compute_fn, DecoratedSolidFunction):
core_gen = create_solid_compute_wrapper(step_context.solid_def)
else:
core_gen = step_context.solid_def.compute_fn
with time_execution_scope() as timer_result:
user_event_sequence = check.generator(
execute_core_compute(
step_context,
inputs,
core_gen,
)
)
# It is important for this loop to be indented within the
# timer block above in order for time to be recorded accurately.
for user_event in check.generator(
_step_output_error_checked_user_event_sequence(step_context, user_event_sequence)
):
if isinstance(user_event, DagsterEvent):
yield user_event
elif isinstance(user_event, (Output, DynamicOutput)):
for evt in _type_check_and_store_output(step_context, user_event, input_lineage):
yield evt
# for now, I'm ignoring AssetMaterializations yielded manually, but we might want
# to do something with these in the above path eventually
elif isinstance(user_event, (AssetMaterialization, Materialization)):
yield DagsterEvent.asset_materialization(step_context, user_event, input_lineage)
elif isinstance(user_event, AssetObservation):
yield DagsterEvent.asset_observation(step_context, user_event)
elif isinstance(user_event, ExpectationResult):
yield DagsterEvent.step_expectation_result(step_context, user_event)
else:
check.failed(
"Unexpected event {event}, should have been caught earlier".format(
event=user_event
)
)
yield DagsterEvent.step_success_event(
step_context, StepSuccessData(duration_ms=timer_result.millis)
)
def execute(pipeline_context, execution_plan):
check.inst_param(pipeline_context, 'pipeline_context',
SystemPipelineExecutionContext)
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
step_levels = execution_plan.execution_step_levels()
step_key_set = set(step.key for step_level in step_levels
for step in step_level)
yield DagsterEvent.engine_event(
pipeline_context,
'Executing steps in process (pid: {pid})'.format(pid=os.getpid()),
event_specific_data=EngineEventData.in_process(
os.getpid(), step_key_set),
)
with time_execution_scope() as timer_result:
check.param_invariant(
isinstance(pipeline_context.executor_config, ExecutorConfig),
'pipeline_context',
'Expected executor_config to be ExecutorConfig got {}'.format(
pipeline_context.executor_config),
)
for event in copy_required_intermediates_for_execution(
pipeline_context, execution_plan):
yield event
failed_or_skipped_steps = set()
# It would be good to implement a reference tracking algorithm here to
# garbage collect results that are no longer needed by any steps
# https://github.com/dagster-io/dagster/issues/811
for step_level in step_levels:
for step in step_level:
step_context = pipeline_context.for_step(step)
with mirror_step_io(step_context):
# capture all of the logs for this step
failed_inputs = []
for step_input in step.step_inputs:
failed_inputs.extend(
failed_or_skipped_steps.intersection(
step_input.dependency_keys))
if failed_inputs:
step_context.log.info((
'Dependencies for step {step} failed: {failed_inputs}. Not executing.'
).format(step=step.key,
failed_inputs=failed_inputs))
failed_or_skipped_steps.add(step.key)
yield DagsterEvent.step_skipped_event(step_context)
continue
uncovered_inputs = pipeline_context.intermediates_manager.uncovered_inputs(
step_context, step)
if uncovered_inputs:
# In partial pipeline execution, we may end up here without having validated the
# missing dependent outputs were optional
_assert_missing_inputs_optional(
uncovered_inputs, execution_plan, step.key)
step_context.log.info((
'Not all inputs covered for {step}. Not executing. Output missing for '
'inputs: {uncovered_inputs}').format(
uncovered_inputs=uncovered_inputs,
step=step.key))
failed_or_skipped_steps.add(step.key)
yield DagsterEvent.step_skipped_event(step_context)
continue
for step_event in check.generator(
dagster_event_sequence_for_step(step_context)):
check.inst(step_event, DagsterEvent)
if step_event.is_step_failure:
failed_or_skipped_steps.add(step.key)
yield step_event
yield DagsterEvent.engine_event(
pipeline_context,
'Finished steps in process (pid: {pid}) in {duration_ms}'.format(
pid=os.getpid(),
duration_ms=format_duration(timer_result.millis)),
event_specific_data=EngineEventData.in_process(
os.getpid(), step_key_set),
)
def execute(self, pipeline_context, execution_plan):
check.inst_param(pipeline_context, "pipeline_context", SystemPipelineExecutionContext)
check.inst_param(execution_plan, "execution_plan", ExecutionPlan)
limit = self.max_concurrent
yield DagsterEvent.engine_event(
pipeline_context,
"Executing steps using multithread executor (pid: {pid})".format(pid=os.getpid()),
event_specific_data=EngineEventData.in_process(os.getpid(), execution_plan.step_keys_to_execute),
)
with time_execution_scope() as timer_result:
with execution_plan.start(retries=self.retries) as active_execution:
active_iters = {}
errors = {}
while not active_execution.is_complete or active_iters:
# start iterators
while len(active_iters) < limit:
steps = active_execution.get_steps_to_execute(limit=(limit - len(active_iters)))
if not steps:
break
for step in steps:
step_context = pipeline_context.for_step(step)
active_iters[step.key] = self.execute_step_in_thread(step.key, step_context, errors)
# process active iterators
empty_iters = []
for key, step_iter in active_iters.items():
try:
event_or_none = next(step_iter)
if event_or_none is None:
continue
yield event_or_none
active_execution.handle_event(event_or_none)
except ThreadCrashException:
serializable_error = serializable_error_info_from_exc_info(sys.exc_info())
yield DagsterEvent.engine_event(
pipeline_context,
f"Multithread executor: thread for step {key} exited unexpectedly",
EngineEventData.engine_error(serializable_error),
)
step_failure_event = DagsterEvent.step_failure_event(
step_context=pipeline_context.for_step(active_execution.get_step_by_key(key)),
step_failure_data=StepFailureData(error=serializable_error, user_failure_data=None),
)
active_execution.handle_event(step_failure_event)
yield step_failure_event
empty_iters.append(key)
except StopIteration:
empty_iters.append(key)
# clear and mark complete finished iterators
for key in empty_iters:
del active_iters[key]
active_execution.verify_complete(pipeline_context, key)
# process skipped and abandoned steps
for event in active_execution.plan_events_iterator(pipeline_context):
yield event
errs = {tid: err for tid, err in errors.items() if err}
if errs:
raise DagsterThreadError(
"During multithread execution errors occurred in threads:\n{error_list}".format(
error_list="\n".join(
[
"In thread {tid}: {err}".format(tid=tid, err=err.to_string())
for tid, err in errs.items()
]
)
),
thread_error_infos=list(errs.values()),
)
yield DagsterEvent.engine_event(
pipeline_context,
"Multithread executor: parent process exiting after {duration} (pid: {pid})".format(
duration=format_duration(timer_result.millis), pid=os.getpid()
),
event_specific_data=EngineEventData.multiprocess(os.getpid()),
)
def execute(self, pipeline_context, execution_plan):
check.inst_param(pipeline_context, "pipeline_context",
SystemPipelineExecutionContext)
check.inst_param(execution_plan, "execution_plan", ExecutionPlan)
limit = self.max_concurrent
yield DagsterEvent.engine_event(
pipeline_context,
"Executing steps using multiprocess engine: parent process (pid: {pid})"
.format(pid=os.getpid()),
event_specific_data=EngineEventData.multiprocess(
os.getpid(),
step_keys_to_execute=execution_plan.step_keys_to_execute),
)
# It would be good to implement a reference tracking algorithm here so we could
# garbage collection results that are no longer needed by any steps
# https://github.com/dagster-io/dagster/issues/811
with time_execution_scope() as timer_result:
with execution_plan.start(
retries=self.retries) as active_execution:
active_iters = {}
errors = {}
term_events = {}
stopping = False
while (not stopping
and not active_execution.is_complete) or active_iters:
try:
# start iterators
while len(active_iters) < limit and not stopping:
steps = active_execution.get_steps_to_execute(
limit=(limit - len(active_iters)))
if not steps:
break
for step in steps:
step_context = pipeline_context.for_step(step)
term_events[step.key] = multiprocessing.Event()
active_iters[
step.
key] = self.execute_step_out_of_process(
step_context, step, errors,
term_events)
# process active iterators
empty_iters = []
for key, step_iter in active_iters.items():
try:
event_or_none = next(step_iter)
if event_or_none is None:
continue
else:
yield event_or_none
active_execution.handle_event(
event_or_none)
except ChildProcessCrashException as crash:
serializable_error = serializable_error_info_from_exc_info(
sys.exc_info())
yield DagsterEvent.engine_event(
pipeline_context,
("Multiprocess executor: child process for step {step_key} "
"unexpectedly exited with code {exit_code}"
).format(step_key=key,
exit_code=crash.exit_code),
EngineEventData.engine_error(
serializable_error),
step_key=key,
)
step_failure_event = DagsterEvent.step_failure_event(
step_context=pipeline_context.for_step(
active_execution.get_step_by_key(key)),
step_failure_data=StepFailureData(
error=serializable_error,
user_failure_data=None),
)
active_execution.handle_event(
step_failure_event)
yield step_failure_event
empty_iters.append(key)
except StopIteration:
empty_iters.append(key)
# clear and mark complete finished iterators
for key in empty_iters:
del active_iters[key]
if term_events[key].is_set():
stopping = True
del term_events[key]
active_execution.verify_complete(
pipeline_context, key)
# process skips from failures or uncovered inputs
for event in active_execution.skipped_step_events_iterator(
pipeline_context):
yield event
# In the very small chance that we get interrupted in this coordination section and not
# polling the subprocesses for events - try to clean up gracefully
except KeyboardInterrupt:
yield DagsterEvent.engine_event(
pipeline_context,
"Multiprocess engine: received KeyboardInterrupt - forwarding to active child processes",
EngineEventData.interrupted(
list(term_events.keys())),
)
stopping = True
for event in term_events.values():
event.set()
errs = {pid: err for pid, err in errors.items() if err}
if errs:
raise DagsterSubprocessError(
"During multiprocess execution errors occurred in child processes:\n{error_list}"
.format(error_list="\n".join([
"In process {pid}: {err}".format(
pid=pid, err=err.to_string())
for pid, err in errs.items()
])),
subprocess_error_infos=list(errs.values()),
)
yield DagsterEvent.engine_event(
pipeline_context,
"Multiprocess engine: parent process exiting after {duration} (pid: {pid})"
.format(duration=format_duration(timer_result.millis),
pid=os.getpid()),
event_specific_data=EngineEventData.multiprocess(os.getpid()),
)
def execute(self, pipeline_context, execution_plan):
check.inst_param(pipeline_context, "pipeline_context",
SystemPipelineExecutionContext)
check.inst_param(execution_plan, "execution_plan", ExecutionPlan)
limit = self.max_concurrent
yield DagsterEvent.engine_event(
pipeline_context,
"Executing steps using multiprocess executor: parent process (pid: {pid})"
.format(pid=os.getpid()),
event_specific_data=EngineEventData.multiprocess(
os.getpid(),
step_keys_to_execute=execution_plan.step_keys_to_execute),
)
# It would be good to implement a reference tracking algorithm here so we could
# garbage collect results that are no longer needed by any steps
# https://github.com/dagster-io/dagster/issues/811
with time_execution_scope() as timer_result:
with execution_plan.start(
retry_mode=self.retries) as active_execution:
active_iters = {}
errors = {}
term_events = {}
stopping = False
while (not stopping
and not active_execution.is_complete) or active_iters:
if active_execution.check_for_interrupts():
yield DagsterEvent.engine_event(
pipeline_context,
"Multiprocess executor: received termination signal - "
"forwarding to active child processes",
EngineEventData.interrupted(
list(term_events.keys())),
)
stopping = True
active_execution.mark_interrupted()
for key, event in term_events.items():
event.set()
# start iterators
while len(active_iters) < limit and not stopping:
steps = active_execution.get_steps_to_execute(
limit=(limit - len(active_iters)))
if not steps:
break
for step in steps:
step_context = pipeline_context.for_step(step)
term_events[step.key] = multiprocessing.Event()
active_iters[
step.key] = self.execute_step_out_of_process(
step_context,
step,
errors,
term_events,
active_execution.get_known_state(),
)
# process active iterators
empty_iters = []
for key, step_iter in active_iters.items():
try:
event_or_none = next(step_iter)
if event_or_none is None:
continue
else:
yield event_or_none
active_execution.handle_event(event_or_none)
except ChildProcessCrashException as crash:
serializable_error = serializable_error_info_from_exc_info(
sys.exc_info())
yield DagsterEvent.engine_event(
pipeline_context,
("Multiprocess executor: child process for step {step_key} "
"unexpectedly exited with code {exit_code}"
).format(step_key=key,
exit_code=crash.exit_code),
EngineEventData.engine_error(
serializable_error),
step_handle=active_execution.get_step_by_key(
key).handle,
)
step_failure_event = DagsterEvent.step_failure_event(
step_context=pipeline_context.for_step(
active_execution.get_step_by_key(key)),
step_failure_data=StepFailureData(
error=serializable_error,
user_failure_data=None),
)
active_execution.handle_event(step_failure_event)
yield step_failure_event
empty_iters.append(key)
except StopIteration:
empty_iters.append(key)
# clear and mark complete finished iterators
for key in empty_iters:
del active_iters[key]
del term_events[key]
active_execution.verify_complete(pipeline_context, key)
# process skipped and abandoned steps
yield from active_execution.plan_events_iterator(
pipeline_context)
errs = {pid: err for pid, err in errors.items() if err}
# After termination starts, raise an interrupted exception once all subprocesses
# have finished cleaning up (and the only errors were from being interrupted)
if (stopping and (not active_iters) and all([
err_info.cls_name == "DagsterExecutionInterruptedError"
for err_info in errs.values()
])):
yield DagsterEvent.engine_event(
pipeline_context,
"Multiprocess executor: interrupted all active child processes",
event_specific_data=EngineEventData(),
)
raise DagsterExecutionInterruptedError()
elif errs:
raise DagsterSubprocessError(
"During multiprocess execution errors occurred in child processes:\n{error_list}"
.format(error_list="\n".join([
"In process {pid}: {err}".format(
pid=pid, err=err.to_string())
for pid, err in errs.items()
])),
subprocess_error_infos=list(errs.values()),
)
yield DagsterEvent.engine_event(
pipeline_context,
"Multiprocess executor: parent process exiting after {duration} (pid: {pid})"
.format(duration=format_duration(timer_result.millis),
pid=os.getpid()),
event_specific_data=EngineEventData.multiprocess(os.getpid()),
)