def _user_event_sequence_for_step_compute_fn(
step_context: StepExecutionContext,
evaluated_inputs: Dict[str, Any]) -> Iterator[SolidOutputUnion]:
check.inst_param(step_context, "step_context", StepExecutionContext)
check.dict_param(evaluated_inputs, "evaluated_inputs", key_type=str)
gen = execute_core_compute(
step_context,
evaluated_inputs,
step_context.solid_def.compute_fn,
)
for event in iterate_with_context(
lambda: solid_execution_error_boundary(
DagsterExecutionStepExecutionError,
msg_fn=lambda:
f'Error occurred while executing solid "{step_context.solid.name}":',
step_context=step_context,
step_key=step_context.step.key,
solid_def_name=step_context.solid_def.name,
solid_name=step_context.solid.name,
),
gen,
):
yield event
def _user_event_sequence_for_step_compute_fn(
step_context: SystemStepExecutionContext,
evaluated_inputs: Dict[str, Any]) -> Iterator[SolidOutputUnion]:
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.dict_param(evaluated_inputs, "evaluated_inputs", key_type=str)
gen = execute_core_compute(
step_context.for_compute(),
evaluated_inputs,
step_context.solid_def.compute_fn,
)
for event in iterate_with_context(
lambda: user_code_error_boundary(
DagsterExecutionStepExecutionError,
control_flow_exceptions=[Failure, RetryRequested],
msg_fn=lambda: """Error occurred during the execution of step:
step key: "{key}"
solid invocation: "{solid}"
solid definition: "{solid_def}"
""".format(
key=step_context.step.key,
solid_def=step_context.solid_def.name,
solid=step_context.solid.name,
),
step_key=step_context.step.key,
solid_def_name=step_context.solid_def.name,
solid_name=step_context.solid.name,
),
gen,
):
yield event
def core_dagster_event_sequence_for_step(
step_context: StepExecutionContext,
) -> Iterator[DagsterEvent]:
"""
Execute the step within the step_context argument given the in-memory
events. This function yields a sequence of DagsterEvents, but without
catching any exceptions that have bubbled up during the computation
of the step.
"""
check.inst_param(step_context, "step_context", StepExecutionContext)
if step_context.previous_attempt_count > 0:
yield DagsterEvent.step_restarted_event(step_context, step_context.previous_attempt_count)
else:
yield DagsterEvent.step_start_event(step_context)
inputs = {}
for step_input in step_context.step.step_inputs:
input_def = step_input.source.get_input_def(step_context.pipeline_def)
dagster_type = input_def.dagster_type
if dagster_type.kind == DagsterTypeKind.NOTHING:
continue
for event_or_input_value in ensure_gen(step_input.source.load_input_object(step_context)):
if isinstance(event_or_input_value, DagsterEvent):
yield event_or_input_value
else:
check.invariant(step_input.name not in inputs)
inputs[step_input.name] = event_or_input_value
for input_name, input_value in inputs.items():
for evt in check.generator(
_type_checked_event_sequence_for_input(step_context, input_name, input_value)
):
yield evt
input_lineage = step_context.get_input_lineage()
# The core execution loop expects a compute generator in a specific format: a generator that
# takes a context and dictionary of inputs as input, yields output events. If a solid definition
# was generated from the @solid or @lambda_solid decorator, then compute_fn needs to be coerced
# into this format. If the solid definition was created directly, then it is expected that the
# compute_fn is already in this format.
if isinstance(step_context.solid_def.compute_fn, DecoratedSolidFunction):
core_gen = create_solid_compute_wrapper(step_context.solid_def)
else:
core_gen = step_context.solid_def.compute_fn
with time_execution_scope() as timer_result:
user_event_sequence = check.generator(
execute_core_compute(
step_context,
inputs,
core_gen,
)
)
# It is important for this loop to be indented within the
# timer block above in order for time to be recorded accurately.
for user_event in check.generator(
_step_output_error_checked_user_event_sequence(step_context, user_event_sequence)
):
if isinstance(user_event, DagsterEvent):
yield user_event
elif isinstance(user_event, (Output, DynamicOutput)):
for evt in _type_check_and_store_output(step_context, user_event, input_lineage):
yield evt
# for now, I'm ignoring AssetMaterializations yielded manually, but we might want
# to do something with these in the above path eventually
elif isinstance(user_event, (AssetMaterialization, Materialization)):
yield DagsterEvent.asset_materialization(step_context, user_event, input_lineage)
elif isinstance(user_event, AssetObservation):
yield DagsterEvent.asset_observation(step_context, user_event)
elif isinstance(user_event, ExpectationResult):
yield DagsterEvent.step_expectation_result(step_context, user_event)
else:
check.failed(
"Unexpected event {event}, should have been caught earlier".format(
event=user_event
)
)
yield DagsterEvent.step_success_event(
step_context, StepSuccessData(duration_ms=timer_result.millis)
)