def _validate_event(event: Any,
step_context: StepExecutionContext) -> SolidOutputUnion:
if not isinstance(
event,
(
DynamicOutput,
Output,
AssetMaterialization,
Materialization,
ExpectationResult,
AssetObservation,
DagsterEvent,
),
):
raise DagsterInvariantViolationError((
"Compute function for {described_node} yielded a value of type {type_} "
"rather than an instance of Output, AssetMaterialization, or ExpectationResult."
" Values yielded by {node_type}s must be wrapped in one of these types. If your "
"{node_type} has a single output and yields no other events, you may want to use "
"`return` instead of `yield` in the body of your {node_type} compute function. If "
"you are already using `return`, and you expected to return a value of type "
"{type_}, you may be inadvertently returning a generator rather than the value "
"you expected.").format(
described_node=step_context.describe_op(),
type_=type(event),
node_type=step_context.solid_def.node_type_str,
))
return event
def _type_check_output(
step_context: StepExecutionContext,
step_output_handle: StepOutputHandle,
output: Any,
version: Optional[str],
) -> Iterator[DagsterEvent]:
check.inst_param(step_context, "step_context", StepExecutionContext)
check.inst_param(output, "output", (Output, DynamicOutput))
step_output = step_context.step.step_output_named(output.output_name)
step_output_def = step_context.solid_def.output_def_named(step_output.name)
dagster_type = step_output_def.dagster_type
type_check_context = step_context.for_type(dagster_type)
op_label = step_context.describe_op()
output_type = type(output.value)
with user_code_error_boundary(
DagsterTypeCheckError,
lambda:
(f'Error occurred while type-checking output "{output.output_name}" of {op_label}, with '
f"Python type {output_type} and Dagster type {dagster_type.display_name}"
),
log_manager=type_check_context.log,
):
type_check = do_type_check(type_check_context, dagster_type,
output.value)
yield DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=step_output_handle,
type_check_data=TypeCheckData(
success=type_check.success,
label=step_output_handle.output_name,
description=type_check.description if type_check else None,
metadata_entries=type_check.metadata_entries
if type_check else [],
),
version=version,
metadata_entries=[
entry for entry in output.metadata_entries
if isinstance(entry, MetadataEntry)
],
),
)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=(
f'Type check failed for step output "{output.output_name}" - '
f'expected type "{dagster_type.display_name}". '
f"Description: {type_check.description}"),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
def _yield_compute_results(step_context: StepExecutionContext,
inputs: Dict[str, Any],
compute_fn: Callable) -> Iterator[SolidOutputUnion]:
check.inst_param(step_context, "step_context", StepExecutionContext)
context = SolidExecutionContext(step_context)
user_event_generator = compute_fn(context, inputs)
if isinstance(user_event_generator, Output):
raise DagsterInvariantViolationError((
"Compute function for {described_node} returned an Output rather than "
"yielding it. The compute_fn of the {node_type} must yield "
"its results").format(
described_node=step_context.describe_op(),
node_type=step_context.solid_def.node_type_str,
))
if user_event_generator is None:
return
if inspect.isasyncgen(user_event_generator):
user_event_generator = gen_from_async_gen(user_event_generator)
op_label = step_context.describe_op()
for event in iterate_with_context(
lambda: solid_execution_error_boundary(
DagsterExecutionStepExecutionError,
msg_fn=lambda: f"Error occurred while executing {op_label}:",
step_context=step_context,
step_key=step_context.step.key,
op_def_name=step_context.solid_def.name,
op_name=step_context.solid.name,
),
user_event_generator,
):
if context.has_events():
yield from context.consume_events()
yield _validate_event(event, step_context)
if context.has_events():
yield from context.consume_events()
def _trigger_hook(
step_context: StepExecutionContext, step_event_list: List[DagsterEvent]
) -> Iterator[DagsterEvent]:
"""Trigger hooks and record hook's operatonal events"""
hook_defs = step_context.pipeline_def.get_all_hooks_for_handle(step_context.solid_handle)
# when the solid doesn't have a hook configured
if hook_defs is None:
return
op_label = step_context.describe_op()
# when there are multiple hooks set on a solid, the hooks will run sequentially for the solid.
# * we will not able to execute hooks asynchronously until we drop python 2.
for hook_def in hook_defs:
hook_context = step_context.for_hook(hook_def)
try:
with user_code_error_boundary(
HookExecutionError,
lambda: f"Error occurred during the execution of hook_fn triggered for {op_label}",
log_manager=hook_context.log,
):
hook_execution_result = hook_def.hook_fn(hook_context, step_event_list)
except HookExecutionError as hook_execution_error:
# catch hook execution error and field a failure event instead of failing the pipeline run
yield DagsterEvent.hook_errored(step_context, hook_execution_error)
continue
check.invariant(
isinstance(hook_execution_result, HookExecutionResult),
(
"Error in hook {hook_name}: hook unexpectedly returned result {result} of "
"type {type_}. Should be a HookExecutionResult"
).format(
hook_name=hook_def.name,
result=hook_execution_result,
type_=type(hook_execution_result),
),
)
if hook_execution_result and hook_execution_result.is_skipped:
# when the triggering condition didn't meet in the hook_fn, for instance,
# a @success_hook decorated user-defined function won't run on a failed solid
# but internally the hook_fn still runs, so we yield HOOK_SKIPPED event instead
yield DagsterEvent.hook_skipped(step_context, hook_def)
else:
# hook_fn finishes successfully
yield DagsterEvent.hook_completed(step_context, hook_def)
def _type_checked_event_sequence_for_input(
step_context: StepExecutionContext,
input_name: str,
input_value: Any,
) -> Iterator[DagsterEvent]:
check.inst_param(step_context, "step_context", StepExecutionContext)
check.str_param(input_name, "input_name")
step_input = step_context.step.step_input_named(input_name)
input_def = step_context.solid_def.input_def_named(step_input.name)
check.invariant(
input_def.name == input_name,
f"InputDefinition name does not match, expected {input_name} got {input_def.name}",
)
dagster_type = input_def.dagster_type
type_check_context = step_context.for_type(dagster_type)
input_type = type(input_value)
op_label = step_context.describe_op()
with user_code_error_boundary(
DagsterTypeCheckError,
lambda:
(f'Error occurred while type-checking input "{input_name}" of {op_label}, with Python '
f"type {input_type} and Dagster type {dagster_type.display_name}"),
log_manager=type_check_context.log,
):
type_check = do_type_check(type_check_context, dagster_type,
input_value)
yield _create_step_input_event(step_context,
input_name,
type_check=type_check,
success=type_check.success)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=(f'Type check failed for step input "{input_name}" - '
f'expected type "{dagster_type.display_name}". '
f"Description: {type_check.description}"),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
def _step_output_error_checked_user_event_sequence(
step_context: StepExecutionContext, user_event_sequence: Iterator[SolidOutputUnion]
) -> Iterator[SolidOutputUnion]:
"""
Process the event sequence to check for invariant violations in the event
sequence related to Output events emitted from the compute_fn.
This consumes and emits an event sequence.
"""
check.inst_param(step_context, "step_context", StepExecutionContext)
check.generator_param(user_event_sequence, "user_event_sequence")
step = step_context.step
op_label = step_context.describe_op()
output_names = list([output_def.name for output_def in step.step_outputs])
for user_event in user_event_sequence:
if not isinstance(user_event, (Output, DynamicOutput)):
yield user_event
continue
# do additional processing on Outputs
output = user_event
if not step.has_step_output(cast(str, output.output_name)):
raise DagsterInvariantViolationError(
f'Core compute for {op_label} returned an output "{output.output_name}" that does '
f"not exist. The available outputs are {output_names}"
)
step_output = step.step_output_named(cast(str, output.output_name))
output_def = step_context.pipeline_def.get_solid(step_output.solid_handle).output_def_named(
step_output.name
)
if isinstance(output, Output):
if step_context.has_seen_output(output.output_name):
raise DagsterInvariantViolationError(
f'Compute for {op_label} returned an output "{output.output_name}" multiple '
"times"
)
if output_def.is_dynamic:
raise DagsterInvariantViolationError(
f'Compute for {op_label} for output "{output.output_name}" defined as dynamic '
"must yield DynamicOutput, got Output."
)
step_context.observe_output(output.output_name)
metadata = step_context.get_output_metadata(output.output_name)
output = Output(
value=output.value,
output_name=output.output_name,
metadata_entries=output.metadata_entries
+ normalize_metadata(cast(Dict[str, Any], metadata), []),
)
else:
if not output_def.is_dynamic:
raise DagsterInvariantViolationError(
f"Compute for {op_label} yielded a DynamicOutput, but did not use "
"DynamicOutputDefinition."
)
if step_context.has_seen_output(output.output_name, output.mapping_key):
raise DagsterInvariantViolationError(
f"Compute for {op_label} yielded a DynamicOutput with mapping_key "
f'"{output.mapping_key}" multiple times.'
)
step_context.observe_output(output.output_name, output.mapping_key)
metadata = step_context.get_output_metadata(
output.output_name, mapping_key=output.mapping_key
)
output = DynamicOutput(
value=output.value,
output_name=output.output_name,
metadata_entries=output.metadata_entries
+ normalize_metadata(cast(Dict[str, Any], metadata), []),
mapping_key=output.mapping_key,
)
yield output
for step_output in step.step_outputs:
step_output_def = step_context.solid_def.output_def_named(step_output.name)
if not step_context.has_seen_output(step_output_def.name) and not step_output_def.optional:
if step_output_def.dagster_type.kind == DagsterTypeKind.NOTHING:
step_context.log.info(
f'Emitting implicit Nothing for output "{step_output_def.name}" on {op_label}'
)
yield Output(output_name=step_output_def.name, value=None)
elif not step_output_def.is_dynamic:
raise DagsterStepOutputNotFoundError(
(
f"Core compute for {op_label} did not return an output for non-optional "
f'output "{step_output_def.name}"'
),
step_key=step.key,
output_name=step_output_def.name,
)