def _store_output(
step_context: SystemStepExecutionContext,
step_output_handle: StepOutputHandle,
output: Union[Output, DynamicOutput],
) -> Iterator[DagsterEvent]:
output_def = step_context.solid_def.output_def_named(
step_output_handle.output_name)
output_manager = step_context.get_io_manager(step_output_handle)
output_context = step_context.get_output_context(step_output_handle)
with user_code_error_boundary(
DagsterExecutionHandleOutputError,
control_flow_exceptions=[Failure, RetryRequested],
msg_fn=lambda:
(f"Error occurred during the the handling of step output:"
f' step key: "{step_context.step.key}"'
f' output name: "{output_context.name}"'),
step_key=step_context.step.key,
output_name=output_context.name,
):
materializations = output_manager.handle_output(
output_context, output.value)
for evt in _materializations_to_events(step_context, step_output_handle,
materializations):
yield evt
yield DagsterEvent.handled_output(
step_context,
output_name=step_output_handle.output_name,
manager_key=output_def.io_manager_key,
message_override=
f'Handled input "{step_output_handle.output_name}" using intermediate storage'
if isinstance(output_manager, IntermediateStorageAdapter) else None,
)
def _set_objects(
step_context: SystemStepExecutionContext,
step_output: StepOutput,
step_output_handle: StepOutputHandle,
output: Union[Output, DynamicOutput],
) -> Iterator[DagsterEvent]:
output_def = step_output.output_def
output_manager = step_context.get_io_manager(step_output_handle)
output_context = step_context.get_output_context(step_output_handle)
with user_code_error_boundary(
DagsterExecutionHandleOutputError,
control_flow_exceptions=[Failure, RetryRequested],
msg_fn=lambda:
(f"Error occurred during the the handling of step output:"
f' step key: "{step_context.step.key}"'
f' output name: "{output_context.name}"'),
step_key=step_context.step.key,
output_name=output_context.name,
):
materializations = output_manager.handle_output(
output_context, output.value)
for evt in _materializations_to_events(step_context, step_output_handle,
materializations):
yield evt
yield DagsterEvent.handled_output(
step_context,
output_name=step_output_handle.output_name,
manager_key=output_def.io_manager_key,
)
def _user_event_sequence_for_step_compute_fn(
step_context: SystemStepExecutionContext,
evaluated_inputs: Dict[str, Any]) -> Iterator[SolidOutputUnion]:
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.dict_param(evaluated_inputs, "evaluated_inputs", key_type=str)
gen = execute_core_compute(
step_context.for_compute(),
evaluated_inputs,
step_context.solid_def.compute_fn,
)
for event in iterate_with_context(
lambda: user_code_error_boundary(
DagsterExecutionStepExecutionError,
control_flow_exceptions=[Failure, RetryRequested],
msg_fn=lambda: """Error occurred during the execution of step:
step key: "{key}"
solid invocation: "{solid}"
solid definition: "{solid_def}"
""".format(
key=step_context.step.key,
solid_def=step_context.solid_def.name,
solid=step_context.solid.name,
),
step_key=step_context.step.key,
solid_def_name=step_context.solid_def.name,
solid_name=step_context.solid.name,
),
gen,
):
yield event
def _type_checked_event_sequence_for_input(
step_context: SystemStepExecutionContext, input_name: str,
input_value: Any) -> Iterator[DagsterEvent]:
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.str_param(input_name, "input_name")
step_input = step_context.step.step_input_named(input_name)
input_def = step_input.source.get_input_def(step_context.pipeline_def)
dagster_type = input_def.dagster_type
with user_code_error_boundary(
DagsterTypeCheckError,
lambda:
(f'Error occurred while type-checking input "{input_name}" of solid '
f'"{str(step_context.step.solid_handle)}", with Python type {type(input_value)} and '
f"Dagster type {dagster_type.display_name}"),
):
type_check = _do_type_check(step_context.for_type(dagster_type),
dagster_type, input_value)
yield _create_step_input_event(step_context,
input_name,
type_check=type_check,
success=type_check.success)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=(f'Type check failed for step input "{input_name}" - '
f'expected type "{dagster_type.display_name}". '
f"Description: {type_check.description}."),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
def _type_check_and_store_output(
step_context: SystemStepExecutionContext, output: Union[DynamicOutput, Output]
) -> Iterator[DagsterEvent]:
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.inst_param(output, "output", (Output, DynamicOutput))
mapping_key = output.mapping_key if isinstance(output, DynamicOutput) else None
step_output_handle = StepOutputHandle(
step_key=step_context.step.key, output_name=output.output_name, mapping_key=mapping_key
)
# If we are executing using the execute_in_process API, then we allow for the outputs of solids
# to be directly captured to a dictionary after they are computed.
if step_context.output_capture is not None:
step_context.output_capture[step_output_handle] = output.value
version = (
step_context.execution_plan.resolve_step_output_versions().get(step_output_handle)
if MEMOIZED_RUN_TAG in step_context.pipeline.get_definition().tags
else None
)
for output_event in _type_check_output(step_context, step_output_handle, output, version):
yield output_event
for evt in _store_output(step_context, step_output_handle, output):
yield evt
for evt in _create_type_materializations(step_context, output.output_name, output.value):
yield evt
def _type_check_output(
step_context: SystemStepExecutionContext,
step_output_handle: StepOutputHandle,
output: Any,
version: Optional[str],
) -> Iterator[DagsterEvent]:
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.inst_param(output, "output", (Output, DynamicOutput))
step_output = step_context.step.step_output_named(output.output_name)
step_output_def = step_context.solid_def.output_def_named(step_output.name)
dagster_type = step_output_def.dagster_type
with user_code_error_boundary(
DagsterTypeCheckError,
lambda: ('In solid "{handle}" the output "{output_name}" received '
"value {output_value} of Python type {output_type} which "
"does not pass the typecheck for Dagster type "
"{dagster_type_name}. Step {step_key}.").format(
handle=str(step_context.step.solid_handle),
output_name=output.output_name,
output_value=output.value,
output_type=type(output.value),
dagster_type_name=dagster_type.display_name,
step_key=step_context.step.key,
),
):
type_check = _do_type_check(step_context.for_type(dagster_type),
dagster_type, output.value)
yield DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=step_output_handle,
type_check_data=TypeCheckData(
success=type_check.success,
label=step_output_handle.output_name,
description=type_check.description if type_check else None,
metadata_entries=type_check.metadata_entries
if type_check else [],
),
version=version,
),
)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=
'Type check failed for step output "{output_name}" - expected type "{dagster_type}".'
.format(
output_name=output.output_name,
dagster_type=dagster_type.display_name,
),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
def _type_check_output(
step_context: SystemStepExecutionContext,
step_output_handle: StepOutputHandle,
output: Any,
version: Optional[str],
) -> Iterator[DagsterEvent]:
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.inst_param(output, "output", (Output, DynamicOutput))
step_output = step_context.step.step_output_named(output.output_name)
step_output_def = step_context.solid_def.output_def_named(step_output.name)
dagster_type = step_output_def.dagster_type
with user_code_error_boundary(
DagsterTypeCheckError,
lambda:
(f'Error occurred while type-checking output "{output.output_name}" of solid '
f'"{str(step_context.step.solid_handle)}", with Python type {type(output.value)} and '
f"Dagster type {dagster_type.display_name}"),
):
type_check = _do_type_check(step_context.for_type(dagster_type),
dagster_type, output.value)
yield DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=step_output_handle,
type_check_data=TypeCheckData(
success=type_check.success,
label=step_output_handle.output_name,
description=type_check.description if type_check else None,
metadata_entries=type_check.metadata_entries
if type_check else [],
),
version=version,
metadata_entries=[
entry for entry in output.metadata_entries
if isinstance(entry, EventMetadataEntry)
],
),
)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=
'Type check failed for step output "{output_name}" - expected type "{dagster_type}".'
.format(
output_name=output.output_name,
dagster_type=dagster_type.display_name,
),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
def _trigger_hook(
step_context: SystemStepExecutionContext,
step_event_list: List[DagsterEvent]) -> Iterator[DagsterEvent]:
"""Trigger hooks and record hook's operatonal events"""
hook_defs = step_context.pipeline_def.get_all_hooks_for_handle(
step_context.solid_handle)
# when the solid doesn't have a hook configured
if hook_defs is None:
return
# when there are multiple hooks set on a solid, the hooks will run sequentially for the solid.
# * we will not able to execute hooks asynchronously until we drop python 2.
for hook_def in hook_defs:
hook_context = step_context.for_hook(hook_def)
try:
with user_code_error_boundary(
HookExecutionError,
lambda:
"Error occurred during the execution of hook_fn triggered for solid "
'"{solid_name}"'.format(solid_name=step_context.solid.name
),
):
hook_execution_result = hook_def.hook_fn(
hook_context, step_event_list)
except HookExecutionError as hook_execution_error:
# catch hook execution error and field a failure event instead of failing the pipeline run
# is_hook_completed = False
yield DagsterEvent.hook_errored(hook_context, hook_execution_error)
continue
check.invariant(
isinstance(hook_execution_result, HookExecutionResult),
("Error in hook {hook_name}: hook unexpectedly returned result {result} of "
"type {type_}. Should be a HookExecutionResult").format(
hook_name=hook_def.name,
result=hook_execution_result,
type_=type(hook_execution_result),
),
)
if hook_execution_result and hook_execution_result.is_skipped:
# when the triggering condition didn't meet in the hook_fn, for instance,
# a @success_hook decorated user-defined function won't run on a failed solid
# but internally the hook_fn still runs, so we yield HOOK_SKIPPED event instead
yield DagsterEvent.hook_skipped(hook_context, hook_def)
else:
# hook_fn finishes successfully
yield DagsterEvent.hook_completed(hook_context, hook_def)
def _type_checked_event_sequence_for_input(
step_context: SystemStepExecutionContext, input_name: str,
input_value: Any) -> Iterator[DagsterEvent]:
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.str_param(input_name, "input_name")
step_input = step_context.step.step_input_named(input_name)
with user_code_error_boundary(
DagsterTypeCheckError,
lambda:
('In solid "{handle}" the input "{input_name}" received '
"value {input_value} of Python type {input_type} which "
"does not pass the typecheck for Dagster type "
"{dagster_type_name}. Step {step_key}.").format(
handle=str(step_context.step.solid_handle),
input_name=input_name,
input_value=input_value,
input_type=type(input_value),
dagster_type_name=step_input.dagster_type.display_name,
step_key=step_context.step.key,
),
):
type_check = _do_type_check(
step_context.for_type(step_input.dagster_type),
step_input.dagster_type,
input_value,
)
yield _create_step_input_event(step_context,
input_name,
type_check=type_check,
success=type_check.success)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=
'Type check failed for step input "{input_name}" - expected type "{dagster_type}".'
.format(
input_name=input_name,
dagster_type=step_input.dagster_type.display_name,
),
metadata_entries=type_check.metadata_entries,
dagster_type=step_input.dagster_type,
)
def _store_output(
step_context: SystemStepExecutionContext,
step_output_handle: StepOutputHandle,
output: Union[Output, DynamicOutput],
input_lineage: List[AssetLineageInfo],
) -> Iterator[DagsterEvent]:
output_def = step_context.solid_def.output_def_named(
step_output_handle.output_name)
output_manager = step_context.get_io_manager(step_output_handle)
output_context = step_context.get_output_context(step_output_handle)
with user_code_error_boundary(
DagsterExecutionHandleOutputError,
control_flow_exceptions=[Failure, RetryRequested],
msg_fn=lambda:
(f'Error occurred while handling output "{output_context.name}" of '
f'step "{step_context.step.key}":'),
step_key=step_context.step.key,
output_name=output_context.name,
):
handle_output_res = output_manager.handle_output(
output_context, output.value)
manager_materializations = []
manager_metadata_entries = []
if handle_output_res is not None:
for elt in ensure_gen(handle_output_res):
if isinstance(elt, AssetMaterialization):
manager_materializations.append(elt)
elif isinstance(elt, (EventMetadataEntry, PartitionMetadataEntry)):
experimental_functionality_warning(
"Yielding metadata from an IOManager's handle_output() function"
)
manager_metadata_entries.append(elt)
else:
raise DagsterInvariantViolationError(
f"IO manager on output {output_def.name} has returned "
f"value {elt} of type {type(elt).__name__}. The return type can only be "
"one of AssetMaterialization, EventMetadataEntry, PartitionMetadataEntry."
)
# do not alter explicitly created AssetMaterializations
for materialization in manager_materializations:
yield DagsterEvent.asset_materialization(step_context, materialization,
input_lineage)
asset_key, partitions = _asset_key_and_partitions_for_output(
output_context, output_def, output_manager)
if asset_key:
for materialization in _get_output_asset_materializations(
asset_key,
partitions,
output,
output_def,
manager_metadata_entries,
):
yield DagsterEvent.asset_materialization(step_context,
materialization,
input_lineage)
yield DagsterEvent.handled_output(
step_context,
output_name=step_output_handle.output_name,
manager_key=output_def.io_manager_key,
message_override=
f'Handled input "{step_output_handle.output_name}" using intermediate storage'
if isinstance(output_manager, IntermediateStorageAdapter) else None,
metadata_entries=[
entry for entry in manager_metadata_entries
if isinstance(entry, EventMetadataEntry)
],
)