def _store_output( step_context: SystemStepExecutionContext, step_output_handle: StepOutputHandle, output: Union[Output, DynamicOutput], ) -> Iterator[DagsterEvent]: output_def = step_context.solid_def.output_def_named( step_output_handle.output_name) output_manager = step_context.get_io_manager(step_output_handle) output_context = step_context.get_output_context(step_output_handle) with user_code_error_boundary( DagsterExecutionHandleOutputError, control_flow_exceptions=[Failure, RetryRequested], msg_fn=lambda: (f"Error occurred during the the handling of step output:" f' step key: "{step_context.step.key}"' f' output name: "{output_context.name}"'), step_key=step_context.step.key, output_name=output_context.name, ): materializations = output_manager.handle_output( output_context, output.value) for evt in _materializations_to_events(step_context, step_output_handle, materializations): yield evt yield DagsterEvent.handled_output( step_context, output_name=step_output_handle.output_name, manager_key=output_def.io_manager_key, message_override= f'Handled input "{step_output_handle.output_name}" using intermediate storage' if isinstance(output_manager, IntermediateStorageAdapter) else None, )
def _set_objects( step_context: SystemStepExecutionContext, step_output: StepOutput, step_output_handle: StepOutputHandle, output: Union[Output, DynamicOutput], ) -> Iterator[DagsterEvent]: output_def = step_output.output_def output_manager = step_context.get_io_manager(step_output_handle) output_context = step_context.get_output_context(step_output_handle) with user_code_error_boundary( DagsterExecutionHandleOutputError, control_flow_exceptions=[Failure, RetryRequested], msg_fn=lambda: (f"Error occurred during the the handling of step output:" f' step key: "{step_context.step.key}"' f' output name: "{output_context.name}"'), step_key=step_context.step.key, output_name=output_context.name, ): materializations = output_manager.handle_output( output_context, output.value) for evt in _materializations_to_events(step_context, step_output_handle, materializations): yield evt yield DagsterEvent.handled_output( step_context, output_name=step_output_handle.output_name, manager_key=output_def.io_manager_key, )
def _user_event_sequence_for_step_compute_fn( step_context: SystemStepExecutionContext, evaluated_inputs: Dict[str, Any]) -> Iterator[SolidOutputUnion]: check.inst_param(step_context, "step_context", SystemStepExecutionContext) check.dict_param(evaluated_inputs, "evaluated_inputs", key_type=str) gen = execute_core_compute( step_context.for_compute(), evaluated_inputs, step_context.solid_def.compute_fn, ) for event in iterate_with_context( lambda: user_code_error_boundary( DagsterExecutionStepExecutionError, control_flow_exceptions=[Failure, RetryRequested], msg_fn=lambda: """Error occurred during the execution of step: step key: "{key}" solid invocation: "{solid}" solid definition: "{solid_def}" """.format( key=step_context.step.key, solid_def=step_context.solid_def.name, solid=step_context.solid.name, ), step_key=step_context.step.key, solid_def_name=step_context.solid_def.name, solid_name=step_context.solid.name, ), gen, ): yield event
def _type_checked_event_sequence_for_input( step_context: SystemStepExecutionContext, input_name: str, input_value: Any) -> Iterator[DagsterEvent]: check.inst_param(step_context, "step_context", SystemStepExecutionContext) check.str_param(input_name, "input_name") step_input = step_context.step.step_input_named(input_name) input_def = step_input.source.get_input_def(step_context.pipeline_def) dagster_type = input_def.dagster_type with user_code_error_boundary( DagsterTypeCheckError, lambda: (f'Error occurred while type-checking input "{input_name}" of solid ' f'"{str(step_context.step.solid_handle)}", with Python type {type(input_value)} and ' f"Dagster type {dagster_type.display_name}"), ): type_check = _do_type_check(step_context.for_type(dagster_type), dagster_type, input_value) yield _create_step_input_event(step_context, input_name, type_check=type_check, success=type_check.success) if not type_check.success: raise DagsterTypeCheckDidNotPass( description=(f'Type check failed for step input "{input_name}" - ' f'expected type "{dagster_type.display_name}". ' f"Description: {type_check.description}."), metadata_entries=type_check.metadata_entries, dagster_type=dagster_type, )
def _type_check_and_store_output( step_context: SystemStepExecutionContext, output: Union[DynamicOutput, Output] ) -> Iterator[DagsterEvent]: check.inst_param(step_context, "step_context", SystemStepExecutionContext) check.inst_param(output, "output", (Output, DynamicOutput)) mapping_key = output.mapping_key if isinstance(output, DynamicOutput) else None step_output_handle = StepOutputHandle( step_key=step_context.step.key, output_name=output.output_name, mapping_key=mapping_key ) # If we are executing using the execute_in_process API, then we allow for the outputs of solids # to be directly captured to a dictionary after they are computed. if step_context.output_capture is not None: step_context.output_capture[step_output_handle] = output.value version = ( step_context.execution_plan.resolve_step_output_versions().get(step_output_handle) if MEMOIZED_RUN_TAG in step_context.pipeline.get_definition().tags else None ) for output_event in _type_check_output(step_context, step_output_handle, output, version): yield output_event for evt in _store_output(step_context, step_output_handle, output): yield evt for evt in _create_type_materializations(step_context, output.output_name, output.value): yield evt
def _type_check_output( step_context: SystemStepExecutionContext, step_output_handle: StepOutputHandle, output: Any, version: Optional[str], ) -> Iterator[DagsterEvent]: check.inst_param(step_context, "step_context", SystemStepExecutionContext) check.inst_param(output, "output", (Output, DynamicOutput)) step_output = step_context.step.step_output_named(output.output_name) step_output_def = step_context.solid_def.output_def_named(step_output.name) dagster_type = step_output_def.dagster_type with user_code_error_boundary( DagsterTypeCheckError, lambda: ('In solid "{handle}" the output "{output_name}" received ' "value {output_value} of Python type {output_type} which " "does not pass the typecheck for Dagster type " "{dagster_type_name}. Step {step_key}.").format( handle=str(step_context.step.solid_handle), output_name=output.output_name, output_value=output.value, output_type=type(output.value), dagster_type_name=dagster_type.display_name, step_key=step_context.step.key, ), ): type_check = _do_type_check(step_context.for_type(dagster_type), dagster_type, output.value) yield DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, type_check_data=TypeCheckData( success=type_check.success, label=step_output_handle.output_name, description=type_check.description if type_check else None, metadata_entries=type_check.metadata_entries if type_check else [], ), version=version, ), ) if not type_check.success: raise DagsterTypeCheckDidNotPass( description= 'Type check failed for step output "{output_name}" - expected type "{dagster_type}".' .format( output_name=output.output_name, dagster_type=dagster_type.display_name, ), metadata_entries=type_check.metadata_entries, dagster_type=dagster_type, )
def _type_check_output( step_context: SystemStepExecutionContext, step_output_handle: StepOutputHandle, output: Any, version: Optional[str], ) -> Iterator[DagsterEvent]: check.inst_param(step_context, "step_context", SystemStepExecutionContext) check.inst_param(output, "output", (Output, DynamicOutput)) step_output = step_context.step.step_output_named(output.output_name) step_output_def = step_context.solid_def.output_def_named(step_output.name) dagster_type = step_output_def.dagster_type with user_code_error_boundary( DagsterTypeCheckError, lambda: (f'Error occurred while type-checking output "{output.output_name}" of solid ' f'"{str(step_context.step.solid_handle)}", with Python type {type(output.value)} and ' f"Dagster type {dagster_type.display_name}"), ): type_check = _do_type_check(step_context.for_type(dagster_type), dagster_type, output.value) yield DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, type_check_data=TypeCheckData( success=type_check.success, label=step_output_handle.output_name, description=type_check.description if type_check else None, metadata_entries=type_check.metadata_entries if type_check else [], ), version=version, metadata_entries=[ entry for entry in output.metadata_entries if isinstance(entry, EventMetadataEntry) ], ), ) if not type_check.success: raise DagsterTypeCheckDidNotPass( description= 'Type check failed for step output "{output_name}" - expected type "{dagster_type}".' .format( output_name=output.output_name, dagster_type=dagster_type.display_name, ), metadata_entries=type_check.metadata_entries, dagster_type=dagster_type, )
def _trigger_hook( step_context: SystemStepExecutionContext, step_event_list: List[DagsterEvent]) -> Iterator[DagsterEvent]: """Trigger hooks and record hook's operatonal events""" hook_defs = step_context.pipeline_def.get_all_hooks_for_handle( step_context.solid_handle) # when the solid doesn't have a hook configured if hook_defs is None: return # when there are multiple hooks set on a solid, the hooks will run sequentially for the solid. # * we will not able to execute hooks asynchronously until we drop python 2. for hook_def in hook_defs: hook_context = step_context.for_hook(hook_def) try: with user_code_error_boundary( HookExecutionError, lambda: "Error occurred during the execution of hook_fn triggered for solid " '"{solid_name}"'.format(solid_name=step_context.solid.name ), ): hook_execution_result = hook_def.hook_fn( hook_context, step_event_list) except HookExecutionError as hook_execution_error: # catch hook execution error and field a failure event instead of failing the pipeline run # is_hook_completed = False yield DagsterEvent.hook_errored(hook_context, hook_execution_error) continue check.invariant( isinstance(hook_execution_result, HookExecutionResult), ("Error in hook {hook_name}: hook unexpectedly returned result {result} of " "type {type_}. Should be a HookExecutionResult").format( hook_name=hook_def.name, result=hook_execution_result, type_=type(hook_execution_result), ), ) if hook_execution_result and hook_execution_result.is_skipped: # when the triggering condition didn't meet in the hook_fn, for instance, # a @success_hook decorated user-defined function won't run on a failed solid # but internally the hook_fn still runs, so we yield HOOK_SKIPPED event instead yield DagsterEvent.hook_skipped(hook_context, hook_def) else: # hook_fn finishes successfully yield DagsterEvent.hook_completed(hook_context, hook_def)
def _type_checked_event_sequence_for_input( step_context: SystemStepExecutionContext, input_name: str, input_value: Any) -> Iterator[DagsterEvent]: check.inst_param(step_context, "step_context", SystemStepExecutionContext) check.str_param(input_name, "input_name") step_input = step_context.step.step_input_named(input_name) with user_code_error_boundary( DagsterTypeCheckError, lambda: ('In solid "{handle}" the input "{input_name}" received ' "value {input_value} of Python type {input_type} which " "does not pass the typecheck for Dagster type " "{dagster_type_name}. Step {step_key}.").format( handle=str(step_context.step.solid_handle), input_name=input_name, input_value=input_value, input_type=type(input_value), dagster_type_name=step_input.dagster_type.display_name, step_key=step_context.step.key, ), ): type_check = _do_type_check( step_context.for_type(step_input.dagster_type), step_input.dagster_type, input_value, ) yield _create_step_input_event(step_context, input_name, type_check=type_check, success=type_check.success) if not type_check.success: raise DagsterTypeCheckDidNotPass( description= 'Type check failed for step input "{input_name}" - expected type "{dagster_type}".' .format( input_name=input_name, dagster_type=step_input.dagster_type.display_name, ), metadata_entries=type_check.metadata_entries, dagster_type=step_input.dagster_type, )
def _store_output( step_context: SystemStepExecutionContext, step_output_handle: StepOutputHandle, output: Union[Output, DynamicOutput], input_lineage: List[AssetLineageInfo], ) -> Iterator[DagsterEvent]: output_def = step_context.solid_def.output_def_named( step_output_handle.output_name) output_manager = step_context.get_io_manager(step_output_handle) output_context = step_context.get_output_context(step_output_handle) with user_code_error_boundary( DagsterExecutionHandleOutputError, control_flow_exceptions=[Failure, RetryRequested], msg_fn=lambda: (f'Error occurred while handling output "{output_context.name}" of ' f'step "{step_context.step.key}":'), step_key=step_context.step.key, output_name=output_context.name, ): handle_output_res = output_manager.handle_output( output_context, output.value) manager_materializations = [] manager_metadata_entries = [] if handle_output_res is not None: for elt in ensure_gen(handle_output_res): if isinstance(elt, AssetMaterialization): manager_materializations.append(elt) elif isinstance(elt, (EventMetadataEntry, PartitionMetadataEntry)): experimental_functionality_warning( "Yielding metadata from an IOManager's handle_output() function" ) manager_metadata_entries.append(elt) else: raise DagsterInvariantViolationError( f"IO manager on output {output_def.name} has returned " f"value {elt} of type {type(elt).__name__}. The return type can only be " "one of AssetMaterialization, EventMetadataEntry, PartitionMetadataEntry." ) # do not alter explicitly created AssetMaterializations for materialization in manager_materializations: yield DagsterEvent.asset_materialization(step_context, materialization, input_lineage) asset_key, partitions = _asset_key_and_partitions_for_output( output_context, output_def, output_manager) if asset_key: for materialization in _get_output_asset_materializations( asset_key, partitions, output, output_def, manager_metadata_entries, ): yield DagsterEvent.asset_materialization(step_context, materialization, input_lineage) yield DagsterEvent.handled_output( step_context, output_name=step_output_handle.output_name, manager_key=output_def.io_manager_key, message_override= f'Handled input "{step_output_handle.output_name}" using intermediate storage' if isinstance(output_manager, IntermediateStorageAdapter) else None, metadata_entries=[ entry for entry in manager_metadata_entries if isinstance(entry, EventMetadataEntry) ], )