def _validate_event(event: Any, step_context: StepExecutionContext) -> SolidOutputUnion: if not isinstance( event, ( DynamicOutput, Output, AssetMaterialization, Materialization, ExpectationResult, AssetObservation, DagsterEvent, ), ): raise DagsterInvariantViolationError(( "Compute function for {described_node} yielded a value of type {type_} " "rather than an instance of Output, AssetMaterialization, or ExpectationResult." " Values yielded by {node_type}s must be wrapped in one of these types. If your " "{node_type} has a single output and yields no other events, you may want to use " "`return` instead of `yield` in the body of your {node_type} compute function. If " "you are already using `return`, and you expected to return a value of type " "{type_}, you may be inadvertently returning a generator rather than the value " "you expected.").format( described_node=step_context.describe_op(), type_=type(event), node_type=step_context.solid_def.node_type_str, )) return event
def _type_check_output( step_context: StepExecutionContext, step_output_handle: StepOutputHandle, output: Any, version: Optional[str], ) -> Iterator[DagsterEvent]: check.inst_param(step_context, "step_context", StepExecutionContext) check.inst_param(output, "output", (Output, DynamicOutput)) step_output = step_context.step.step_output_named(output.output_name) step_output_def = step_context.solid_def.output_def_named(step_output.name) dagster_type = step_output_def.dagster_type type_check_context = step_context.for_type(dagster_type) op_label = step_context.describe_op() output_type = type(output.value) with user_code_error_boundary( DagsterTypeCheckError, lambda: (f'Error occurred while type-checking output "{output.output_name}" of {op_label}, with ' f"Python type {output_type} and Dagster type {dagster_type.display_name}" ), log_manager=type_check_context.log, ): type_check = do_type_check(type_check_context, dagster_type, output.value) yield DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, type_check_data=TypeCheckData( success=type_check.success, label=step_output_handle.output_name, description=type_check.description if type_check else None, metadata_entries=type_check.metadata_entries if type_check else [], ), version=version, metadata_entries=[ entry for entry in output.metadata_entries if isinstance(entry, MetadataEntry) ], ), ) if not type_check.success: raise DagsterTypeCheckDidNotPass( description=( f'Type check failed for step output "{output.output_name}" - ' f'expected type "{dagster_type.display_name}". ' f"Description: {type_check.description}"), metadata_entries=type_check.metadata_entries, dagster_type=dagster_type, )
def _yield_compute_results(step_context: StepExecutionContext, inputs: Dict[str, Any], compute_fn: Callable) -> Iterator[SolidOutputUnion]: check.inst_param(step_context, "step_context", StepExecutionContext) context = SolidExecutionContext(step_context) user_event_generator = compute_fn(context, inputs) if isinstance(user_event_generator, Output): raise DagsterInvariantViolationError(( "Compute function for {described_node} returned an Output rather than " "yielding it. The compute_fn of the {node_type} must yield " "its results").format( described_node=step_context.describe_op(), node_type=step_context.solid_def.node_type_str, )) if user_event_generator is None: return if inspect.isasyncgen(user_event_generator): user_event_generator = gen_from_async_gen(user_event_generator) op_label = step_context.describe_op() for event in iterate_with_context( lambda: solid_execution_error_boundary( DagsterExecutionStepExecutionError, msg_fn=lambda: f"Error occurred while executing {op_label}:", step_context=step_context, step_key=step_context.step.key, op_def_name=step_context.solid_def.name, op_name=step_context.solid.name, ), user_event_generator, ): if context.has_events(): yield from context.consume_events() yield _validate_event(event, step_context) if context.has_events(): yield from context.consume_events()
def _trigger_hook( step_context: StepExecutionContext, step_event_list: List[DagsterEvent] ) -> Iterator[DagsterEvent]: """Trigger hooks and record hook's operatonal events""" hook_defs = step_context.pipeline_def.get_all_hooks_for_handle(step_context.solid_handle) # when the solid doesn't have a hook configured if hook_defs is None: return op_label = step_context.describe_op() # when there are multiple hooks set on a solid, the hooks will run sequentially for the solid. # * we will not able to execute hooks asynchronously until we drop python 2. for hook_def in hook_defs: hook_context = step_context.for_hook(hook_def) try: with user_code_error_boundary( HookExecutionError, lambda: f"Error occurred during the execution of hook_fn triggered for {op_label}", log_manager=hook_context.log, ): hook_execution_result = hook_def.hook_fn(hook_context, step_event_list) except HookExecutionError as hook_execution_error: # catch hook execution error and field a failure event instead of failing the pipeline run yield DagsterEvent.hook_errored(step_context, hook_execution_error) continue check.invariant( isinstance(hook_execution_result, HookExecutionResult), ( "Error in hook {hook_name}: hook unexpectedly returned result {result} of " "type {type_}. Should be a HookExecutionResult" ).format( hook_name=hook_def.name, result=hook_execution_result, type_=type(hook_execution_result), ), ) if hook_execution_result and hook_execution_result.is_skipped: # when the triggering condition didn't meet in the hook_fn, for instance, # a @success_hook decorated user-defined function won't run on a failed solid # but internally the hook_fn still runs, so we yield HOOK_SKIPPED event instead yield DagsterEvent.hook_skipped(step_context, hook_def) else: # hook_fn finishes successfully yield DagsterEvent.hook_completed(step_context, hook_def)
def _type_checked_event_sequence_for_input( step_context: StepExecutionContext, input_name: str, input_value: Any, ) -> Iterator[DagsterEvent]: check.inst_param(step_context, "step_context", StepExecutionContext) check.str_param(input_name, "input_name") step_input = step_context.step.step_input_named(input_name) input_def = step_context.solid_def.input_def_named(step_input.name) check.invariant( input_def.name == input_name, f"InputDefinition name does not match, expected {input_name} got {input_def.name}", ) dagster_type = input_def.dagster_type type_check_context = step_context.for_type(dagster_type) input_type = type(input_value) op_label = step_context.describe_op() with user_code_error_boundary( DagsterTypeCheckError, lambda: (f'Error occurred while type-checking input "{input_name}" of {op_label}, with Python ' f"type {input_type} and Dagster type {dagster_type.display_name}"), log_manager=type_check_context.log, ): type_check = do_type_check(type_check_context, dagster_type, input_value) yield _create_step_input_event(step_context, input_name, type_check=type_check, success=type_check.success) if not type_check.success: raise DagsterTypeCheckDidNotPass( description=(f'Type check failed for step input "{input_name}" - ' f'expected type "{dagster_type.display_name}". ' f"Description: {type_check.description}"), metadata_entries=type_check.metadata_entries, dagster_type=dagster_type, )
def _step_output_error_checked_user_event_sequence( step_context: StepExecutionContext, user_event_sequence: Iterator[SolidOutputUnion] ) -> Iterator[SolidOutputUnion]: """ Process the event sequence to check for invariant violations in the event sequence related to Output events emitted from the compute_fn. This consumes and emits an event sequence. """ check.inst_param(step_context, "step_context", StepExecutionContext) check.generator_param(user_event_sequence, "user_event_sequence") step = step_context.step op_label = step_context.describe_op() output_names = list([output_def.name for output_def in step.step_outputs]) for user_event in user_event_sequence: if not isinstance(user_event, (Output, DynamicOutput)): yield user_event continue # do additional processing on Outputs output = user_event if not step.has_step_output(cast(str, output.output_name)): raise DagsterInvariantViolationError( f'Core compute for {op_label} returned an output "{output.output_name}" that does ' f"not exist. The available outputs are {output_names}" ) step_output = step.step_output_named(cast(str, output.output_name)) output_def = step_context.pipeline_def.get_solid(step_output.solid_handle).output_def_named( step_output.name ) if isinstance(output, Output): if step_context.has_seen_output(output.output_name): raise DagsterInvariantViolationError( f'Compute for {op_label} returned an output "{output.output_name}" multiple ' "times" ) if output_def.is_dynamic: raise DagsterInvariantViolationError( f'Compute for {op_label} for output "{output.output_name}" defined as dynamic ' "must yield DynamicOutput, got Output." ) step_context.observe_output(output.output_name) metadata = step_context.get_output_metadata(output.output_name) output = Output( value=output.value, output_name=output.output_name, metadata_entries=output.metadata_entries + normalize_metadata(cast(Dict[str, Any], metadata), []), ) else: if not output_def.is_dynamic: raise DagsterInvariantViolationError( f"Compute for {op_label} yielded a DynamicOutput, but did not use " "DynamicOutputDefinition." ) if step_context.has_seen_output(output.output_name, output.mapping_key): raise DagsterInvariantViolationError( f"Compute for {op_label} yielded a DynamicOutput with mapping_key " f'"{output.mapping_key}" multiple times.' ) step_context.observe_output(output.output_name, output.mapping_key) metadata = step_context.get_output_metadata( output.output_name, mapping_key=output.mapping_key ) output = DynamicOutput( value=output.value, output_name=output.output_name, metadata_entries=output.metadata_entries + normalize_metadata(cast(Dict[str, Any], metadata), []), mapping_key=output.mapping_key, ) yield output for step_output in step.step_outputs: step_output_def = step_context.solid_def.output_def_named(step_output.name) if not step_context.has_seen_output(step_output_def.name) and not step_output_def.optional: if step_output_def.dagster_type.kind == DagsterTypeKind.NOTHING: step_context.log.info( f'Emitting implicit Nothing for output "{step_output_def.name}" on {op_label}' ) yield Output(output_name=step_output_def.name, value=None) elif not step_output_def.is_dynamic: raise DagsterStepOutputNotFoundError( ( f"Core compute for {op_label} did not return an output for non-optional " f'output "{step_output_def.name}"' ), step_key=step.key, output_name=step_output_def.name, )