def _type_check_output( step_context: StepExecutionContext, step_output_handle: StepOutputHandle, output: Any, version: Optional[str], ) -> Iterator[DagsterEvent]: check.inst_param(step_context, "step_context", StepExecutionContext) check.inst_param(output, "output", (Output, DynamicOutput)) step_output = step_context.step.step_output_named(output.output_name) step_output_def = step_context.solid_def.output_def_named(step_output.name) dagster_type = step_output_def.dagster_type type_check_context = step_context.for_type(dagster_type) op_label = step_context.describe_op() output_type = type(output.value) with user_code_error_boundary( DagsterTypeCheckError, lambda: (f'Error occurred while type-checking output "{output.output_name}" of {op_label}, with ' f"Python type {output_type} and Dagster type {dagster_type.display_name}" ), log_manager=type_check_context.log, ): type_check = do_type_check(type_check_context, dagster_type, output.value) yield DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, type_check_data=TypeCheckData( success=type_check.success, label=step_output_handle.output_name, description=type_check.description if type_check else None, metadata_entries=type_check.metadata_entries if type_check else [], ), version=version, metadata_entries=[ entry for entry in output.metadata_entries if isinstance(entry, MetadataEntry) ], ), ) if not type_check.success: raise DagsterTypeCheckDidNotPass( description=( f'Type check failed for step output "{output.output_name}" - ' f'expected type "{dagster_type.display_name}". ' f"Description: {type_check.description}"), metadata_entries=type_check.metadata_entries, dagster_type=dagster_type, )
def _type_check_output( step_context: SystemStepExecutionContext, step_output_handle: StepOutputHandle, output: Any, version: Optional[str], ) -> Iterator[DagsterEvent]: check.inst_param(step_context, "step_context", SystemStepExecutionContext) check.inst_param(output, "output", (Output, DynamicOutput)) step_output = step_context.step.step_output_named(output.output_name) step_output_def = step_context.solid_def.output_def_named(step_output.name) dagster_type = step_output_def.dagster_type with user_code_error_boundary( DagsterTypeCheckError, lambda: ('In solid "{handle}" the output "{output_name}" received ' "value {output_value} of Python type {output_type} which " "does not pass the typecheck for Dagster type " "{dagster_type_name}. Step {step_key}.").format( handle=str(step_context.step.solid_handle), output_name=output.output_name, output_value=output.value, output_type=type(output.value), dagster_type_name=dagster_type.display_name, step_key=step_context.step.key, ), ): type_check = _do_type_check(step_context.for_type(dagster_type), dagster_type, output.value) yield DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, type_check_data=TypeCheckData( success=type_check.success, label=step_output_handle.output_name, description=type_check.description if type_check else None, metadata_entries=type_check.metadata_entries if type_check else [], ), version=version, ), ) if not type_check.success: raise DagsterTypeCheckDidNotPass( description= 'Type check failed for step output "{output_name}" - expected type "{dagster_type}".' .format( output_name=output.output_name, dagster_type=dagster_type.display_name, ), metadata_entries=type_check.metadata_entries, dagster_type=dagster_type, )
def _create_step_output_event(step_context, step_output_handle, type_check, success, version): return DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, type_check_data=TypeCheckData( success=success, label=step_output_handle.output_name, description=type_check.description if type_check else None, metadata_entries=type_check.metadata_entries if type_check else [], ), version=version, ), )
def _create_step_output_event(step_context, output, type_check, success): return DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=StepOutputHandle.from_step( step=step_context.step, output_name=output.output_name), type_check_data=TypeCheckData( success=success, label=output.output_name, description=type_check.description if type_check else None, metadata_entries=type_check.metadata_entries if type_check else [], ), ), )
def _type_check_output( step_context: SystemStepExecutionContext, step_output_handle: StepOutputHandle, output: Any, version: Optional[str], ) -> Iterator[DagsterEvent]: check.inst_param(step_context, "step_context", SystemStepExecutionContext) check.inst_param(output, "output", (Output, DynamicOutput)) step_output = step_context.step.step_output_named(output.output_name) step_output_def = step_context.solid_def.output_def_named(step_output.name) dagster_type = step_output_def.dagster_type with user_code_error_boundary( DagsterTypeCheckError, lambda: ( f'Error occurred while type-checking output "{output.output_name}" of solid ' f'"{str(step_context.step.solid_handle)}", with Python type {type(output.value)} and ' f"Dagster type {dagster_type.display_name}" ), ): type_check = _do_type_check(step_context.for_type(dagster_type), dagster_type, output.value) yield DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, type_check_data=TypeCheckData( success=type_check.success, label=step_output_handle.output_name, description=type_check.description if type_check else None, metadata_entries=type_check.metadata_entries if type_check else [], ), version=version, ), ) if not type_check.success: raise DagsterTypeCheckDidNotPass( description='Type check failed for step output "{output_name}" - expected type "{dagster_type}".'.format( output_name=output.output_name, dagster_type=dagster_type.display_name, ), metadata_entries=type_check.metadata_entries, dagster_type=dagster_type, )
def _create_step_output_event( step_context: SystemStepExecutionContext, step_output_handle: StepOutputHandle, type_check: TypeCheck, success: bool, version: Optional[str], ) -> DagsterEvent: return DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, type_check_data=TypeCheckData( success=success, label=step_output_handle.output_name, description=type_check.description if type_check else None, metadata_entries=type_check.metadata_entries if type_check else [], ), version=version, ), )
def _create_step_output_event(step_context, result, intermediates_manager): check.inst_param(step_context, 'step_context', SystemStepExecutionContext) check.inst_param(result, 'result', Result) check.inst_param(intermediates_manager, 'intermediates_manager', IntermediatesManager) step = step_context.step step_output = step.step_output_named(result.output_name) try: value = step_output.runtime_type.coerce_runtime_value(result.value) step_output_handle = StepOutputHandle.from_step(step=step, output_name=result.output_name) object_key = intermediates_manager.set_intermediate( context=step_context, runtime_type=step_output.runtime_type, step_output_handle=step_output_handle, value=value, ) return DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, value_repr=repr(value), intermediate_materialization=Materialization(path=object_key) if object_key else None, ), ) except DagsterRuntimeCoercionError as e: raise DagsterInvariantViolationError( ( 'In solid "{handle}" the output "{output_name}" returned ' 'an invalid type: {error_msg}.' ).format( handle=str(step.solid_handle), error_msg=','.join(e.args), output_name=result.output_name, ) )
def _create_step_output_event(step_context, step_output_value, intermediates_manager): check.inst_param(step_context, 'step_context', SystemStepExecutionContext) check.inst_param(step_output_value, 'step_output_value', StepOutputValue) check.inst_param(intermediates_manager, 'intermediates_manager', IntermediatesManager) step = step_context.step step_output = step.step_output_named(step_output_value.output_name) try: value = step_output.runtime_type.coerce_runtime_value(step_output_value.value) step_output_handle = StepOutputHandle.from_step( step=step, output_name=step_output_value.output_name ) object_key = intermediates_manager.set_intermediate( context=step_context, runtime_type=step_output.runtime_type, step_output_handle=step_output_handle, value=value, ) return DagsterEvent.step_output_event( step_context=step_context, step_output_data=StepOutputData( step_output_handle=step_output_handle, value_repr=repr(value), storage_object_id=object_key, storage_mode_value=intermediates_manager.storage_mode.value, ), ) except DagsterRuntimeCoercionError as e: raise DagsterInvariantViolationError( ( 'In solid "{step.solid.name}" the output "{output_name}" returned ' 'an invalid type: {error_msg}.' ).format( step=step, error_msg=','.join(e.args), output_name=step_output_value.output_name ) )