def _type_check_output(
step_context: StepExecutionContext,
step_output_handle: StepOutputHandle,
output: Any,
version: Optional[str],
) -> Iterator[DagsterEvent]:
check.inst_param(step_context, "step_context", StepExecutionContext)
check.inst_param(output, "output", (Output, DynamicOutput))
step_output = step_context.step.step_output_named(output.output_name)
step_output_def = step_context.solid_def.output_def_named(step_output.name)
dagster_type = step_output_def.dagster_type
type_check_context = step_context.for_type(dagster_type)
op_label = step_context.describe_op()
output_type = type(output.value)
with user_code_error_boundary(
DagsterTypeCheckError,
lambda:
(f'Error occurred while type-checking output "{output.output_name}" of {op_label}, with '
f"Python type {output_type} and Dagster type {dagster_type.display_name}"
),
log_manager=type_check_context.log,
):
type_check = do_type_check(type_check_context, dagster_type,
output.value)
yield DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=step_output_handle,
type_check_data=TypeCheckData(
success=type_check.success,
label=step_output_handle.output_name,
description=type_check.description if type_check else None,
metadata_entries=type_check.metadata_entries
if type_check else [],
),
version=version,
metadata_entries=[
entry for entry in output.metadata_entries
if isinstance(entry, MetadataEntry)
],
),
)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=(
f'Type check failed for step output "{output.output_name}" - '
f'expected type "{dagster_type.display_name}". '
f"Description: {type_check.description}"),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
def _type_check_output(
step_context: SystemStepExecutionContext,
step_output_handle: StepOutputHandle,
output: Any,
version: Optional[str],
) -> Iterator[DagsterEvent]:
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.inst_param(output, "output", (Output, DynamicOutput))
step_output = step_context.step.step_output_named(output.output_name)
step_output_def = step_context.solid_def.output_def_named(step_output.name)
dagster_type = step_output_def.dagster_type
with user_code_error_boundary(
DagsterTypeCheckError,
lambda: ('In solid "{handle}" the output "{output_name}" received '
"value {output_value} of Python type {output_type} which "
"does not pass the typecheck for Dagster type "
"{dagster_type_name}. Step {step_key}.").format(
handle=str(step_context.step.solid_handle),
output_name=output.output_name,
output_value=output.value,
output_type=type(output.value),
dagster_type_name=dagster_type.display_name,
step_key=step_context.step.key,
),
):
type_check = _do_type_check(step_context.for_type(dagster_type),
dagster_type, output.value)
yield DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=step_output_handle,
type_check_data=TypeCheckData(
success=type_check.success,
label=step_output_handle.output_name,
description=type_check.description if type_check else None,
metadata_entries=type_check.metadata_entries
if type_check else [],
),
version=version,
),
)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=
'Type check failed for step output "{output_name}" - expected type "{dagster_type}".'
.format(
output_name=output.output_name,
dagster_type=dagster_type.display_name,
),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
def _create_step_output_event(step_context, step_output_handle, type_check, success, version):
return DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=step_output_handle,
type_check_data=TypeCheckData(
success=success,
label=step_output_handle.output_name,
description=type_check.description if type_check else None,
metadata_entries=type_check.metadata_entries if type_check else [],
),
version=version,
),
)
def _create_step_output_event(step_context, output, type_check, success):
return DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=StepOutputHandle.from_step(
step=step_context.step, output_name=output.output_name),
type_check_data=TypeCheckData(
success=success,
label=output.output_name,
description=type_check.description if type_check else None,
metadata_entries=type_check.metadata_entries
if type_check else [],
),
),
)
def _type_check_output(
step_context: SystemStepExecutionContext,
step_output_handle: StepOutputHandle,
output: Any,
version: Optional[str],
) -> Iterator[DagsterEvent]:
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.inst_param(output, "output", (Output, DynamicOutput))
step_output = step_context.step.step_output_named(output.output_name)
step_output_def = step_context.solid_def.output_def_named(step_output.name)
dagster_type = step_output_def.dagster_type
with user_code_error_boundary(
DagsterTypeCheckError,
lambda: (
f'Error occurred while type-checking output "{output.output_name}" of solid '
f'"{str(step_context.step.solid_handle)}", with Python type {type(output.value)} and '
f"Dagster type {dagster_type.display_name}"
),
):
type_check = _do_type_check(step_context.for_type(dagster_type), dagster_type, output.value)
yield DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=step_output_handle,
type_check_data=TypeCheckData(
success=type_check.success,
label=step_output_handle.output_name,
description=type_check.description if type_check else None,
metadata_entries=type_check.metadata_entries if type_check else [],
),
version=version,
),
)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description='Type check failed for step output "{output_name}" - expected type "{dagster_type}".'.format(
output_name=output.output_name,
dagster_type=dagster_type.display_name,
),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
def _create_step_output_event(
step_context: SystemStepExecutionContext,
step_output_handle: StepOutputHandle,
type_check: TypeCheck,
success: bool,
version: Optional[str],
) -> DagsterEvent:
return DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=step_output_handle,
type_check_data=TypeCheckData(
success=success,
label=step_output_handle.output_name,
description=type_check.description if type_check else None,
metadata_entries=type_check.metadata_entries
if type_check else [],
),
version=version,
),
)
def _create_step_output_event(step_context, result, intermediates_manager):
check.inst_param(step_context, 'step_context', SystemStepExecutionContext)
check.inst_param(result, 'result', Result)
check.inst_param(intermediates_manager, 'intermediates_manager', IntermediatesManager)
step = step_context.step
step_output = step.step_output_named(result.output_name)
try:
value = step_output.runtime_type.coerce_runtime_value(result.value)
step_output_handle = StepOutputHandle.from_step(step=step, output_name=result.output_name)
object_key = intermediates_manager.set_intermediate(
context=step_context,
runtime_type=step_output.runtime_type,
step_output_handle=step_output_handle,
value=value,
)
return DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=step_output_handle,
value_repr=repr(value),
intermediate_materialization=Materialization(path=object_key)
if object_key
else None,
),
)
except DagsterRuntimeCoercionError as e:
raise DagsterInvariantViolationError(
(
'In solid "{handle}" the output "{output_name}" returned '
'an invalid type: {error_msg}.'
).format(
handle=str(step.solid_handle),
error_msg=','.join(e.args),
output_name=result.output_name,
)
)
def _create_step_output_event(step_context, step_output_value, intermediates_manager):
check.inst_param(step_context, 'step_context', SystemStepExecutionContext)
check.inst_param(step_output_value, 'step_output_value', StepOutputValue)
check.inst_param(intermediates_manager, 'intermediates_manager', IntermediatesManager)
step = step_context.step
step_output = step.step_output_named(step_output_value.output_name)
try:
value = step_output.runtime_type.coerce_runtime_value(step_output_value.value)
step_output_handle = StepOutputHandle.from_step(
step=step, output_name=step_output_value.output_name
)
object_key = intermediates_manager.set_intermediate(
context=step_context,
runtime_type=step_output.runtime_type,
step_output_handle=step_output_handle,
value=value,
)
return DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=step_output_handle,
value_repr=repr(value),
storage_object_id=object_key,
storage_mode_value=intermediates_manager.storage_mode.value,
),
)
except DagsterRuntimeCoercionError as e:
raise DagsterInvariantViolationError(
(
'In solid "{step.solid.name}" the output "{output_name}" returned '
'an invalid type: {error_msg}.'
).format(
step=step, error_msg=','.join(e.args), output_name=step_output_value.output_name
)
)
