def _type_checked_event_sequence_for_input(
step_context: StepExecutionContext, input_name: str,
input_value: Any) -> Iterator[DagsterEvent]:
check.inst_param(step_context, "step_context", StepExecutionContext)
check.str_param(input_name, "input_name")
step_input = step_context.step.step_input_named(input_name)
input_def = step_input.source.get_input_def(step_context.pipeline_def)
dagster_type = input_def.dagster_type
with user_code_error_boundary(
DagsterTypeCheckError,
lambda:
(f'Error occurred while type-checking input "{input_name}" of solid '
f'"{str(step_context.step.solid_handle)}", with Python type {type(input_value)} and '
f"Dagster type {dagster_type.display_name}"),
):
type_check = do_type_check(step_context.for_type(dagster_type),
dagster_type, input_value)
yield _create_step_input_event(step_context,
input_name,
type_check=type_check,
success=type_check.success)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=(f'Type check failed for step input "{input_name}" - '
f'expected type "{dagster_type.display_name}". '
f"Description: {type_check.description}."),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
def _type_check_output(output_def: "OutputDefinition", output: Any,
context: "BoundSolidExecutionContext") -> Any:
"""Validates and performs core type check on a provided output.
Args:
output_def (OutputDefinition): The output definition to validate against.
output (Any): The output to validate.
context (BoundSolidExecutionContext): Context containing resources to be used for type
check.
"""
from ..execution.plan.execute_step import do_type_check
op_label = context.describe_op()
if isinstance(output, (Output, DynamicOutput)):
dagster_type = output_def.dagster_type
type_check = do_type_check(context.for_type(dagster_type),
dagster_type, output.value)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=
(f'Type check failed for {op_label} output "{output.output_name}" - '
f'expected type "{dagster_type.display_name}". '
f"Description: {type_check.description}"),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
context.observe_output(
output.output_name,
output.mapping_key if isinstance(output, DynamicOutput) else None)
return output
else:
dagster_type = output_def.dagster_type
type_check = do_type_check(context.for_type(dagster_type),
dagster_type, output)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=
(f'Type check failed for {op_label} output "{output_def.name}" - '
f'expected type "{dagster_type.display_name}". '
f"Description: {type_check.description}"),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
return output
def _type_check_output(
step_context: SystemStepExecutionContext,
step_output_handle: StepOutputHandle,
output: Any,
version: Optional[str],
) -> Iterator[DagsterEvent]:
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.inst_param(output, "output", (Output, DynamicOutput))
step_output = step_context.step.step_output_named(output.output_name)
step_output_def = step_context.solid_def.output_def_named(step_output.name)
dagster_type = step_output_def.dagster_type
with user_code_error_boundary(
DagsterTypeCheckError,
lambda: ('In solid "{handle}" the output "{output_name}" received '
"value {output_value} of Python type {output_type} which "
"does not pass the typecheck for Dagster type "
"{dagster_type_name}. Step {step_key}.").format(
handle=str(step_context.step.solid_handle),
output_name=output.output_name,
output_value=output.value,
output_type=type(output.value),
dagster_type_name=dagster_type.display_name,
step_key=step_context.step.key,
),
):
type_check = _do_type_check(step_context.for_type(dagster_type),
dagster_type, output.value)
yield DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=step_output_handle,
type_check_data=TypeCheckData(
success=type_check.success,
label=step_output_handle.output_name,
description=type_check.description if type_check else None,
metadata_entries=type_check.metadata_entries
if type_check else [],
),
version=version,
),
)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=
'Type check failed for step output "{output_name}" - expected type "{dagster_type}".'
.format(
output_name=output.output_name,
dagster_type=dagster_type.display_name,
),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
def _type_check_output(
step_context: StepExecutionContext,
step_output_handle: StepOutputHandle,
output: Any,
version: Optional[str],
) -> Iterator[DagsterEvent]:
check.inst_param(step_context, "step_context", StepExecutionContext)
check.inst_param(output, "output", (Output, DynamicOutput))
step_output = step_context.step.step_output_named(output.output_name)
step_output_def = step_context.solid_def.output_def_named(step_output.name)
dagster_type = step_output_def.dagster_type
type_check_context = step_context.for_type(dagster_type)
op_label = step_context.describe_op()
output_type = type(output.value)
with user_code_error_boundary(
DagsterTypeCheckError,
lambda:
(f'Error occurred while type-checking output "{output.output_name}" of {op_label}, with '
f"Python type {output_type} and Dagster type {dagster_type.display_name}"
),
log_manager=type_check_context.log,
):
type_check = do_type_check(type_check_context, dagster_type,
output.value)
yield DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=step_output_handle,
type_check_data=TypeCheckData(
success=type_check.success,
label=step_output_handle.output_name,
description=type_check.description if type_check else None,
metadata_entries=type_check.metadata_entries
if type_check else [],
),
version=version,
metadata_entries=[
entry for entry in output.metadata_entries
if isinstance(entry, MetadataEntry)
],
),
)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=(
f'Type check failed for step output "{output.output_name}" - '
f'expected type "{dagster_type.display_name}". '
f"Description: {type_check.description}"),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
def _type_checked_step_output_event_sequence(step_context, output):
from dagster.core.execution.api import create_execution_plan
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.inst_param(output, "output", Output)
step_output = step_context.step.step_output_named(output.output_name)
speculative_execution_plan = create_execution_plan(
step_context.pipeline_def,
run_config=step_context.run_config,
mode=step_context.mode_def.name,
)
version = resolve_step_output_versions(
speculative_execution_plan,
run_config=step_context.run_config,
mode=step_context.mode_def.name,
)[StepOutputHandle(step_context.step.key, output.output_name)]
with user_code_error_boundary(
DagsterTypeCheckError,
lambda: ('In solid "{handle}" the output "{output_name}" received '
"value {output_value} of Python type {output_type} which "
"does not pass the typecheck for Dagster type "
"{dagster_type_name}. Step {step_key}.").format(
handle=str(step_context.step.solid_handle),
output_name=output.output_name,
output_value=output.value,
output_type=type(output.value),
dagster_type_name=step_output.dagster_type.name,
step_key=step_context.step.key,
),
):
type_check = _do_type_check(
step_context.for_type(step_output.dagster_type),
step_output.dagster_type, output.value)
yield _create_step_output_event(
step_context,
output,
type_check=type_check,
success=type_check.success,
version=version,
)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=
"Type check failed for step output {output_name} of type {dagster_type}."
.format(
output_name=output.output_name,
dagster_type=step_output.dagster_type.name,
),
metadata_entries=type_check.metadata_entries,
dagster_type=step_output.dagster_type,
)
def _execute_and_retrieve_outputs(solid_def: "SolidDefinition",
context: "DirectSolidExecutionContext",
input_dict: Dict[str, Any]) -> tuple:
from dagster.core.execution.plan.execute_step import do_type_check
output_values = {}
output_defs = {
output_def.name: output_def
for output_def in solid_def.output_defs
}
for output in _core_generator(solid_def, context, input_dict):
if not isinstance(output, AssetMaterialization):
if output.output_name in output_values:
raise DagsterInvariantViolationError(
f'Solid "{solid_def.name}" returned an output "{output.output_name}" multiple '
"times")
elif output.output_name not in output_defs:
raise DagsterInvariantViolationError(
f'Solid "{solid_def.name}" returned an output "{output.output_name}" that does '
f"not exist. The available outputs are {list(output_defs)}"
)
else:
dagster_type = output_defs[output.output_name].dagster_type
type_check = do_type_check(context.for_type(dagster_type),
dagster_type, output.value)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=
(f'Type check failed for solid output "{output.output_name}" - '
f'expected type "{dagster_type.display_name}". '
f"Description: {type_check.description}."),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
output_values[output.output_name] = output.value
else:
context.record_materialization(output)
# Check to make sure all non-optional outputs were yielded.
for output_def in solid_def.output_defs:
if output_def.name not in output_values and output_def.is_required:
raise DagsterInvariantViolationError(
f'Solid "{solid_def.name}" did not return an output for non-optional '
f'output "{output_def.name}"')
# Explicitly preserve the ordering of output defs
return tuple([
output_values[output_def.name] for output_def in solid_def.output_defs
])
def _resolve_inputs(solid_def: "SolidDefinition", args, kwargs,
context: "BoundSolidExecutionContext"):
from dagster.core.execution.plan.execute_step import do_type_check
input_defs = solid_def.input_defs
# Fail early if too many inputs were provided.
if len(input_defs) < len(args) + len(kwargs):
raise DagsterInvalidInvocationError(
f"Too many input arguments were provided for solid '{context.alias}'. This may be because "
"an argument was provided for the context parameter, but no context parameter was defined "
"for the solid.")
input_dict = {
input_def.name: input_val
for input_val, input_def in zip(args, input_defs[:len(args)])
}
for input_def in input_defs[len(args):]:
if not input_def.has_default_value and input_def.name not in kwargs:
raise DagsterInvalidInvocationError(
f'No value provided for required input "{input_def.name}".')
input_dict[input_def.name] = (kwargs[input_def.name] if input_def.name
in kwargs else input_def.default_value)
# Type check inputs
input_defs_by_name = {
input_def.name: input_def
for input_def in input_defs
}
for input_name, val in input_dict.items():
input_def = input_defs_by_name[input_name]
dagster_type = input_def.dagster_type
type_check = do_type_check(context.for_type(dagster_type),
dagster_type, val)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=(
f'Type check failed for solid input "{input_def.name}" - '
f'expected type "{dagster_type.display_name}". '
f"Description: {type_check.description}."),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
return input_dict
def _type_check_output(
step_context: SystemStepExecutionContext,
step_output_handle: StepOutputHandle,
output: Any,
version: Optional[str],
) -> Iterator[DagsterEvent]:
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.inst_param(output, "output", (Output, DynamicOutput))
step_output = step_context.step.step_output_named(output.output_name)
step_output_def = step_context.solid_def.output_def_named(step_output.name)
dagster_type = step_output_def.dagster_type
with user_code_error_boundary(
DagsterTypeCheckError,
lambda: (
f'Error occurred while type-checking output "{output.output_name}" of solid '
f'"{str(step_context.step.solid_handle)}", with Python type {type(output.value)} and '
f"Dagster type {dagster_type.display_name}"
),
):
type_check = _do_type_check(step_context.for_type(dagster_type), dagster_type, output.value)
yield DagsterEvent.step_output_event(
step_context=step_context,
step_output_data=StepOutputData(
step_output_handle=step_output_handle,
type_check_data=TypeCheckData(
success=type_check.success,
label=step_output_handle.output_name,
description=type_check.description if type_check else None,
metadata_entries=type_check.metadata_entries if type_check else [],
),
version=version,
),
)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description='Type check failed for step output "{output_name}" - expected type "{dagster_type}".'.format(
output_name=output.output_name,
dagster_type=dagster_type.display_name,
),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
def _type_checked_event_sequence_for_input(
step_context: StepExecutionContext,
input_name: str,
input_value: Any,
) -> Iterator[DagsterEvent]:
check.inst_param(step_context, "step_context", StepExecutionContext)
check.str_param(input_name, "input_name")
step_input = step_context.step.step_input_named(input_name)
input_def = step_context.solid_def.input_def_named(step_input.name)
check.invariant(
input_def.name == input_name,
f"InputDefinition name does not match, expected {input_name} got {input_def.name}",
)
dagster_type = input_def.dagster_type
type_check_context = step_context.for_type(dagster_type)
input_type = type(input_value)
op_label = step_context.describe_op()
with user_code_error_boundary(
DagsterTypeCheckError,
lambda:
(f'Error occurred while type-checking input "{input_name}" of {op_label}, with Python '
f"type {input_type} and Dagster type {dagster_type.display_name}"),
log_manager=type_check_context.log,
):
type_check = do_type_check(type_check_context, dagster_type,
input_value)
yield _create_step_input_event(step_context,
input_name,
type_check=type_check,
success=type_check.success)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=(f'Type check failed for step input "{input_name}" - '
f'expected type "{dagster_type.display_name}". '
f"Description: {type_check.description}"),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
def _type_checked_event_sequence_for_input(
step_context: SystemStepExecutionContext, input_name: str,
input_value: Any) -> Iterator[DagsterEvent]:
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.str_param(input_name, "input_name")
step_input = step_context.step.step_input_named(input_name)
with user_code_error_boundary(
DagsterTypeCheckError,
lambda:
('In solid "{handle}" the input "{input_name}" received '
"value {input_value} of Python type {input_type} which "
"does not pass the typecheck for Dagster type "
"{dagster_type_name}. Step {step_key}.").format(
handle=str(step_context.step.solid_handle),
input_name=input_name,
input_value=input_value,
input_type=type(input_value),
dagster_type_name=step_input.dagster_type.display_name,
step_key=step_context.step.key,
),
):
type_check = _do_type_check(
step_context.for_type(step_input.dagster_type),
step_input.dagster_type,
input_value,
)
yield _create_step_input_event(step_context,
input_name,
type_check=type_check,
success=type_check.success)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=
'Type check failed for step input "{input_name}" - expected type "{dagster_type}".'
.format(
input_name=input_name,
dagster_type=step_input.dagster_type.display_name,
),
metadata_entries=type_check.metadata_entries,
dagster_type=step_input.dagster_type,
)
def _type_checked_step_output_event_sequence(step_context, output):
check.inst_param(step_context, "step_context", SystemStepExecutionContext)
check.inst_param(output, "output", Output)
step_output = step_context.step.step_output_named(output.output_name)
version = compute_version_for_step_output(step_context, output)
with user_code_error_boundary(
DagsterTypeCheckError,
lambda: ('In solid "{handle}" the output "{output_name}" received '
"value {output_value} of Python type {output_type} which "
"does not pass the typecheck for Dagster type "
"{dagster_type_name}. Step {step_key}.").format(
handle=str(step_context.step.solid_handle),
output_name=output.output_name,
output_value=output.value,
output_type=type(output.value),
dagster_type_name=step_output.dagster_type.name,
step_key=step_context.step.key,
),
):
type_check = _do_type_check(
step_context.for_type(step_output.dagster_type),
step_output.dagster_type, output.value)
yield _create_step_output_event(
step_context,
output,
type_check=type_check,
success=type_check.success,
version=version,
)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=
"Type check failed for step output {output_name} of type {dagster_type}."
.format(
output_name=output.output_name,
dagster_type=step_output.dagster_type.name,
),
metadata_entries=type_check.metadata_entries,
dagster_type=step_output.dagster_type,
)
def _type_checked_event_sequence_for_input(step_context, input_name,
input_value):
check.inst_param(step_context, 'step_context', SystemStepExecutionContext)
check.str_param(input_name, 'input_name')
step_input = step_context.step.step_input_named(input_name)
with user_code_error_boundary(
DagsterTypeCheckError,
lambda: ('In solid "{handle}" the input "{input_name}" received '
'value {input_value} of Python type {input_type} which '
'does not pass the typecheck for Dagster type '
'{dagster_type_name}. Step {step_key}.').format(
handle=str(step_context.step.solid_handle),
input_name=input_name,
input_value=input_value,
input_type=type(input_value),
dagster_type_name=step_input.dagster_type.name,
step_key=step_context.step.key,
),
):
type_check = _do_type_check(
step_context.for_type_check(step_input.dagster_type),
step_input.dagster_type,
input_value,
)
yield _create_step_input_event(step_context,
input_name,
type_check=type_check,
success=type_check.success)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=
'Type check failed for step input {input_name} of type {dagster_type}.'
.format(
input_name=input_name,
dagster_type=step_input.dagster_type.name,
),
metadata_entries=type_check.metadata_entries,
dagster_type=step_input.dagster_type,
)
def _resolve_inputs(solid_def: "SolidDefinition", args, kwargs,
context: "BoundSolidExecutionContext"):
from dagster.core.execution.plan.execute_step import do_type_check
nothing_input_defs = [
input_def for input_def in solid_def.input_defs
if input_def.dagster_type.is_nothing
]
# Check kwargs for nothing inputs, and error if someone provided one.
for input_def in nothing_input_defs:
if input_def.name in kwargs:
node_label = solid_def.node_type_str # string "solid" for solids, "op" for ops
raise DagsterInvalidInvocationError(
f"Attempted to provide value for nothing input '{input_def.name}'. Nothing "
f"dependencies are ignored when directly invoking {node_label}s."
)
# Discard nothing dependencies - we ignore them during invocation.
input_defs_by_name = {
input_def.name: input_def
for input_def in solid_def.input_defs
if not input_def.dagster_type.is_nothing
}
# Fail early if too many inputs were provided.
if len(input_defs_by_name) < len(args) + len(kwargs):
if len(nothing_input_defs) > 0:
suggestion = (
"This may be because you attempted to provide a value for a nothing "
"dependency. Nothing dependencies are ignored when directly invoking solids."
)
else:
suggestion = (
"This may be because an argument was provided for the context parameter, "
"but no context parameter was defined for the solid.")
node_label = solid_def.node_type_str
raise DagsterInvalidInvocationError(
f"Too many input arguments were provided for {node_label} '{context.alias}'. {suggestion}"
)
positional_inputs = cast("DecoratedSolidFunction",
solid_def.compute_fn).positional_inputs()
input_dict = {}
for position, value in enumerate(args):
input_dict[positional_inputs[position]] = value
for positional_input in positional_inputs[len(args):]:
input_def = input_defs_by_name[positional_input]
if not input_def.has_default_value and positional_input not in kwargs:
raise DagsterInvalidInvocationError(
f'No value provided for required input "{positional_input}".')
input_dict[positional_input] = (kwargs[positional_input]
if positional_input in kwargs else
input_def.default_value)
# Type check inputs
op_label = context.describe_op()
for input_name, val in input_dict.items():
input_def = input_defs_by_name[input_name]
dagster_type = input_def.dagster_type
type_check = do_type_check(context.for_type(dagster_type),
dagster_type, val)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=
(f'Type check failed for {op_label} input "{input_def.name}" - '
f'expected type "{dagster_type.display_name}". '
f"Description: {type_check.description}"),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
return input_dict
