def _resolve_inputs(solid_def, args, kwargs):
input_defs = solid_def.input_defs
# Fail early if too many inputs were provided.
if len(input_defs) < len(args) + len(kwargs):
raise DagsterInvalidInvocationError(
f"Too many input arguments were provided for solid '{solid_def.name}'. This may be because "
"an argument was provided for the context parameter, but no context parameter was defined "
"for the solid."
)
input_dict = {
input_def.name: input_val for input_val, input_def in zip(args, input_defs[: len(args)])
}
for input_def in input_defs[len(args) :]:
if not input_def.has_default_value and input_def.name not in kwargs:
raise DagsterInvalidInvocationError(
f'No value provided for required input "{input_def.name}".'
)
input_dict[input_def.name] = (
kwargs[input_def.name] if input_def.name in kwargs else input_def.default_value
)
return input_dict
def __call__(self, *args, **kwargs) -> Any:
from .composition import is_in_composition
from .decorators.solid import DecoratedSolidFunction
from ..execution.context.invocation import UnboundSolidExecutionContext
if is_in_composition():
return super(SolidDefinition, self).__call__(*args, **kwargs)
else:
if not isinstance(self.compute_fn, DecoratedSolidFunction):
raise DagsterInvalidInvocationError(
"Attemped to invoke solid that was not constructed using the `@solid` "
"decorator. Only solids constructed using the `@solid` decorator can be "
"directly invoked.")
if self.compute_fn.has_context_arg():
if len(args) == 0:
raise DagsterInvalidInvocationError(
f"Compute function of solid '{self.name}' has context argument, but no context "
"was provided when invoking.")
elif args[0] is not None and not isinstance(
args[0], UnboundSolidExecutionContext):
raise DagsterInvalidInvocationError(
f"Compute function of solid '{self.name}' has context argument, but no context "
"was provided when invoking.")
context = args[0]
return solid_invocation_result(self, context, *args[1:],
**kwargs)
else:
if len(args) > 0 and isinstance(args[0],
UnboundSolidExecutionContext):
raise DagsterInvalidInvocationError(
f"Compute function of solid '{self.name}' has no context argument, but "
"context was provided when invoking.")
return solid_invocation_result(self, None, *args, **kwargs)
def __call__(self, *args, **kwargs) -> Any:
from .composition import is_in_composition
from dagster.core.execution.context.invocation import DirectSolidExecutionContext
if is_in_composition():
return super(SolidDefinition, self).__call__(*args, **kwargs)
else:
if self._context_arg_provided:
if len(args) == 0:
raise DagsterInvalidInvocationError(
f"Compute function of solid '{self.name}' has context argument, but no context "
"was provided when invoking.")
elif args[0] is not None and not isinstance(
args[0], DirectSolidExecutionContext):
raise DagsterInvalidInvocationError(
f"Compute function of solid '{self.name}' has context argument, but no context "
"was provided when invoking.")
context = args[0]
return solid_invocation_result(self, context, *args[1:],
**kwargs)
else:
if len(args) > 0 and isinstance(args[0],
DirectSolidExecutionContext):
raise DagsterInvalidInvocationError(
f"Compute function of solid '{self.name}' has no context argument, but "
"context was provided when invoking.")
return solid_invocation_result(self, None, *args, **kwargs)
def __call__(self, *args, **kwargs):
from dagster.core.execution.resources_init import InitResourceContext
context_provided = is_context_provided(
get_function_params(self.resource_fn))
if context_provided:
if len(args) + len(kwargs) == 0:
raise DagsterInvalidInvocationError(
"Resource initialization function has context argument, but no context was provided "
"when invoking.")
if len(args) + len(kwargs) > 1:
raise DagsterInvalidInvocationError(
"Initialization of resource received multiple arguments. Only a first "
"positional context parameter should be provided when invoking."
)
context_param_name = get_function_params(self.resource_fn)[0].name
if args:
check.opt_inst_param(args[0], context_param_name,
InitResourceContext)
return resource_invocation_result(self, args[0])
else:
if context_param_name not in kwargs:
raise DagsterInvalidInvocationError(
f"Resource initialization expected argument '{context_param_name}'."
)
check.opt_inst_param(kwargs[context_param_name],
context_param_name, InitResourceContext)
return resource_invocation_result(self,
kwargs[context_param_name])
else:
return resource_invocation_result(self, None)
def _check_invocation_requirements(
solid_def: "SolidDefinition", context: Optional["DirectSolidExecutionContext"]
) -> "DirectSolidExecutionContext":
"""Ensure that provided context fulfills requirements of solid definition.
If no context was provided, then construct an enpty DirectSolidExecutionContext
"""
from dagster.core.execution.context.invocation import DirectSolidExecutionContext
from dagster.config.validate import validate_config
# Check resource requirements
if solid_def.required_resource_keys and context is None:
raise DagsterInvalidInvocationError(
f'Solid "{solid_def.name}" has required resources, but no context was provided. Use the'
"`build_solid_context` function to construct a context with the required "
"resources."
)
if context is not None and solid_def.required_resource_keys:
resources_dict = cast( # type: ignore[attr-defined]
"DirectSolidExecutionContext",
context,
).resources._asdict()
for resource_key in solid_def.required_resource_keys:
if resource_key not in resources_dict:
raise DagsterInvalidInvocationError(
f'Solid "{solid_def.name}" requires resource "{resource_key}", but no resource '
"with that key was found on the context."
)
# Check config requirements
if not context and solid_def.config_schema.as_field().is_required:
raise DagsterInvalidInvocationError(
f'Solid "{solid_def.name}" has required config schema, but no context was provided. '
"Use the `build_solid_context` function to create a context with config."
)
config = None
if solid_def.config_field:
solid_config = check.opt_dict_param(
context.solid_config if context else None, "solid_config"
)
config_evr = validate_config(solid_def.config_field.config_type, solid_config)
if not config_evr.success:
raise DagsterInvalidConfigError(
"Error in config for solid ", config_evr.errors, solid_config
)
mapped_config_evr = solid_def.apply_config_mapping({"config": solid_config})
if not mapped_config_evr.success:
raise DagsterInvalidConfigError(
"Error in config for solid ", mapped_config_evr.errors, solid_config
)
config = mapped_config_evr.value.get("config", {})
return (
context
if context
else DirectSolidExecutionContext(solid_config=config, resources_dict=None, instance=None)
)
def __call__(self, *args, **kwargs) -> Any:
from .composition import is_in_composition
from .decorators.solid import DecoratedSolidFunction
from ..execution.context.invocation import UnboundSolidExecutionContext
if is_in_composition():
return super(SolidDefinition, self).__call__(*args, **kwargs)
else:
node_label = self.node_type_str # string "solid" for solids, "op" for ops
if not isinstance(self.compute_fn, DecoratedSolidFunction):
raise DagsterInvalidInvocationError(
f"Attemped to invoke {node_label} that was not constructed using the `@{node_label}` "
f"decorator. Only {node_label}s constructed using the `@{node_label}` decorator can be "
"directly invoked.")
if self.compute_fn.has_context_arg():
if len(args) + len(kwargs) == 0:
raise DagsterInvalidInvocationError(
f"Compute function of {node_label} '{self.name}' has context argument, but no context "
"was provided when invoking.")
if len(args) > 0:
if args[0] is not None and not isinstance(
args[0], UnboundSolidExecutionContext):
raise DagsterInvalidInvocationError(
f"Compute function of {node_label} '{self.name}' has context argument, "
"but no context was provided when invoking.")
context = args[0]
return solid_invocation_result(self, context, *args[1:],
**kwargs)
# Context argument is provided under kwargs
else:
context_param_name = get_function_params(
self.compute_fn.decorated_fn)[0].name
if context_param_name not in kwargs:
raise DagsterInvalidInvocationError(
f"Compute function of {node_label} '{self.name}' has context argument "
f"'{context_param_name}', but no value for '{context_param_name}' was "
f"found when invoking. Provided kwargs: {kwargs}")
context = kwargs[context_param_name]
kwargs_sans_context = {
kwarg: val
for kwarg, val in kwargs.items()
if not kwarg == context_param_name
}
return solid_invocation_result(self, context, *args,
**kwargs_sans_context)
else:
if len(args) > 0 and isinstance(args[0],
UnboundSolidExecutionContext):
raise DagsterInvalidInvocationError(
f"Compute function of {node_label} '{self.name}' has no context argument, but "
"context was provided when invoking.")
return solid_invocation_result(self, None, *args, **kwargs)
def _resolve_inputs(solid_def: "SolidDefinition", args, kwargs,
context: "BoundSolidExecutionContext"):
from dagster.core.execution.plan.execute_step import do_type_check
input_defs = solid_def.input_defs
# Fail early if too many inputs were provided.
if len(input_defs) < len(args) + len(kwargs):
raise DagsterInvalidInvocationError(
f"Too many input arguments were provided for solid '{context.alias}'. This may be because "
"an argument was provided for the context parameter, but no context parameter was defined "
"for the solid.")
input_dict = {
input_def.name: input_val
for input_val, input_def in zip(args, input_defs[:len(args)])
}
for input_def in input_defs[len(args):]:
if not input_def.has_default_value and input_def.name not in kwargs:
raise DagsterInvalidInvocationError(
f'No value provided for required input "{input_def.name}".')
input_dict[input_def.name] = (kwargs[input_def.name] if input_def.name
in kwargs else input_def.default_value)
# Type check inputs
input_defs_by_name = {
input_def.name: input_def
for input_def in input_defs
}
for input_name, val in input_dict.items():
input_def = input_defs_by_name[input_name]
dagster_type = input_def.dagster_type
type_check = do_type_check(context.for_type(dagster_type),
dagster_type, val)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=(
f'Type check failed for solid input "{input_def.name}" - '
f'expected type "{dagster_type.display_name}". '
f"Description: {type_check.description}."),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
return input_dict
def __call__(self, *args, **kwargs):
from .decorators.sensor import is_context_provided
context_provided = is_context_provided(get_function_params(self._decorated_fn))
if not self._decorated_fn:
raise DagsterInvalidInvocationError(
"Sensor invocation is only supported for sensors created via the `@sensor` "
"decorator."
)
if context_provided:
if len(args) + len(kwargs) == 0:
raise DagsterInvalidInvocationError(
"Sensor decorated function has context argument, but no context argument was "
"provided when invoking."
)
if len(args) + len(kwargs) > 1:
raise DagsterInvalidInvocationError(
"Sensor invocation received multiple arguments. Only a first "
"positional context parameter should be provided when invoking."
)
context_param_name = get_function_params(self._decorated_fn)[0].name
if args:
context = check.opt_inst_param(args[0], context_param_name, SensorEvaluationContext)
else:
if context_param_name not in kwargs:
raise DagsterInvalidInvocationError(
f"Sensor invocation expected argument '{context_param_name}'."
)
context = check.opt_inst_param(
kwargs[context_param_name], context_param_name, SensorEvaluationContext
)
context = context if context else build_sensor_context()
return self._decorated_fn(context)
else:
if len(args) + len(kwargs) > 0:
raise DagsterInvalidInvocationError(
"Sensor decorated function has no arguments, but arguments were provided to "
"invocation."
)
return self._decorated_fn()
def _check_invocation_requirements(
resource_def: "ResourceDefinition",
init_context: Optional["InitResourceContext"]
) -> "InitResourceContext":
from dagster.core.execution.context.init import InitResourceContext, build_init_resource_context
if resource_def.required_resource_keys and init_context is None:
raise DagsterInvalidInvocationError(
"Resource has required resources, but no context was provided. Use the "
"`build_init_resource_context` function to construct a context with the required "
"resources.")
if init_context is not None and resource_def.required_resource_keys:
resources_dict = cast(
"InitResourceContext",
init_context,
).resources._asdict() # type: ignore[attr-defined]
for resource_key in resource_def.required_resource_keys:
if resource_key not in resources_dict:
raise DagsterInvalidInvocationError(
f'Resource requires resource "{resource_key}", but no resource '
"with that key was found on the context.")
# Check config requirements
if not init_context and resource_def.config_schema.as_field().is_required:
raise DagsterInvalidInvocationError(
"Resource has required config schema, but no context was provided. "
"Use the `build_init_resource_context` function to create a context with config."
)
resource_config = _resolve_bound_config(
init_context.resource_config if init_context else None, resource_def)
# Construct a context if None was provided. This will initialize an ephemeral instance, and
# console log manager.
init_context = init_context or build_init_resource_context()
return InitResourceContext(
resource_config=resource_config,
resources=init_context.resources,
resource_def=resource_def,
instance=init_context.instance,
log_manager=init_context.log,
)
def __call__(self, *args, **kwargs):
context_provided = is_context_provided(get_function_params(self._run_status_sensor_fn))
if context_provided:
if len(args) + len(kwargs) == 0:
raise DagsterInvalidInvocationError(
"Run status sensor function expected context argument, but no context argument "
"was provided when invoking."
)
if len(args) + len(kwargs) > 1:
raise DagsterInvalidInvocationError(
"Run status sensor invocation received multiple arguments. Only a first "
"positional context parameter should be provided when invoking."
)
context_param_name = get_function_params(self._run_status_sensor_fn)[0].name
if args:
context = check.opt_inst_param(args[0], context_param_name, RunStatusSensorContext)
else:
if context_param_name not in kwargs:
raise DagsterInvalidInvocationError(
f"Run status sensor invocation expected argument '{context_param_name}'."
)
context = check.opt_inst_param(
kwargs[context_param_name], context_param_name, RunStatusSensorContext
)
if not context:
raise DagsterInvalidInvocationError(
"Context must be provided for direct invocation of run status sensor."
)
return self._run_status_sensor_fn(context)
else:
if len(args) + len(kwargs) > 0:
raise DagsterInvalidInvocationError(
"Run status sensor decorated function has no arguments, but arguments were "
"provided to invocation."
)
return self._run_status_sensor_fn()
def build_solid_context(
resources: Optional[Dict[str, Any]] = None,
solid_config: Any = None,
resources_config: Optional[Dict[str, Any]] = None,
instance: Optional[DagsterInstance] = None,
config: Any = None,
partition_key: Optional[str] = None,
) -> UnboundSolidExecutionContext:
"""Builds solid execution context from provided parameters.
``build_solid_context`` can be used as either a function or context manager. If there is a
provided resource that is a context manager, then ``build_solid_context`` must be used as a
context manager. This function can be used to provide the context argument when directly
invoking a solid.
Args:
resources (Optional[Dict[str, Any]]): The resources to provide to the context. These can be
either values or resource definitions.
solid_config (Optional[Any]): The solid config to provide to the context. The value provided
here will be available as ``context.solid_config``.
resources_config (Optional[Dict[str, Any]]): Configuration for any resource definitions
provided to the resources arg. The configuration under a specific key should match the
resource under a specific key in the resources dictionary.
instance (Optional[DagsterInstance]): The dagster instance configured for the context.
Defaults to DagsterInstance.ephemeral().
Examples:
.. code-block:: python
context = build_solid_context()
solid_to_invoke(context)
with build_solid_context(resources={"foo": context_manager_resource}) as context:
solid_to_invoke(context)
"""
if solid_config and config:
raise DagsterInvalidInvocationError(
"Attempted to invoke ``build_solid_context`` with both ``solid_config``, and its "
"legacy version, ``config``. Please provide one or the other.")
solid_config = solid_config if solid_config else config
return UnboundSolidExecutionContext(
resources_dict=check.opt_dict_param(resources,
"resources",
key_type=str),
resources_config=check.opt_dict_param(resources_config,
"resources_config",
key_type=str),
solid_config=solid_config,
instance=check.opt_inst_param(instance, "instance", DagsterInstance),
partition_key=check.opt_str_param(partition_key, "partition_key"),
)
def _check_invocation_requirements(
solid_def: "SolidDefinition",
context: Optional["UnboundSolidExecutionContext"]) -> None:
"""Ensure that provided context fulfills requirements of solid definition.
If no context was provided, then construct an enpty UnboundSolidExecutionContext
"""
# Check resource requirements
if solid_def.required_resource_keys and context is None:
raise DagsterInvalidInvocationError(
f'Solid "{solid_def.name}" has required resources, but no context was provided. Use the'
"`build_solid_context` function to construct a context with the required "
"resources.")
# Check config requirements
if not context and solid_def.config_schema.as_field().is_required:
raise DagsterInvalidInvocationError(
f'Solid "{solid_def.name}" has required config schema, but no context was provided. '
"Use the `build_solid_context` function to create a context with config."
)
def _validate_resource_requirements(resources: "Resources", solid_def: SolidDefinition) -> None:
"""Validate correctness of resources against required resource keys"""
resources_dict = resources._asdict() # type: ignore[attr-defined]
required_resource_keys: AbstractSet[str] = solid_def.required_resource_keys or set()
for resource_key in required_resource_keys:
if resource_key not in resources_dict:
raise DagsterInvalidInvocationError(
f'Solid "{solid_def.name}" requires resource "{resource_key}", but no resource '
"with that key was found on the context."
)
def __call__(self, *args, **kwargs):
from dagster.core.execution.context.logger import UnboundInitLoggerContext
from .logger_invocation import logger_invocation_result
if len(args) == 0 and len(kwargs) == 0:
raise DagsterInvalidInvocationError(
"Logger initialization function has context argument, but no context argument was "
"provided when invoking.")
if len(args) + len(kwargs) > 1:
raise DagsterInvalidInvocationError(
"Initialization of logger received multiple arguments. Only a first "
"positional context parameter should be provided when invoking."
)
context_param_name = get_function_params(self.logger_fn)[0].name
if args:
context = check.opt_inst_param(
args[0],
context_param_name,
UnboundInitLoggerContext,
default=UnboundInitLoggerContext(logger_config=None,
pipeline_def=None),
)
return logger_invocation_result(self, context)
else:
if context_param_name not in kwargs:
raise DagsterInvalidInvocationError(
f"Logger initialization expected argument '{context_param_name}'."
)
context = check.opt_inst_param(
kwargs[context_param_name],
context_param_name,
UnboundInitLoggerContext,
default=UnboundInitLoggerContext(logger_config=None,
pipeline_def=None),
)
return logger_invocation_result(self, context)
def execute_in_process(
self,
run_config: Optional[Any] = None,
instance: Optional["DagsterInstance"] = None,
resources: Optional[Dict[str, Any]] = None,
raise_on_error: bool = True,
run_id: Optional[str] = None,
) -> "ExecuteInProcessResult":
if not isinstance(self.node_def, GraphDefinition):
raise DagsterInvalidInvocationError(
"Attemped to call `execute_in_process` on a composite solid. Only graphs "
"constructed using the `@graph` decorator support this method."
)
from dagster.core.execution.build_resources import wrap_resources_for_execution
from dagster.core.execution.execute_in_process import core_execute_in_process
from .executor_definition import execute_in_process_executor
from .job_definition import JobDefinition
from .mode import ModeDefinition
if len(self.node_def.input_defs) > 0:
raise DagsterInvariantViolationError(
"Graphs with inputs cannot be used with execute_in_process at this time."
)
ephemeral_job = JobDefinition(
name=self.given_alias,
graph_def=self.node_def,
mode_def=ModeDefinition(
executor_defs=[execute_in_process_executor],
resource_defs=wrap_resources_for_execution(resources),
),
tags=self.tags,
hook_defs=self.hook_defs,
op_retry_policy=self.retry_policy,
)
return core_execute_in_process(
node=self.node_def,
ephemeral_pipeline=ephemeral_job,
run_config=run_config if run_config is not None else {},
instance=instance,
output_capturing_enabled=True,
raise_on_error=raise_on_error,
run_id=run_id,
)
def build_op_context(
resources: Optional[Dict[str, Any]] = None,
op_config: Any = None,
resources_config: Optional[Dict[str, Any]] = None,
instance: Optional[DagsterInstance] = None,
config: Any = None,
partition_key: Optional[str] = None,
) -> OpExecutionContext:
"""Builds op execution context from provided parameters.
``op`` is currently built on top of `solid`, and thus this function creates a `SolidExecutionContext`.
``build_op_context`` can be used as either a function or context manager. If there is a
provided resource that is a context manager, then ``build_op_context`` must be used as a
context manager. This function can be used to provide the context argument when directly
invoking a op.
Args:
resources (Optional[Dict[str, Any]]): The resources to provide to the context. These can be
either values or resource definitions.
config (Optional[Any]): The op config to provide to the context.
instance (Optional[DagsterInstance]): The dagster instance configured for the context.
Defaults to DagsterInstance.ephemeral().
Examples:
.. code-block:: python
context = build_op_context()
op_to_invoke(context)
with build_op_context(resources={"foo": context_manager_resource}) as context:
op_to_invoke(context)
"""
if op_config and config:
raise DagsterInvalidInvocationError(
"Attempted to invoke ``build_op_context`` with both ``op_config``, and its "
"legacy version, ``config``. Please provide one or the other.")
op_config = op_config if op_config else config
return build_solid_context(
resources=resources,
resources_config=resources_config,
solid_config=op_config,
instance=instance,
partition_key=partition_key,
)
def to_job(
self,
name: Optional[str] = None,
description: Optional[str] = None,
resource_defs: Optional[Dict[str, ResourceDefinition]] = None,
config: Optional[Union[ConfigMapping, Dict[str, Any],
"PartitionedConfig"]] = None,
tags: Optional[Dict[str, Any]] = None,
logger_defs: Optional[Dict[str, LoggerDefinition]] = None,
executor_def: Optional["ExecutorDefinition"] = None,
hooks: Optional[AbstractSet[HookDefinition]] = None,
op_retry_policy: Optional[RetryPolicy] = None,
version_strategy: Optional[VersionStrategy] = None,
partitions_def: Optional["PartitionsDefinition"] = None,
) -> "JobDefinition":
if not isinstance(self.node_def, GraphDefinition):
raise DagsterInvalidInvocationError(
"Attemped to call `execute_in_process` on a composite solid. Only graphs "
"constructed using the `@graph` decorator support this method."
)
tags = check.opt_dict_param(tags, "tags", key_type=str)
hooks = check.opt_set_param(hooks, "hooks", HookDefinition)
op_retry_policy = check.opt_inst_param(op_retry_policy,
"op_retry_policy", RetryPolicy)
job_hooks: Set[HookDefinition] = set()
job_hooks.update(check.opt_set_param(hooks, "hooks", HookDefinition))
job_hooks.update(self.hook_defs)
return self.node_def.to_job(
name=name or self.given_alias,
description=description,
resource_defs=resource_defs,
config=config,
tags=tags if not self.tags else self.tags.updated_with(tags),
logger_defs=logger_defs,
executor_def=executor_def,
hooks=job_hooks,
op_retry_policy=op_retry_policy,
version_strategy=version_strategy,
partitions_def=partitions_def,
)
def __call__(self, *args, **kwargs):
raise DagsterInvalidInvocationError(
"Direct invocation of RunStatusSensors is not yet supported."
)
def _resolve_inputs(solid_def: "SolidDefinition", args, kwargs,
context: "BoundSolidExecutionContext"):
from dagster.core.execution.plan.execute_step import do_type_check
nothing_input_defs = [
input_def for input_def in solid_def.input_defs
if input_def.dagster_type.is_nothing
]
# Check kwargs for nothing inputs, and error if someone provided one.
for input_def in nothing_input_defs:
if input_def.name in kwargs:
node_label = solid_def.node_type_str # string "solid" for solids, "op" for ops
raise DagsterInvalidInvocationError(
f"Attempted to provide value for nothing input '{input_def.name}'. Nothing "
f"dependencies are ignored when directly invoking {node_label}s."
)
# Discard nothing dependencies - we ignore them during invocation.
input_defs_by_name = {
input_def.name: input_def
for input_def in solid_def.input_defs
if not input_def.dagster_type.is_nothing
}
# Fail early if too many inputs were provided.
if len(input_defs_by_name) < len(args) + len(kwargs):
if len(nothing_input_defs) > 0:
suggestion = (
"This may be because you attempted to provide a value for a nothing "
"dependency. Nothing dependencies are ignored when directly invoking solids."
)
else:
suggestion = (
"This may be because an argument was provided for the context parameter, "
"but no context parameter was defined for the solid.")
node_label = solid_def.node_type_str
raise DagsterInvalidInvocationError(
f"Too many input arguments were provided for {node_label} '{context.alias}'. {suggestion}"
)
positional_inputs = cast("DecoratedSolidFunction",
solid_def.compute_fn).positional_inputs()
input_dict = {}
for position, value in enumerate(args):
input_dict[positional_inputs[position]] = value
for positional_input in positional_inputs[len(args):]:
input_def = input_defs_by_name[positional_input]
if not input_def.has_default_value and positional_input not in kwargs:
raise DagsterInvalidInvocationError(
f'No value provided for required input "{positional_input}".')
input_dict[positional_input] = (kwargs[positional_input]
if positional_input in kwargs else
input_def.default_value)
# Type check inputs
op_label = context.describe_op()
for input_name, val in input_dict.items():
input_def = input_defs_by_name[input_name]
dagster_type = input_def.dagster_type
type_check = do_type_check(context.for_type(dagster_type),
dagster_type, val)
if not type_check.success:
raise DagsterTypeCheckDidNotPass(
description=
(f'Type check failed for {op_label} input "{input_def.name}" - '
f'expected type "{dagster_type.display_name}". '
f"Description: {type_check.description}"),
metadata_entries=type_check.metadata_entries,
dagster_type=dagster_type,
)
return input_dict
def __call__(self, *args, **kwargs):
from .solid_invocation import solid_invocation_result
from ..execution.context.invocation import UnboundSolidExecutionContext
node_name = self.given_alias if self.given_alias else self.node_def.name
# If PendingNodeInvocation is not within composition context, and underlying node definition
# is a solid definition, then permit it to be invoked and executed like a solid definition.
if not is_in_composition() and isinstance(self.node_def,
SolidDefinition):
if not isinstance(self.node_def.compute_fn,
DecoratedSolidFunction):
raise DagsterInvalidInvocationError(
"Attemped to invoke solid that was not constructed using the `@solid` "
"decorator. Only solids constructed using the `@solid` decorator can be "
"directly invoked.")
if self.node_def.compute_fn.has_context_arg():
if len(args) == 0:
raise DagsterInvalidInvocationError(
f"Compute function of solid '{self.given_alias}' has context argument, but no context "
"was provided when invoking.")
elif args[0] is not None and not isinstance(
args[0], UnboundSolidExecutionContext):
raise DagsterInvalidInvocationError(
f"Compute function of solid '{self.given_alias}' has context argument, but no context "
"was provided when invoking.")
context = args[0]
return solid_invocation_result(self, context, *args[1:],
**kwargs)
else:
if len(args) > 0 and isinstance(args[0],
UnboundSolidExecutionContext):
raise DagsterInvalidInvocationError(
f"Compute function of solid '{self.given_alias}' has no context argument, but "
"context was provided when invoking.")
return solid_invocation_result(self, None, *args, **kwargs)
assert_in_composition(node_name)
input_bindings = {}
# handle *args
for idx, output_node in enumerate(args):
if idx >= len(self.node_def.input_defs):
raise DagsterInvalidDefinitionError(
"In {source} {name}, received too many inputs for "
"invocation {node_name}. Only {def_num} defined, received {arg_num}"
.format(
source=current_context().source,
name=current_context().name,
node_name=node_name,
def_num=len(self.node_def.input_defs),
arg_num=len(args),
))
input_name = self.node_def.resolve_input_name_at_position(idx)
if input_name is None:
raise DagsterInvalidDefinitionError(
"In {source} {name}, could not resolve input based on position at "
"index {idx} for invocation {node_name}. Use keyword args instead, "
"available inputs are: {inputs}".format(
idx=idx,
source=current_context().source,
name=current_context().name,
node_name=node_name,
inputs=list(
map(lambda inp: inp.name,
self.node_def.input_defs)),
))
self._process_argument_node(
node_name,
output_node,
input_name,
input_bindings,
"(at position {idx})".format(idx=idx),
)
# then **kwargs
for input_name, output_node in kwargs.items():
self._process_argument_node(
node_name,
output_node,
input_name,
input_bindings,
"(passed by keyword)",
)
# the node name is potentially reassigned for aliasing
resolved_node_name = current_context().observe_invocation(
self.given_alias,
self.node_def,
input_bindings,
self.tags,
self.hook_defs,
self.retry_policy,
)
if len(self.node_def.output_defs) == 0:
return None
if len(self.node_def.output_defs) == 1:
output_def = self.node_def.output_defs[0]
output_name = output_def.name
if output_def.is_dynamic:
return InvokedSolidDynamicOutputWrapper(
resolved_node_name, output_name)
else:
return InvokedSolidOutputHandle(resolved_node_name,
output_name)
outputs = [output_def for output_def in self.node_def.output_defs]
invoked_output_handles = {}
for output_def in outputs:
if output_def.is_dynamic:
invoked_output_handles[
output_def.name] = InvokedSolidDynamicOutputWrapper(
resolved_node_name, output_def.name)
else:
invoked_output_handles[
output_def.name] = InvokedSolidOutputHandle(
resolved_node_name, output_def.name)
return namedtuple(
"_{node_def}_outputs".format(node_def=self.node_def.name),
" ".join([output_def.name for output_def in outputs]),
)(**invoked_output_handles)