def ensure_all_args_defined_in_sub(super_sig, sub_sig):
sub_has_var_args = any(p.kind == Parameter.VAR_POSITIONAL
for p in sub_sig.parameters.values())
sub_has_var_kwargs = any(p.kind == Parameter.VAR_KEYWORD
for p in sub_sig.parameters.values())
for super_index, (name,
super_param) in enumerate(super_sig.parameters.items()):
if not is_param_defined_in_sub(name, sub_has_var_args,
sub_has_var_kwargs, sub_sig,
super_param):
raise TypeError(f"`{name}` is not present.")
elif (name in sub_sig.parameters
and super_param.kind != Parameter.VAR_POSITIONAL
and super_param.kind != Parameter.VAR_KEYWORD):
sub_index = list(sub_sig.parameters.keys()).index(name)
sub_param = sub_sig.parameters[name]
if (super_param.kind != sub_param.kind
and not (super_param.kind == Parameter.POSITIONAL_ONLY and
sub_param.kind == Parameter.POSITIONAL_OR_KEYWORD)
and
not (super_param.kind == Parameter.KEYWORD_ONLY and
sub_param.kind == Parameter.POSITIONAL_OR_KEYWORD)):
raise TypeError(
f"`{name}` is not `{super_param.kind.description}`")
elif super_index != sub_index and super_param.kind != Parameter.KEYWORD_ONLY:
raise TypeError(f"`{name}` is not parameter `{super_index}`")
elif (super_param.annotation != Parameter.empty
and sub_param.annotation != Parameter.empty and
not issubtype(super_param.annotation, sub_param.annotation)):
raise TypeError(
f"`{name} must be a supertype of `{super_param.annotation}`"
)
def _validate_target(
target_name: Text,
target_type: Text,
expected_type: Type,
schema: GraphSchema,
) -> None:
if target_name not in schema.nodes:
raise GraphSchemaValidationException(
f"Graph schema specifies invalid {target_type} target '{target_name}'. "
f"Please make sure specify a valid node name as target.")
if any(target_name in node.needs.values()
for node in schema.nodes.values()):
raise GraphSchemaValidationException(
f"One graph node uses the {target_type} target '{target_name}' as input. "
f"This is not allowed as NLU prediction and Core prediction are run "
f"separately.")
target_node = schema.nodes[target_name]
_, target_return_type = _get_parameter_information(target_node.uses,
target_node.fn)
if not typing_utils.issubtype(target_return_type, expected_type):
raise GraphSchemaValidationException(
f"Your {target_type} model's output component "
f"'{target_node.uses.__name__}' returns an invalid return "
f"type '{target_return_type}'. This is not allowed. The {target_type} "
f"model's last component is expected to return the type '{expected_type}'. "
f"See {DOCS_URL_GRAPH_COMPONENTS} for more information.")
def ensure_return_type_compatibility(super_sig, sub_sig):
if (super_sig.return_annotation != Signature.empty
and sub_sig.return_annotation != Signature.empty and not issubtype(
sub_sig.return_annotation, super_sig.return_annotation)):
raise TypeError(
f"`{sub_sig.return_annotation}` is not a `{super_sig.return_annotation}`."
)
def type_check_expr(sigma: dict, func_sigma: dict, expected, expr: Expr):
actual = type_infer_expr(sigma, func_sigma, expr)
if actual == TANY and isinstance(expr, Var):
sigma[expr.name] = expected
return expected
if actual == expected or typing_utils.issubtype(actual, expected):
return actual
else:
raise TypeError(
f'{expr}: expected type {expected}, actual type {actual};\
issubtype({actual}, {expected})={typing_utils.issubtype(actual, expected)}'
)
def _issubtype(left, right):
if _contains_unbound_typevar(left):
return True
if right is None:
return True
if _contains_unbound_typevar(right):
return True
try:
return issubtype(left, right)
except TypeError:
# Ignore all broken cases
return True
def build_variable_value_tuple(
self, variable_values: Tuple[hints.VariableValue,
...]) -> VariableValueTuple:
"""Prepares and validates a raw tuple of variable values to be passed into
`compute_residual_vector` or `compute_residual_jacobians`.
Slightly sketchy: checks the type hinting on `compute_residual_vector` and if
the user expects a named tuple, we wrap the input accordingly. Otherwise, we
just cast and return the input."""
assert isinstance(variable_values, tuple)
output: VariableValueTuple
try:
value_type: Type[VariableValueTuple] = get_type_hints(
self.compute_residual_vector)["variable_values"]
except KeyError as e:
raise NotImplementedError(
f"Missing type hints for {type(self).__name__}.compute_residual_vector"
) from e
# Function should be hinted with a tuple of some kind, but not `tuple` itself
assert issubtype(value_type, tuple), value_type is not tuple
# Heuristic: evaluates to `True` for NamedTuple types but `False` for
# `Tuple[...]` types. Note that standard superclass checking approaches don't
# work for NamedTuple types.
if type(value_type) is type:
# Hint is `NamedTuple`
tuple_content_types = tuple(get_type_hints(value_type).values())
output = value_type(*variable_values)
else:
# Hint is `typing.Tuple` annotation
tuple_content_types = get_args(value_type)
output = cast(VariableValueTuple, variable_values)
# Handle Ellipsis in type hints, eg `Tuple[SomeType, ...]`
if len(tuple_content_types
) == 2 and tuple_content_types[1] is Ellipsis:
tuple_content_types = tuple_content_types[0:1] * len(
variable_values)
# Validate expected and received types
assert len(variable_values) == len(tuple_content_types)
for i, (value, expected_type) in enumerate(
zip(variable_values, tuple_content_types)):
assert isinstance(value, expected_type), (
f"Variable value type hint inconsistency: expected {expected_type} at, "
f"position {i} but got {type(value)}.")
return output
def _validate_types_of_reserved_keywords(params: Dict[Text, ParameterInfo],
node: SchemaNode,
fn_name: Text) -> None:
for param_name, param in params.items():
if param_name in KEYWORDS_EXPECTED_TYPES:
if not typing_utils.issubtype(param.type_annotation,
KEYWORDS_EXPECTED_TYPES[param_name]):
raise GraphSchemaValidationException(
f"Your model uses a component '{node.uses.__name__}' which has an "
f"incompatible type '{param.type_annotation}' for "
f"the '{param_name}' parameter in its '{fn_name}' method. "
f"The expected type is '{KEYWORDS_EXPECTED_TYPES[param_name]}'."
f"See {DOCS_URL_GRAPH_COMPONENTS} for more information.")
def _validate_types_of_reserved_keywords(params: Dict[Text, ParameterInfo],
node_name: Text, node: SchemaNode,
fn_name: Text) -> None:
for param_name, param in params.items():
if param_name in KEYWORDS_EXPECTED_TYPES:
if not typing_utils.issubtype(param.type_annotation,
KEYWORDS_EXPECTED_TYPES[param_name]):
raise GraphSchemaValidationException(
f"Node '{node_name}' uses a graph component "
f"'{node.uses.__name__}' which has an incompatible type "
f"'{param.type_annotation}' for the '{param_name}' parameter in "
f"its '{fn_name}' method. Expected type "
f"'{ KEYWORDS_EXPECTED_TYPES[param_name]}'.")
def _validate_parent_return_type(
node_name: Text,
node: SchemaNode,
parent_name: Text,
parent: SchemaNode,
required_type: TypeAnnotation,
) -> None:
_, parent_return_type = _get_parameter_information(parent_name,
parent.uses, parent.fn)
if not typing_utils.issubtype(parent_return_type, required_type):
raise GraphSchemaValidationException(
f"Parent of node '{node_name}' returns type "
f"'{parent_return_type}' but type '{required_type}' "
f"was expected by component '{node.uses.__name__}'.")
def inject_to(self, func: Callable[..., Awaitable]) -> Iterator[Any]:
sig: Signature = signature(func)
keywords: List[str] = [name for name in sig.parameters.keys()]
combinations: Iterator[Tuple[Any, ...]] = product(
*(self._dependencies.get(p.annotation, [])
for p in sig.parameters.values()))
async def _wrapper(**kwargs: Any) -> List[Any]:
return [await func(**kwargs)]
f: Callable[..., Awaitable] = _wrapper \
if not issubtype(sig.return_annotation, List[Any]) else func # type: ignore
for values in combinations:
yield f(**dict(zip(keywords, values)))
def _validate_parent_return_type(
node: SchemaNode,
parent_node: Optional[SchemaNode],
parent_return_type: TypeAnnotation,
required_type: TypeAnnotation,
) -> None:
if not typing_utils.issubtype(parent_return_type, required_type):
parent_node_text = ""
if parent_node:
parent_node_text = f" by the component '{parent_node.uses.__name__}'"
raise GraphSchemaValidationException(
f"Your component '{node.uses.__name__}' expects an input of type "
f"'{required_type}' but it receives an input of type '{parent_return_type}'"
f"{parent_node_text}. "
f"Please make sure that you registered "
f"your component correctly and and that your model configuration is "
f"valid."
f"See {DOCS_URL_GRAPH_COMPONENTS} for more information.")
def _validate_run_fn_return_type(node: SchemaNode, return_type: Type,
is_training: bool) -> None:
if return_type == inspect.Parameter.empty:
raise GraphSchemaValidationException(
f"Your model uses a component '{node.uses.__name__}' whose "
f"method '{node.fn}' does not have a type annotation for "
f"its return value. Type annotations are required for all "
f"components to validate your model's structure."
f"See {DOCS_URL_GRAPH_COMPONENTS} for more information.")
# TODO: Handle forward references here
if typing_utils.issubtype(return_type, list):
return_type = typing_utils.get_args(return_type)[0]
if is_training and not isinstance(return_type, Fingerprintable):
raise GraphSchemaValidationException(
f"Your model uses a component '{node.uses.__name__}' whose method "
f"'{node.fn}' does not return a fingerprintable "
f"output. This is required for proper caching between model trainings. "
f"Please make sure you're using a return type which implements the "
f"'{Fingerprintable.__name__}' protocol.")
def test_is_subtype():
# Any
assert issubtype(typing.List, typing.Any)
assert issubtype(typing.Any, typing.Any)
# Self
assert issubtype(list, list)
assert issubtype(typing.List, typing.List)
assert not issubtype(list, dict)
assert not issubtype(typing.List, typing.Dict)
# None
assert issubtype(None, type(None))
assert issubtype(type(None), None)
assert issubtype(None, None)
# alias
assert issubtype(list, typing.List)
assert issubtype(typing.List, list)
assert issubtype(bytes, typing.ByteString)
# Subclass
assert issubtype(list, typing.Sequence)
# FileLike
with open("test", "wb") as file_ref:
assert issubtype(type(file_ref), typing.BinaryIO)
with open("test", "rb") as file_ref:
assert issubtype(type(file_ref), typing.BinaryIO)
with open("test", "w") as file_ref:
assert issubtype(type(file_ref), typing.TextIO)
with open("test", "r") as file_ref:
assert issubtype(type(file_ref), typing.TextIO)
assert issubtype(type(io.BytesIO(b"0")), typing.BinaryIO)
assert issubtype(type(io.StringIO("0")), typing.TextIO)
# subscribed generic
assert issubtype(typing.List[int], list)
assert issubtype(typing.List[typing.List], list)
assert not issubtype(list, typing.List[int])
# Union
assert issubtype(list, typing.Union[typing.List, typing.Tuple])
assert issubtype(typing.Union[list, tuple], typing.Union[list, tuple,
None])
assert issubtype(typing.Union[list, tuple], typing.Sequence)
assert not issubtype(list, typing.Union[typing.Tuple, typing.Set])
assert not issubtype(typing.Tuple[typing.Union[int, None]],
typing.Tuple[None])
# Nested containers
assert issubtype(typing.List[int], typing.List[int])
assert issubtype(typing.List[typing.List], typing.List[typing.Sequence])
assert issubtype(typing.Dict[typing.List, int],
typing.Dict[typing.Sequence, int])
assert issubtype(
typing.Callable[[typing.List, int], int],
typing.Callable[[typing.Sequence, int], int],
)
assert not issubtype(
typing.Callable[[typing.Sequence, int], int],
typing.Callable[[typing.List, int], int],
)
# Callable
assert issubtype(
typing.Callable[[typing.List, int], None],
typing.Callable[[list, int], None],
)
assert issubtype(
typing.Callable[[typing.List, int], None],
typing.Callable[[typing.Sequence, int], None],
)
assert issubtype(
typing.Callable[[typing.List[int], int], None],
typing.Callable[[typing.Sequence[int], int], None],
)
assert not issubtype(
typing.Callable[[typing.List[int], int], None],
typing.Callable[[typing.List[str], int], None],
)
assert not issubtype(
typing.Callable[[typing.List[int], int], None],
typing.Callable[[typing.List[int], int], int],
)
assert not issubtype(
typing.Callable[[typing.List[int], int, None], None],
typing.Callable[[typing.List[int], int], None],
)
# ForwardRef
assert issubtype(int, JSON, forward_refs={'JSON': JSON})
assert issubtype(str, JSON, forward_refs={'JSON': JSON})
assert issubtype(typing.Dict[str, str], JSON, forward_refs={'JSON': JSON})
assert not issubtype(
typing.Dict[str, bytes], JSON, forward_refs={'JSON': JSON})
assert issubtype(typing.Dict[str, str],
typing.Union[JSON, bytes],
forward_refs={'JSON': JSON})
assert not issubtype(typing.Dict[str, bytes],
typing.Union[JSON, bytes],
forward_refs={'JSON': JSON})
# Ellipsis
assert issubtype(typing.Tuple[list], typing.Tuple[list, ...])
assert issubtype(typing.Tuple[typing.List], typing.Tuple[list, ...])
assert issubtype(typing.Tuple[list, list], typing.Tuple[typing.Sequence,
...])
assert not issubtype(typing.Tuple[list, int], typing.Tuple[typing.Sequence,
...])
assert issubtype(typing.Tuple[list, ...], typing.Tuple[list, ...])
assert issubtype(typing.Tuple[list, ...], typing.Tuple[typing.Sequence,
...])
assert not issubtype(typing.Tuple[list, ...], typing.Tuple[list])
# TypeVar
T1 = typing.TypeVar("T1")
T2 = typing.TypeVar("T2")
T3 = typing.TypeVar("T3", bound=str)
T4 = typing.TypeVar("T4", bound="typing.Union[list, tuple]")
assert issubtype(T1, T1)
assert not issubtype(T1, T2) and issubtype(T1, T2) is unknown
assert not issubtype(T3, T4) and issubtype(T3, T4) is not unknown
assert issubtype(T3, str)
assert issubtype(T4, typing.Sequence)