def __setattr__(self, key, value):
if hasattr(self, key) or key in get_type_hints(self.__class__,
include_extras=True):
if not check_int((self.__class__, key),
value,
suppress_warning_for_unresolved_hints=True):
typ = get_type_hints(self.__class__, include_extras=True)[key]
r = get_range(typ)
if r is not None:
raise IntegerBoundError(
f(
_("The value '{value}' does not fit into the field '{key}'. "
"The value must fit into the range [{r.min},{r.max}]."
)))
super().__setattr__(key, value)
def _infer_multiple_outputs(self):
try:
return_type = typing_extensions.get_type_hints(self.function).get("return", Any)
except Exception: # Can't evaluate retrurn type.
return False
ttype = getattr(return_type, "__origin__", return_type)
return ttype == dict or ttype == Dict
def _collect_fields(cls) -> Dict[str, Tuple[AnnotationInfo, FieldInfo]]:
"""Centralize publicly named fields and their corresponding annotations."""
annotations = get_type_hints( # pylint:disable=unexpected-keyword-arg
cls, include_extras=True)
attrs = cls._get_model_attrs()
missing = []
for name, attr in attrs.items():
if inspect.isroutine(attr):
continue
if not _is_field(name):
annotations.pop(name, None)
elif name not in annotations:
missing.append(name)
if missing:
raise SchemaInitError(f"Found missing annotations: {missing}")
fields = {}
for field_name, annotation in annotations.items():
field = attrs[field_name] # __init_subclass__ guarantees existence
if not isinstance(field, FieldInfo):
raise SchemaInitError(
f"'{field_name}' can only be assigned a 'Field', " +
f"not a '{type(field)}.'")
fields[field.name] = (AnnotationInfo(annotation), field)
return fields
def get_return_type_hints(func: Callable) -> Any:
"""Return function return types.
This is because `get_type_hints` result in error for some types in older
versions of python and also that `__annotations__` contains only string, not types.
Note:
Sometimes it may use eval as literal_eval cannot use users globals so types like pd.DataFrame would
fail. Therefore do not use it for evaluating types of users input for sake of security.
Args:
func (Callable): Function with type hints.
Returns:
Any: Type of return.
Example:
>>> # You can use Union as well as Literal
>>> def union_return() -> int | float:
... return 1
>>> inferred_type = get_return_type_hints(union_return)
>>> 'int' in str(inferred_type) and 'float' in str(inferred_type)
True
>>> def literal_return() -> Literal[1, 2, 3]:
... return 1
>>> inferred_type = get_return_type_hints(literal_return)
>>> 'Literal' in str(inferred_type)
True
"""
if isinstance(func, staticmethod):
func = func.__func__
try:
types = get_type_hints(func).get("return")
except Exception:
types = func.__annotations__.get("return")
if isinstance(types, str) and "Union" in types:
types = eval(types, func.__globals__)
# If Union operator |, e.g. int | str - get_type_hints() result in TypeError
# Convert it to Union
elif isinstance(types, str) and "|" in types:
str_types = [i.strip() for i in types.split("|")]
for i, j in enumerate(str_types):
for k in ["list", "dict", "tuple"]:
if k in j:
str_types[i] = j.replace(k, k.capitalize())
try:
evaluated_types = [eval(i, {**globals(), **func.__globals__}) for i in str_types]
except Exception:
raise RuntimeError("Evaluating of function return type failed. Try it on python 3.9+.")
types = Union[evaluated_types[0], evaluated_types[1]] # type: ignore
if len(evaluated_types) > 2:
for i in evaluated_types[2:]:
types = Union[types, i]
return types
def decode(self, msg: RosMessage, buffer: bytes, offset: int = 0) -> int:
for name, t in get_type_hints(msg, include_extras=True).items(): # type: ignore
if get_origin(t) is Final:
continue
val, offset = self._decode_value(msg, buffer, offset, t)
setattr(msg, name, val)
return offset
def args_to_matchers(
function: Callable[..., Any], ) -> Dict[str, libcst_typing.Matcher]:
"""Extract node matchers from the function arguments.
Untyped arguments will get a `DoNotCare` matcher, while arguments typed and
annoated with a `BaseMatcherNode` will return that matcher.
"""
matchers: Dict[str, libcst_typing.Matcher] = {}
# Create default matchers for all arguments
args = function.__code__.co_varnames[:function.__code__.co_argcount]
if args:
for name in args:
if name == "self":
continue
matchers[name] = libcst.matchers.DoNotCare()
# Check if any of the arguments was annotated with a Matcher
for name, type_declaration in typing_extensions.get_type_hints(
function, include_extras=True).items():
if typing_extensions.get_origin(type_declaration) is Annotated:
# Check if there is a matcher
arg = typing_extensions.get_args(type_declaration)[1]
if isinstance(arg, libcst.matchers.BaseMatcherNode):
matchers[name] = arg
return matchers
def eval_field_types(dataclass: DataClass) -> None:
"""Evaluate field types of a dataclass or its object."""
hints = get_type_hints(dataclass, include_extras=True) # type: ignore
for field in dataclass.__dataclass_fields__.values():
if isinstance(field.type, str):
field.type = hints[field.name]
def add_api_route(
self,
path: str,
endpoint: Callable[..., Any],
**kwargs: Any,
) -> None:
return_type = get_type_hints(endpoint)["return"]
success_result, failure_result = return_type.__args__
endpoint = self._unpacked_container(endpoint)
responses = {}
if get_origin(failure_result) is Union:
for error in failure_result.__args__:
annotation = error.__annotations__["detail"]
responses[error.status_code] = {"model": ApiErrorSchema[annotation]} # type: ignore[valid-type]
else:
annotation = failure_result.__annotations__["detail"]
responses[failure_result.status_code] = {"model": ApiErrorSchema[annotation]} # type: ignore[valid-type]
if kwargs["response_model"] is None:
kwargs["response_model"] = success_result
if kwargs["responses"] is None:
kwargs["responses"] = responses
return super().add_api_route(path, endpoint, **kwargs)
def __initialize__(self):
"""Effectively gather configuration information"""
# Check if not initialized
if self.__initialized__:
return
self.__initialized__ = True
from .objects import Task
# Get the module
module = inspect.getmodule(self.originaltype)
self._file = Path(inspect.getfile(self.originaltype)).absolute()
self._module = module.__name__
self._package = module.__package__
# The class of the object
self._arguments = ChainMap(
{}, *(tp.arguments
for tp in self.parents())) # type: ChainMap[Argument, Any]
# Add arguments from annotations
for annotation in self.annotations:
annotation.process(self)
# Add task
if self.taskcommandfactory is not None:
self.task = self.taskcommandfactory(self)
elif issubclass(self.basetype, Task):
self.task = self.getpythontaskcommand()
# Add arguments from type hints
from .arguments import TypeAnnotation
if hasattr(self.basetype, "__annotations__"):
typekeys = set(
self.basetype.__dict__.get("__annotations__", {}).keys())
hints = get_type_hints(self.basetype, include_extras=True)
for key, typehint in hints.items():
# Filter out hints from parent classes
if key in typekeys:
options = None
if isinstance(typehint, _AnnotatedAlias):
for value in typehint.__metadata__:
if isinstance(value, TypeAnnotation):
options = value(options)
if options is not None:
try:
self.addArgument(
options.create(key, self.objecttype,
typehint.__args__[0]))
except Exception:
logger.error(
"while adding argument %s of %s",
key,
self.objecttype,
)
raise
def napari_type_hints(obj: Any) -> Dict[str, Any]:
import napari
from .. import layers, viewer
return get_type_hints(obj, {
'napari': napari,
**viewer.__dict__,
**layers.__dict__
})
def _is_valid_typeddict_item(
td: type[TypedDict], key: str, value: Any # type: ignore [valid-type]
) -> bool:
"""Check if `key` and `value` form a valid item for the TypedDict `td`."""
annotations = get_type_hints(td)
if key not in annotations:
return False
if get_origin(annotations[key]) is Literal:
return value in get_args(annotations[key])
return isinstance(value, annotations[key])
def napari_type_hints(obj: Any) -> Dict[str, Any]:
"""variant of get_type_hints with napari namespace awareness."""
import napari
return get_type_hints(
obj,
{
'napari': napari,
**viewer.__dict__,
**layers.__dict__,
**components.__dict__,
},
)
def napari_type_hints(obj: Any) -> Dict[str, Any]:
import napari
from .. import components, layers, viewer
return get_type_hints(
obj,
{
'napari': napari,
**viewer.__dict__,
**layers.__dict__,
'LayerList': components.LayerList,
},
)
def _infer_multiple_outputs(self):
try:
return_type = typing_extensions.get_type_hints(self.function).get("return", Any)
except Exception: # Can't evaluate retrurn type.
return False
# Get the non-subscripted type. The ``__origin__`` attribute is not
# stable until 3.7, but we need to use ``__extra__`` instead.
# TODO: Remove the ``__extra__`` branch when support for Python 3.6 is
# dropped in Airflow 2.3.
if sys.version_info < (3, 7):
ttype = getattr(return_type, "__extra__", return_type)
else:
ttype = getattr(return_type, "__origin__", return_type)
return ttype == dict or ttype == Dict
def provider(func: C) -> C:
"""Decorator that declares `func` as a provider of its return type.
Note, If func returns `Optional[Type]`, it will be registered as a provider
for Type.
Examples
--------
>>> @provider
>>> def provides_int() -> int:
... return 42
"""
return_hint = get_type_hints(func).get('return')
if get_origin(return_hint) == Union:
if ((args := get_args(return_hint)) and len(args) == 2
and type(None) in args):
return_hint = next(a for a in args if a is not type(None)) # noqa
def encode(self, msg: RosMessage, buffer: Optional[bytearray] = None) -> bytearray:
if buffer is None:
buffer = bytearray()
for name, t in get_type_hints(msg, include_extras=True).items(): # type: ignore
origin = get_origin(t)
if origin is Final:
continue
def_factory = t
if origin is Annotated and get_origin(get_args(t)[0]) is list:
def_factory = list
val = getattr(msg, name, def_factory())
self._encode_value(val, t, buffer)
return buffer
def _tf_extension_type_fields(cls): # pylint: disable=no-self-argument
"""An ordered list describing the fields of this ExtensionType.
Returns:
A list of `ExtensionTypeField` objects. Forward references are resolved
if possible, or left unresolved otherwise.
"""
if '_tf_extension_type_cached_fields' in cls.__dict__: # do not inherit.
return cls._tf_extension_type_cached_fields
try:
# Using include_extras=False will replace all Annotated[T, ...] with T.
# The typing_extensions module is used since this is only supported in
# Python 3.9.
type_hints = typing_extensions.get_type_hints(cls,
include_extras=False)
ok_to_cache = True # all forward references have been resolved.
except (NameError, AttributeError):
# Unresolved forward reference -- gather type hints manually.
# * NameError comes from an annotation like `Foo` where class
# `Foo` hasn't been defined yet.
# * AttributeError comes from an annotation like `foo.Bar`, where
# the module `foo` exists but `Bar` hasn't been defined yet.
# Note: If a user attempts to instantiate a `ExtensionType` type that
# still has unresolved forward references (e.g., because of a typo or a
# missing import), then the constructor will raise an exception.
type_hints = {}
for base in reversed(cls.__mro__):
type_hints.update(base.__dict__.get('__annotations__', {}))
ok_to_cache = False
fields = []
for (name, value_type) in type_hints.items():
default = getattr(
cls, name, extension_type_field.ExtensionTypeField.NO_DEFAULT)
fields.append(
extension_type_field.ExtensionTypeField(
name, value_type, default))
fields = tuple(fields)
if ok_to_cache:
cls._tf_extension_type_cached_fields = fields
return fields
def transform_function_to_interface(
fn: typing.Callable,
docstring: Optional[Docstring] = None) -> Interface:
"""
From the annotations on a task function that the user should have provided, and the output names they want to use
for each output parameter, construct the TypedInterface object
For now the fancy object, maybe in the future a dumb object.
"""
type_hints = get_type_hints(fn, include_extras=True)
signature = inspect.signature(fn)
return_annotation = type_hints.get("return", None)
outputs = extract_return_annotation(return_annotation)
for k, v in outputs.items():
outputs[k] = _change_unrecognized_type_to_pickle(v)
inputs = OrderedDict()
for k, v in signature.parameters.items():
annotation = type_hints.get(k, None)
default = v.default if v.default is not inspect.Parameter.empty else None
# Inputs with default values are currently ignored, we may want to look into that in the future
inputs[k] = (_change_unrecognized_type_to_pickle(annotation), default)
# This is just for typing.NamedTuples - in those cases, the user can select a name to call the NamedTuple. We
# would like to preserve that name in our custom collections.namedtuple.
custom_name = None
if hasattr(return_annotation, "__bases__"):
bases = return_annotation.__bases__
if len(bases) == 1 and bases[0] == tuple and hasattr(
return_annotation, "_fields"):
if hasattr(return_annotation,
"__name__") and return_annotation.__name__ != "":
custom_name = return_annotation.__name__
return Interface(inputs,
outputs,
output_tuple_name=custom_name,
docstring=docstring)
def generate_function_stub(func) -> Tuple[Set[str], str]:
"""Generate a stub and imports for a function."""
sig = inspect.signature(func)
globalns = {**getattr(func, '__globals__', {})}
globalns.update(vars(typing))
globalns.update(getattr(func, '_forwardrefns_', {}))
hints = get_type_hints(func, globalns)
sig = sig.replace(
parameters=[
p.replace(annotation=hints.get(p.name, p.empty))
for p in sig.parameters.values()
],
return_annotation=hints.get('return', inspect.Parameter.empty),
)
imports = set()
for hint in hints.values():
imports.update(set(_iter_imports(hint)))
imports -= {'typing'}
doc = f'"""{func.__doc__}"""' if func.__doc__ else '...'
return imports, f'def {func.__name__}{sig}:\n {doc}\n'
def processor(func: C) -> C:
"""Decorator that declares `func` as a processor of its first parameter type.
Examples
--------
>>> @processor
>>> def processes_image(image: napari.layers.Image):
... ... # do something with the image
"""
hints = get_type_hints(func)
hints.pop("return", None)
if not hints:
raise TypeError(
f"{func} has no argument type hints. Cannot be a processor.")
hint0 = list(hints.values())[0]
if hint0 is not None:
if get_origin(hint0) == Union:
for arg in get_args(hint0):
if arg is not None:
_PROCESSORS[arg] = func
else:
_PROCESSORS[hint0] = func
return func
def generate_class_stubs(cls: Type) -> Tuple[Set[str], str]:
"""Generate a stub and imports for a class."""
bases = ", ".join(f'{b.__module__}.{b.__name__}' for b in cls.__bases__)
methods = []
attrs = []
imports = set()
local_names = set(cls.__dict__).union(set(cls.__annotations__))
for sup in cls.mro()[1:]:
local_names.difference_update(set(sup.__dict__))
for methname in sorted(_get_subclass_methods(cls)):
method = getattr(cls, methname)
if not callable(method):
continue
_imports, stub = generate_function_stub(method)
imports.update(_imports)
methods.append(stub)
hints = get_type_hints(cls)
for name, type_ in hints.items():
if name not in local_names:
continue
if hasattr(type_, '__name__'):
hint = f'{type_.__module__}.{type_.__name__}'
else:
hint = repr(type_).replace('typing.', '')
attrs.append(f'{name}: {hint.replace("builtins.", "")}')
imports.update(set(_iter_imports(type_)))
doc = f'"""{cls.__doc__.lstrip()}"""' if cls.__doc__ else '...'
stub = f'class {cls.__name__}({bases}):\n {doc}\n'
stub += textwrap.indent("\n".join(attrs), ' ')
stub += "\n" + textwrap.indent("\n".join(methods), ' ')
return imports, stub
def extract_return_annotation(
return_annotation: Union[Type, Tuple, None]) -> Dict[str, Type]:
"""
The purpose of this function is to sort out whether a function is returning one thing, or multiple things, and to
name the outputs accordingly, either by using our default name function, or from a typing.NamedTuple.
# Option 1
nt1 = typing.NamedTuple("NT1", x_str=str, y_int=int)
def t(a: int, b: str) -> nt1: ...
# Option 2
def t(a: int, b: str) -> typing.NamedTuple("NT1", x_str=str, y_int=int): ...
# Option 3
def t(a: int, b: str) -> typing.Tuple[int, str]: ...
# Option 4
def t(a: int, b: str) -> (int, str): ...
# Option 5
def t(a: int, b: str) -> str: ...
# Option 6
def t(a: int, b: str) -> None: ...
# Options 7/8
def t(a: int, b: str) -> List[int]: ...
def t(a: int, b: str) -> Dict[str, int]: ...
Note that Options 1 and 2 are identical, just syntactic sugar. In the NamedTuple case, we'll use the names in the
definition. In all other cases, we'll automatically generate output names, indexed starting at 0.
"""
# Handle Option 6
# We can think about whether we should add a default output name with type None in the future.
if return_annotation in (None, type(None), inspect.Signature.empty):
return {}
# This statement results in true for typing.Namedtuple, single and void return types, so this
# handles Options 1, 2. Even though NamedTuple for us is multi-valued, it's a single value for Python
if isinstance(return_annotation, Type) or isinstance(
return_annotation, TypeVar):
# isinstance / issubclass does not work for Namedtuple.
# Options 1 and 2
bases = return_annotation.__bases__ # type: ignore
if len(bases) == 1 and bases[0] == tuple and hasattr(
return_annotation, "_fields"):
logger.debug(f"Task returns named tuple {return_annotation}")
return dict(get_type_hints(return_annotation, include_extras=True))
if hasattr(return_annotation, "__origin__"
) and return_annotation.__origin__ is tuple: # type: ignore
# Handle option 3
logger.debug(f"Task returns unnamed typing.Tuple {return_annotation}")
if len(return_annotation.__args__) == 1: # type: ignore
raise FlyteValidationException(
"Tuples should be used to indicate multiple return values, found only one return variable."
)
return OrderedDict(
zip(list(output_name_generator(len(return_annotation.__args__))),
return_annotation.__args__) # type: ignore
)
elif isinstance(return_annotation, tuple):
if len(return_annotation) == 1:
raise FlyteValidationException(
"Please don't use a tuple if you're just returning one thing.")
return OrderedDict(
zip(list(output_name_generator(len(return_annotation))),
return_annotation))
else:
# Handle all other single return types
logger.debug(f"Task returns unnamed native tuple {return_annotation}")
return {default_output_name(): return_annotation}
def unannotate(hint: Any) -> Any:
"""Recursively remove Annotated type hints."""
class Temp:
__annotations__ = dict(hint=hint)
return get_type_hints(Temp)["hint"]
def match(self, pattern):
class _Temporary:
p: pattern
parsed_pattern = get_type_hints(_Temporary,
localns={'pattern': pattern},
include_extras=True)['p']
if get_origin(parsed_pattern) is Annotated:
parsed_pattern, recursive = get_args(parsed_pattern)
else:
recursive = False
annotations = get_type_hints(parsed_pattern, include_extras=True)
for name, expected in annotations.items():
#print(f'{expected=} {self[name]=}')
expected_any = expected is Any
got_something = self[name] is not None
if expected_any and not got_something:
break
expected_tuple = get_origin(expected) is tuple
got_something = self[name] is not None
if expected_tuple and not got_something:
break
elif expected_tuple and got_something:
exp_timestamp, exp_value = get_args(expected)
got_timestamp, got_value = self[name]
#print(f'{exp_timestamp=} {exp_value=}')
#print(f'{got_timestamp=} {got_value=}')
expected_any = exp_timestamp is Any
got_something = got_timestamp is not None
if expected_any and not got_something:
break
expected_any = exp_value is Any
got_something = got_value is not None
if expected_any and not got_something:
break
if get_origin(exp_value) is Literal:
exp_value = get_args(exp_value)[0]
if exp_value != got_value:
break
expected_list = get_origin(expected) is list
got_something = self[name] is not None
if expected_list and not got_something:
break
elif expected_list and got_something:
exp_class = get_args(expected)[0]
for tree in self.children:
if next(tree.match(exp_class), None) is not None:
break
else:
yield self
return
if recursive:
for tree in self.children:
yield from tree.match(pattern)
def __init__(
self,
tp: type,
identifier: Union[str, Identifier] = None,
):
"""Creates a type"""
from .objects import Config, TypeConfig
# Task related attributes
self.taskcommandfactory = None
self.task = None
self._title = None
# Get the identifier
if identifier is None and hasattr(tp, "__xpmid__"):
__xpmid__ = getattr(tp, "__xpmid__")
if inspect.ismethod(__xpmid__):
identifier = Identifier(__xpmid__())
elif "__xpmid__" in tp.__dict__:
identifier = Identifier(__xpmid__)
package = tp.__module__.lower()
name = tp.__qualname__.lower()
if identifier is None:
qname = f"{package}.{name}"
assert (getattr(sys, "_called_from_test", False)
or "<locals>" not in qname
), "Configurations should not be within functions"
identifier = Identifier(qname)
super().__init__(identifier, None)
# --- Creates the config type and not config type
self.originaltype = tp
if not issubclass(tp, Config):
# Adds Config as a base class if not present
__bases__ = () if tp.__bases__ == (object, ) else tp.__bases__
__dict__ = dict(tp.__dict__)
__dict__ = {
key: value
for key, value in tp.__dict__.items()
if key not in ObjectType.FORBIDDEN_KEYS
}
self.basetype = type(tp.__name__, (Config, ) + __bases__, __dict__)
self.basetype.__module__ = tp.__module__
self.basetype.__qualname__ = tp.__qualname__
else:
self.basetype = tp
# Create the type-specific configuration class
__configbases__ = tuple(
s.__getxpmtype__().configtype for s in tp.__bases__
if issubclass(s, Config) and (s is not Config)) or (TypeConfig, )
self.configtype = type("TypeConfig",
__configbases__ + (self.basetype, ), {})
self.configtype.__qualname__ = f"{self.basetype.__qualname__}.TypeConfig"
self.configtype.__module__ = tp.__module__
# Create the type-specific object class
# (now, the same as basetype - TODO: remove references)
self.objecttype = self.basetype # type: type
self.basetype._ = self.configtype
# Return type is used by tasks to change the output
if hasattr(self.basetype, "taskoutputs") or False:
self.returntype = get_type_hints(
getattr(self.basetype,
"taskoutputs")).get("return", typing.Any)
else:
self.returntype = self.objecttype
# Registers ourselves
self.basetype.__xpmtype__ = self
self.configtype.__xpmtype__ = self
# Other initializations
self.__initialized__ = False
self._runtype = None
self.annotations = []
self._deprecated = False
from typing_extensions import Annotated
class Description:
def __init__(self, description: str) -> None:
self.description = description
def hello(*, name: Annotated[str, Description("the name of person")]) -> None:
print(f"hello {name}")
if __name__ == "__main__":
from typing_extensions import get_type_hints
print(get_type_hints(hello))
# {'name': <class 'str'>, 'return': <class 'NoneType'>}
print(get_type_hints(hello, include_extras=True))
# {'name': typing_extensions.Annotated[str, <__main__.Description object at 0x10cd1e730>], 'return': <class 'NoneType'>}
from typing_extensions import get_args, get_origin
hints = get_type_hints(hello, include_extras=True)
print(get_args(hints["name"]))
# (<class 'str'>, <__main__.Description object at 0x106427730>)
print(get_origin(hints["name"]))
# <class 'typing_extensions.Annotated'>
from typing_extensions import Annotated
class Description:
def __init__(self, description: str) -> None:
self.description = description
def hello(*, name: Annotated[str, Description("the name of person")]) -> None:
print(f"hello {name}")
if __name__ == "__main__":
from typing_extensions import get_type_hints
hints = get_type_hints(hello, include_extras=True)
import typing_inspect
print(typing_inspect.get_args(hints["name"]))
# (<class 'str'>,)
print(hasattr(hints["name"], "__metadata__"))
# True
print(hints["name"].__metadata__)
# (<__main__.Description object at 0x104b9b730>,)
from typing import ClassVar, Optional, TypeVar, Type, Union
from typing_extensions import Annotated, get_type_hints
class Param:
def __init__(self, help: str = None):
self.help = help
class A:
x: Annotated[int, Param(help="help")] = 2
def a(self):
return self.x + 1
print(get_type_hints(A, include_extras=True))
a = A()
x = a.a()
print(x)
