def _parse_annotation(self, raw_annotation: Type) -> None:
"""Parse key information from annotation.
:param annotation: A subscripted type.
:returns: Annotation
"""
self.raw_annotation = raw_annotation
self.origin = self.arg = None
self.optional = typing_inspect.is_optional_type(raw_annotation)
if self.optional and typing_inspect.is_union_type(raw_annotation):
# Annotated with Optional or Union[..., NoneType]
if LEGACY_TYPING: # pragma: no cover
# get_args -> ((pandera.typing.Index, <class 'str'>), <class 'NoneType'>)
self.origin, self.arg = typing_inspect.get_args(
raw_annotation)[0]
# get_args -> (pandera.typing.Index[str], <class 'NoneType'>)
raw_annotation = typing_inspect.get_args(raw_annotation)[0]
if not (self.optional and LEGACY_TYPING):
self.origin = typing_inspect.get_origin(raw_annotation)
args = typing_inspect.get_args(raw_annotation)
self.arg = args[0] if args else args
self.metadata = getattr(self.arg, "__metadata__", None)
if self.metadata:
self.arg = typing_inspect.get_args(self.arg)[0]
self.literal = typing_inspect.is_literal_type(self.arg)
if self.literal:
self.arg = typing_inspect.get_args(self.arg)[0]
def validate(self,
raw_config_vars: Optional[RawConfigVarsType] = None) -> Any:
"""
Validate the configuration value.
:param raw_config_vars:
:returns: The validated value.
"""
if raw_config_vars is None:
raw_config_vars = {}
if self.config_var.rtype is None:
self.config_var.rtype = self.config_var.dtype
if self.config_var.dtype in self._dtypes:
return getattr(self,
f"visit_{self._dtypes[self.config_var.dtype]}")(
raw_config_vars)
elif get_origin(self.config_var.dtype) in {list, List}:
return self.visit_list(raw_config_vars)
elif get_origin(self.config_var.dtype) in {dict, Dict}:
return self.visit_dict(raw_config_vars)
elif get_origin(self.config_var.dtype) is Union:
return self.visit_union(raw_config_vars)
elif is_literal_type(self.config_var.dtype):
return self.visit_literal(raw_config_vars)
else:
self.unknown_type()
def parse_annotation(raw_annotation: Type) -> AnnotationInfo:
"""Parse key information from annotation.
:param annotation: A subscripted type.
:returns: Annotation
"""
optional = typing_inspect.is_optional_type(raw_annotation)
if optional:
# e.g: Typing.Union[pandera.typing.Index[str], NoneType]
if _LEGACY_TYPING: # pragma: no cover
# get_args -> ((pandera.typing.Index, <class 'str'>), <class 'NoneType'>)
origin, arg = typing_inspect.get_args(raw_annotation)[0]
return AnnotationInfo(origin, arg, optional)
# get_args -> (pandera.typing.Index[str], <class 'NoneType'>)
raw_annotation = typing_inspect.get_args(raw_annotation)[0]
origin = typing_inspect.get_origin(raw_annotation)
args = typing_inspect.get_args(raw_annotation)
arg = args[0] if args else args
literal = typing_inspect.is_literal_type(arg)
if literal:
arg = typing_inspect.get_args(arg)[0]
return AnnotationInfo(origin=origin,
arg=arg,
optional=optional,
literal=literal)
def _parse_annotation(self, raw_annotation: Type) -> None:
"""Parse key information from annotation.
:param annotation: A subscripted type.
:returns: Annotation
"""
self.raw_annotation = raw_annotation
self.optional = typing_inspect.is_optional_type(raw_annotation)
if self.optional:
# e.g: Typing.Union[pandera.typing.Index[str], NoneType]
if _LEGACY_TYPING: # pragma: no cover
# get_args -> ((pandera.typing.Index, <class 'str'>), <class 'NoneType'>)
self.origin, self.arg = typing_inspect.get_args(
raw_annotation)[0]
# get_args -> (pandera.typing.Index[str], <class 'NoneType'>)
raw_annotation = typing_inspect.get_args(raw_annotation)[0]
if not (self.optional and _LEGACY_TYPING):
self.origin = typing_inspect.get_origin(raw_annotation)
args = typing_inspect.get_args(raw_annotation)
self.arg = args[0] if args else args
self.literal = typing_inspect.is_literal_type(self.arg)
if self.literal:
self.arg = typing_inspect.get_args(self.arg)[0]
def _parse_annotation(self, raw_annotation: Type) -> None:
"""Parse key information from annotation.
:param annotation: A subscripted type.
:returns: Annotation
"""
self.raw_annotation = raw_annotation
self.origin = self.arg = None
self.optional = typing_inspect.is_optional_type(raw_annotation)
if self.optional and typing_inspect.is_union_type(raw_annotation):
# Annotated with Optional or Union[..., NoneType]
# get_args -> (pandera.typing.Index[str], <class 'NoneType'>)
raw_annotation = typing_inspect.get_args(raw_annotation)[0]
self.origin = typing_inspect.get_origin(raw_annotation)
# Replace empty tuple returned from get_args by None
args = typing_inspect.get_args(raw_annotation) or None
self.arg = args[0] if args else args
self.metadata = getattr(self.arg, "__metadata__", None)
if self.metadata:
self.arg = typing_inspect.get_args(self.arg)[0]
self.literal = typing_inspect.is_literal_type(self.arg)
if self.literal:
self.arg = typing_inspect.get_args(self.arg)[0]
self.default_dtype = getattr(raw_annotation, "default_dtype", None)
def visit_list(self, raw_config_vars: RawConfigVarsType) -> List:
"""
Used to validate and convert :class:`list` values.
:param raw_config_vars:
"""
# Lists of strings, numbers, Unions and Literals
buf = []
data = optional_getter(raw_config_vars, self.config_var,
self.config_var.required)
if isinstance(data, str) or not isinstance(data, Iterable):
raise ValueError(
f"'{self.config_var.__name__}' must be a List of {self.config_var.dtype.__args__[0]}"
) from None
if get_origin(self.config_var.dtype.__args__[0]) is Union:
for obj in data:
if not check_union(obj, self.config_var.dtype.__args__[0]):
raise ValueError(
f"'{self.config_var.__name__}' must be a "
f"List of {self.config_var.dtype.__args__[0]}"
) from None
elif is_literal_type(self.config_var.dtype.__args__[0]):
for obj in data:
# if isinstance(obj, str):
# obj = obj.lower()
if obj not in get_literal_values(
self.config_var.dtype.__args__[0]):
raise ValueError(
f"Elements of '{self.config_var.__name__}' must be "
f"one of {get_literal_values(self.config_var.dtype.__args__[0])}"
) from None
else:
for obj in data:
if not check_union(obj, self.config_var.dtype):
raise ValueError(
f"'{self.config_var.__name__}' must be a List of {self.config_var.dtype.__args__[0]}"
) from None
try:
for obj in data:
if self.config_var.rtype.__args__[0] in {
int, str, float, bool
}:
buf.append(self.config_var.rtype.__args__[0](obj))
else:
buf.append(obj)
return buf
except ValueError:
raise ValueError(
f"Values in '{self.config_var.__name__}' must be {self.config_var.rtype.__args__[0]}"
) from None
def __init__(self,
type_: Type[LiteralHintT],
hint_name: HintName,
default: LiteralHintT,
description: str) -> None:
assert is_literal_type(type_), f"{hint_name} is not a Literal[]"
self.type_ = type_
self.valid_values: List[LiteralHintT] = list(get_args(type_))
super().__init__(hint_name=hint_name,
default=default,
description=description)
def _repr(val: t.Any) -> str:
assert val is not None
if types.is_none_type(val):
return 'NoneType'
elif ti.is_literal_type(val):
return str(val)
elif ti.is_new_type(val):
nested_type = val.__supertype__
return f'{_qualified_name(val)}[{get_repr(nested_type)}]'
elif ti.is_typevar(val):
tv_constraints = ti.get_constraints(val)
tv_bound = ti.get_bound(val)
if tv_constraints:
constraints_repr = (get_repr(tt) for tt in tv_constraints)
return f'typing.TypeVar(?, {", ".join(constraints_repr)})'
elif tv_bound:
return get_repr(tv_bound)
else:
return 'typing.Any'
elif ti.is_optional_type(val):
optional_args = ti.get_args(val, True)[:-1]
nested_union = len(optional_args) > 1
optional_reprs = (get_repr(tt) for tt in optional_args)
if nested_union:
return f'typing.Optional[typing.Union[{", ".join(optional_reprs)}]]'
else:
return f'typing.Optional[{", ".join(optional_reprs)}]'
elif ti.is_union_type(val):
union_reprs = (get_repr(tt) for tt in ti.get_args(val, True))
return f'typing.Union[{", ".join(union_reprs)}]'
elif ti.is_generic_type(val):
attr_name = val._name
generic_reprs = (get_repr(tt) for tt in ti.get_args(val, evaluate=True))
return f'typing.{attr_name}[{", ".join(generic_reprs)}]'
else:
val_name = _qualified_name(val)
maybe_td_entries = getattr(val, '__annotations__', {}).copy()
if maybe_td_entries:
# we are dealing with typed dict
# that's quite lovely
td_keys = sorted(maybe_td_entries.keys())
internal_members_repr = ', '.join(
'{key}: {type}'.format(key=k, type=get_repr(maybe_td_entries.get(k)))
for k in td_keys
)
return f'{val_name}{{{internal_members_repr}}}'
elif 'TypedDict' == getattr(val, '__name__', ''):
return 'typing_extensions.TypedDict'
else:
return val_name
def get_yaml_type(type_: Type) -> str:
r"""
Get the YAML type that corresponds to the given Python type.
:param type\_:
"""
if type_ in yaml_type_lookup:
return yaml_type_lookup[type_]
elif get_origin(type_) is Union:
dtype = " or ".join(yaml_type_lookup[x] for x in type_.__args__)
return dtype
elif check_type(type_, list, List):
args = get_args(type_)
inner_types: typing.Iterable[str]
if args:
inner_types = (get_yaml_type(x) for x in args
if not isinstance(x, TypeVar))
else:
inner_types = ()
dtype = " or ".join(inner_types)
if dtype:
return f"Sequence of {dtype}"
else:
return "Sequence"
elif check_type(type_, dict, Dict):
args = get_args(type_)
if not args or any(isinstance(t, TypeVar) for t in args):
return "Mapping"
else:
dtype = " to ".join(get_yaml_type(x) for x in get_args(type_))
return f"Mapping of {dtype}"
elif is_literal_type(type_):
types = [y for y in get_literal_values(type_)]
return " or ".join(repr(x) for x in types)
elif isinstance(type_, EnumMeta):
return " or ".join([repr(x._value_) for x in type_])
else:
return str(type_)
def get_json_type(type_: Type) -> Dict[str, Union[str, List, Dict]]:
r"""
Get the type for the JSON schema that corresponds to the given Python type.
:param type\_:
"""
if type_ in json_type_lookup:
return {"type": json_type_lookup[type_]}
elif get_origin(type_) is Union:
return {"type": [get_json_type(t)["type"] for t in type_.__args__]}
elif check_type(type_, list, List):
args = get_args(type_)
if args:
items = get_json_type(args[0])
if items is NotImplemented:
return {"type": "array"}
elif "type" in items:
return {"type": "array", "items": items}
elif "enum" in items:
return {"type": "array", "items": items}
else:
return {"type": "array"}
return {"type": "array"}
elif check_type(type_, dict, Dict):
return {"type": "object"}
elif check_type(type_, Literal) or is_literal_type(type_): # type: ignore
return {"enum": [x for x in get_literal_values(type_)]}
elif isinstance(type_, EnumMeta):
return {"enum": [x._value_ for x in type_]}
elif type_ is bool:
return {"type": ["boolean", "string"]}
else:
return NotImplemented
def get(cls, type_or_hint, *, is_argument: bool = False) -> "TypeChecker":
# This ensures the validity of the type passed (see typing documentation for info)
type_or_hint = is_valid_type(type_or_hint, "Invalid type.",
is_argument)
if type_or_hint is Any:
return AnyTypeChecker()
if is_type(type_or_hint):
return TypeTypeChecker.make(type_or_hint, is_argument)
if is_literal_type(type_or_hint):
return LiteralTypeChecker.make(type_or_hint, is_argument)
if is_generic_type(type_or_hint):
origin = get_origin(type_or_hint)
if issubclass(origin, MappingCol):
return MappingTypeChecker.make(type_or_hint, is_argument)
if issubclass(origin, Collection):
return CollectionTypeChecker.make(type_or_hint, is_argument)
# CONSIDER: how to cater for exhaustible generators?
if issubclass(origin, Iterable):
raise NotImplementedError(
"No type-checker is setup for iterables that exhaust.")
return GenericTypeChecker.make(type_or_hint, is_argument)
if is_tuple_type(type_or_hint):
return TupleTypeChecker.make(type_or_hint, is_argument)
if is_callable_type(type_or_hint):
return CallableTypeChecker.make(type_or_hint, is_argument)
if isclass(type_or_hint):
if is_typed_dict(type_or_hint):
return TypedDictChecker.make(type_or_hint, is_argument)
return ConcreteTypeChecker.make(type_or_hint, is_argument)
if is_union_type(type_or_hint):
return UnionTypeChecker.make(type_or_hint, is_argument)
if is_typevar(type_or_hint):
bound_type = get_bound(type_or_hint)
if bound_type:
return cls.get(bound_type)
constraints = get_constraints(type_or_hint)
if constraints:
union_type_checkers = tuple(
cls.get(type_) for type_ in constraints)
return UnionTypeChecker(Union.__getitem__(constraints),
union_type_checkers)
else:
return AnyTypeChecker()
if is_new_type(type_or_hint):
super_type = getattr(type_or_hint, "__supertype__", None)
if super_type is None:
raise TypeError(
f"No supertype for NewType: {type_or_hint}. This is not allowed."
)
return cls.get(super_type)
if is_forward_ref(type_or_hint):
return ForwardTypeChecker.make(type_or_hint,
is_argument=is_argument)
if is_classvar(type_or_hint):
var_type = get_args(type_or_hint, evaluate=True)[0]
return cls.get(var_type)
raise NotImplementedError(
f"No {TypeChecker.__qualname__} is available for type or hint: '{type_or_hint}'"
)
def field_for_schema(
typ: type,
default=marshmallow.missing,
metadata: Mapping[str, Any] = None,
base_schema: Optional[Type[marshmallow.Schema]] = None,
) -> marshmallow.fields.Field:
"""
Get a marshmallow Field corresponding to the given python type.
The metadata of the dataclass field is used as arguments to the marshmallow Field.
:param typ: The type for which a field should be generated
:param default: value to use for (de)serialization when the field is missing
:param metadata: Additional parameters to pass to the marshmallow field constructor
:param base_schema: marshmallow schema used as a base class when deriving dataclass schema
>>> int_field = field_for_schema(int, default=9, metadata=dict(required=True))
>>> int_field.__class__
<class 'marshmallow.fields.Integer'>
>>> int_field.dump_default
9
>>> field_for_schema(str, metadata={"marshmallow_field": marshmallow.fields.Url()}).__class__
<class 'marshmallow.fields.Url'>
"""
metadata = {} if metadata is None else dict(metadata)
if default is not marshmallow.missing:
metadata.setdefault("dump_default", default)
# 'missing' must not be set for required fields.
if not metadata.get("required"):
metadata.setdefault("load_default", default)
else:
metadata.setdefault("required", True)
# If the field was already defined by the user
predefined_field = metadata.get("marshmallow_field")
if predefined_field:
return predefined_field
# Generic types specified without type arguments
typ = _generic_type_add_any(typ)
# Base types
field = _field_by_type(typ, base_schema)
if field:
return field(**metadata)
if typ is Any:
metadata.setdefault("allow_none", True)
return marshmallow.fields.Raw(**metadata)
if typing_inspect.is_literal_type(typ):
arguments = typing_inspect.get_args(typ)
return marshmallow.fields.Raw(
validate=(marshmallow.validate.Equal(arguments[0])
if len(arguments) == 1 else
marshmallow.validate.OneOf(arguments)),
**metadata,
)
if typing_inspect.is_final_type(typ):
arguments = typing_inspect.get_args(typ)
if arguments:
subtyp = arguments[0]
elif default is not marshmallow.missing:
subtyp = type(default)
else:
subtyp = Any
return field_for_schema(subtyp, default, metadata, base_schema)
# Generic types
generic_field = _field_for_generic_type(typ, base_schema, **metadata)
if generic_field:
return generic_field
# typing.NewType returns a function with a __supertype__ attribute
newtype_supertype = getattr(typ, "__supertype__", None)
if newtype_supertype and inspect.isfunction(typ):
return _field_by_supertype(
typ=typ,
default=default,
newtype_supertype=newtype_supertype,
metadata=metadata,
base_schema=base_schema,
)
# enumerations
if isinstance(typ, EnumMeta):
import marshmallow_enum
return marshmallow_enum.EnumField(typ, **metadata)
# Nested marshmallow dataclass
nested_schema = getattr(typ, "Schema", None)
# Nested dataclasses
forward_reference = getattr(typ, "__forward_arg__", None)
nested = (nested_schema or forward_reference
or _internal_class_schema(typ, base_schema))
return marshmallow.fields.Nested(nested, **metadata)
def _add_argument(self, *name_or_flags, **kwargs) -> None:
"""Adds an argument to self (i.e. the super class ArgumentParser).
Sets the following attributes of kwargs when not explicitly provided:
- type: Set to the type annotation of the argument.
- default: Set to the default value of the argument (if provided).
- required: True if a default value of the argument is not provided, False otherwise.
- action: Set to "store_true" if the argument is a required bool or a bool with default value False.
Set to "store_false" if the argument is a bool with default value True.
- nargs: Set to "*" if the type annotation is List[str], List[int], or List[float].
- help: Set to the argument documentation from the class docstring.
:param name_or_flags: Either a name or a list of option strings, e.g. foo or -f, --foo.
:param kwargs: Keyword arguments.
"""
# Set explicit bool
explicit_bool = self._explicit_bool
# Get variable name
variable = get_argument_name(*name_or_flags)
# Get default if not specified
if hasattr(self, variable):
kwargs['default'] = kwargs.get('default', getattr(self, variable))
# Set required if option arg
if (is_option_arg(*name_or_flags) and variable != 'help'
and 'default' not in kwargs
and kwargs.get('action') != 'version'):
kwargs['required'] = kwargs.get('required',
not hasattr(self, variable))
# Set help if necessary
if 'help' not in kwargs:
kwargs['help'] = '('
# Type
if variable in self._annotations:
kwargs['help'] += type_to_str(
self._annotations[variable]) + ', '
# Required/default
if kwargs.get('required', False):
kwargs['help'] += 'required'
else:
kwargs['help'] += f'default={kwargs.get("default", None)}'
kwargs['help'] += ')'
# Description
if variable in self.class_variables:
kwargs[
'help'] += ' ' + self.class_variables[variable]['comment']
# Set other kwargs where not provided
if variable in self._annotations:
# Get type annotation
var_type = self._annotations[variable]
# If type is not explicitly provided, set it if it's one of our supported default types
if 'type' not in kwargs:
# Unbox Optional[type] and set var_type = type
if get_origin(var_type) in OPTIONAL_TYPES:
var_args = get_args(var_type)
if len(var_args) > 0:
var_type = get_args(var_type)[0]
# If var_type is tuple as in Python 3.6, change to a typing type
# (e.g., (typing.List, <class 'bool'>) ==> typing.List[bool])
if isinstance(var_type, tuple):
var_type = var_type[0][var_type[1:]]
explicit_bool = True
# First check whether it is a literal type or a boxed literal type
if is_literal_type(var_type):
var_type, kwargs['choices'] = get_literals(
var_type, variable)
elif (get_origin(var_type) in (List, list, Set, set)
and len(get_args(var_type)) > 0
and is_literal_type(get_args(var_type)[0])):
var_type, kwargs['choices'] = get_literals(
get_args(var_type)[0], variable)
kwargs['nargs'] = kwargs.get('nargs', '*')
# Handle Tuple type (with type args) by extracting types of Tuple elements and enforcing them
elif get_origin(var_type) in (Tuple, tuple) and len(
get_args(var_type)) > 0:
loop = False
types = get_args(var_type)
# Don't allow Tuple[()]
if len(types) == 1 and types[0] == tuple():
raise ValueError(
'Empty Tuples (i.e. Tuple[()]) are not a valid Tap type '
'because they have no arguments.')
# Handle Tuple[type, ...]
if len(types) == 2 and types[1] == Ellipsis:
types = types[0:1]
loop = True
kwargs['nargs'] = '*'
else:
kwargs['nargs'] = len(types)
var_type = TupleTypeEnforcer(types=types, loop=loop)
if get_origin(var_type) in BOXED_TYPES:
# If List or Set type, set nargs
if (get_origin(var_type) in BOXED_COLLECTION_TYPES
and kwargs.get('action')
not in {'append', 'append_const'}):
kwargs['nargs'] = kwargs.get('nargs', '*')
# Extract boxed type for Optional, List, Set
arg_types = get_args(var_type)
# Set defaults type to str for Type and Type[()]
if len(arg_types) == 0 or arg_types[0] == EMPTY_TYPE:
var_type = str
else:
var_type = arg_types[0]
# Handle the cases of List[bool], Set[bool], Tuple[bool]
if var_type == bool:
var_type = boolean_type
# If bool then set action, otherwise set type
if var_type == bool:
if explicit_bool:
kwargs['type'] = boolean_type
kwargs['choices'] = [
True, False
] # this makes the help message more helpful
else:
action_cond = "true" if kwargs.get(
"required",
False) or not kwargs["default"] else "false"
kwargs['action'] = kwargs.get('action',
f'store_{action_cond}')
elif kwargs.get('action') not in {'count', 'append_const'}:
kwargs['type'] = var_type
if self._underscores_to_dashes:
name_or_flags = [
name_or_flag.replace('_', '-')
for name_or_flag in name_or_flags
]
super(Tap, self).add_argument(*name_or_flags, **kwargs)
def _type_from_runtime(val, ctx):
if isinstance(val, str):
return _eval_forward_ref(val, ctx)
elif isinstance(val, tuple):
# This happens under some Python versions for types
# nested in tuples, e.g. on 3.6:
# > typing_inspect.get_args(Union[Set[int], List[str]])
# ((typing.Set, int), (typing.List, str))
origin = val[0]
if len(val) == 2:
args = (val[1], )
else:
args = val[1:]
return _value_of_origin_args(origin, args, val, ctx)
elif typing_inspect.is_literal_type(val):
args = typing_inspect.get_args(val)
if len(args) == 0:
return KnownValue(args[0])
else:
return unite_values(*[KnownValue(arg) for arg in args])
elif typing_inspect.is_union_type(val):
args = typing_inspect.get_args(val)
return unite_values(*[_type_from_runtime(arg, ctx) for arg in args])
elif typing_inspect.is_tuple_type(val):
args = typing_inspect.get_args(val)
if not args:
return TypedValue(tuple)
elif len(args) == 2 and args[1] is Ellipsis:
return GenericValue(tuple, [_type_from_runtime(args[0], ctx)])
else:
args_vals = [_type_from_runtime(arg, ctx) for arg in args]
return SequenceIncompleteValue(tuple, args_vals)
elif is_instance_of_typing_name(val, "_TypedDictMeta"):
return TypedDictValue({
key: _type_from_runtime(value, ctx)
for key, value in val.__annotations__.items()
})
elif typing_inspect.is_callable_type(val):
return TypedValue(Callable)
elif typing_inspect.is_generic_type(val):
origin = typing_inspect.get_origin(val)
args = typing_inspect.get_args(val)
return _value_of_origin_args(origin, args, val, ctx)
elif GenericAlias is not None and isinstance(val, GenericAlias):
origin = get_origin(val)
args = get_args(val)
return GenericValue(origin,
[_type_from_runtime(arg, ctx) for arg in args])
elif isinstance(val, type):
if val is type(None):
return KnownValue(None)
return TypedValue(val)
elif val is None:
return KnownValue(None)
elif is_typing_name(val, "NoReturn"):
return NO_RETURN_VALUE
elif val is typing.Any:
return UNRESOLVED_VALUE
elif hasattr(val, "__supertype__"):
if isinstance(val.__supertype__, type):
# NewType
return NewTypeValue(val)
elif typing_inspect.is_tuple_type(val.__supertype__):
# TODO figure out how to make NewTypes over tuples work
return UNRESOLVED_VALUE
else:
ctx.show_error("Invalid NewType %s" % (val, ))
return UNRESOLVED_VALUE
elif typing_inspect.is_typevar(val):
# TypeVar; not supported yet
return UNRESOLVED_VALUE
elif typing_inspect.is_classvar(val):
return UNRESOLVED_VALUE
elif is_instance_of_typing_name(
val, "_ForwardRef") or is_instance_of_typing_name(
val, "ForwardRef"):
# This has issues because the forward ref may be defined in a different file, in
# which case we don't know which names are valid in it.
with qcore.override(ctx, "suppress_undefined_name", True):
return UNRESOLVED_VALUE
elif val is Ellipsis:
# valid in Callable[..., ]
return UNRESOLVED_VALUE
elif is_instance_of_typing_name(val, "_TypeAlias"):
# typing.Pattern and Match, which are not normal generic types for some reason
return GenericValue(val.impl_type,
[_type_from_runtime(val.type_var, ctx)])
else:
origin = get_origin(val)
if origin is not None:
return TypedValue(origin)
ctx.show_error("Invalid type annotation %s" % (val, ))
return UNRESOLVED_VALUE
def field_for_schema(
typ: type,
default=marshmallow.missing,
metadata: Mapping[str, Any] = None,
base_schema: Optional[Type[marshmallow.Schema]] = None,
) -> marshmallow.fields.Field:
"""
Get a marshmallow Field corresponding to the given python type.
The metadata of the dataclass field is used as arguments to the marshmallow Field.
:param typ: The type for which a field should be generated
:param default: value to use for (de)serialization when the field is missing
:param metadata: Additional parameters to pass to the marshmallow field constructor
:param base_schema: marshmallow schema used as a base class when deriving dataclass schema
>>> int_field = field_for_schema(int, default=9, metadata=dict(required=True))
>>> int_field.__class__
<class 'marshmallow.fields.Integer'>
>>> int_field.default
9
>>> field_for_schema(str, metadata={"marshmallow_field": marshmallow.fields.Url()}).__class__
<class 'marshmallow.fields.Url'>
"""
metadata = {} if metadata is None else dict(metadata)
if default is not marshmallow.missing:
metadata.setdefault("default", default)
# 'missing' must not be set for required fields.
if not metadata.get("required"):
metadata.setdefault("missing", default)
else:
metadata.setdefault("required", True)
# If the field was already defined by the user
predefined_field = metadata.get("marshmallow_field")
if predefined_field:
return predefined_field
# Generic types specified without type arguments
if typ is list:
typ = List[Any]
elif typ is dict:
typ = Dict[Any, Any]
# Base types
field = _field_by_type(typ, base_schema)
if field:
return field(**metadata)
if typ is Any:
metadata.setdefault("allow_none", True)
return marshmallow.fields.Raw(**metadata)
if typing_inspect.is_literal_type(typ):
arguments = typing_inspect.get_args(typ)
return marshmallow.fields.Raw(
validate=(marshmallow.validate.Equal(arguments[0])
if len(arguments) == 1 else
marshmallow.validate.OneOf(arguments)),
**metadata,
)
# Generic types
origin = typing_inspect.get_origin(typ)
if origin:
arguments = typing_inspect.get_args(typ, True)
# Override base_schema.TYPE_MAPPING to change the class used for generic types below
type_mapping = base_schema.TYPE_MAPPING if base_schema else {}
if origin in (list, List):
child_type = field_for_schema(arguments[0],
base_schema=base_schema)
list_type = cast(
Type[marshmallow.fields.List],
type_mapping.get(List, marshmallow.fields.List),
)
return list_type(child_type, **metadata)
if origin in (tuple, Tuple):
children = tuple(
field_for_schema(arg, base_schema=base_schema)
for arg in arguments)
tuple_type = cast(
Type[marshmallow.fields.Tuple],
type_mapping.get( # type:ignore[call-overload]
Tuple, marshmallow.fields.Tuple),
)
return tuple_type(children, **metadata)
elif origin in (dict, Dict):
dict_type = type_mapping.get(Dict, marshmallow.fields.Dict)
return dict_type(
keys=field_for_schema(arguments[0], base_schema=base_schema),
values=field_for_schema(arguments[1], base_schema=base_schema),
**metadata,
)
elif typing_inspect.is_union_type(typ):
if typing_inspect.is_optional_type(typ):
metadata["allow_none"] = metadata.get("allow_none", True)
metadata["default"] = metadata.get("default", None)
metadata["missing"] = metadata.get("missing", None)
metadata["required"] = False
subtypes = [t for t in arguments
if t is not NoneType] # type: ignore
if len(subtypes) == 1:
return field_for_schema(subtypes[0],
metadata=metadata,
base_schema=base_schema)
from . import union_field
return union_field.Union(
[(
subtyp,
field_for_schema(
subtyp, metadata=metadata, base_schema=base_schema),
) for subtyp in subtypes],
**metadata,
)
# typing.NewType returns a function with a __supertype__ attribute
newtype_supertype = getattr(typ, "__supertype__", None)
if newtype_supertype and inspect.isfunction(typ):
return _field_by_supertype(
typ=typ,
default=default,
newtype_supertype=newtype_supertype,
metadata=metadata,
base_schema=base_schema,
)
# enumerations
if isinstance(typ, EnumMeta):
import marshmallow_enum
return marshmallow_enum.EnumField(typ, **metadata)
# Nested marshmallow dataclass
nested_schema = getattr(typ, "Schema", None)
# Nested dataclasses
forward_reference = getattr(typ, "__forward_arg__", None)
nested = (nested_schema or forward_reference
or _internal_class_schema(typ, base_schema))
return marshmallow.fields.Nested(nested, **metadata)
def make_documentation(cls):
"""
Returns the reStructuredText documentation for the :class:`~.ConfigVar`.
"""
docstring = cls.__doc__ or ''
docstring = (indent(dedent(docstring), tab))
if not docstring.startswith('\n'):
docstring = '\n' + docstring
buf = StringList()
buf.indent_type = " "
buf.blankline(ensure_single=True)
buf.append(f".. conf:: {cls.__name__}")
buf.append(docstring)
buf.blankline()
buf.indent_size += 1
buf.append(f"**Required**: {'yes' if cls.required else 'no'}")
buf.blankline()
buf.blankline()
if not cls.required:
if cls.default == []:
buf.append("**Default**: [ ]")
elif cls.default == {}:
buf.append("**Default**: { }")
elif isinstance(cls.default, Callable): # type: ignore
buf.append(
f"**Default**: The value of :conf:`{cls.default.__name__}`"
)
elif isinstance(cls.default, bool):
buf.append(f"**Default**: :py:obj:`{cls.default}`")
elif isinstance(cls.default, str):
if cls.default == '':
buf.append("**Default**: <blank>")
else:
buf.append(f"**Default**: ``{cls.default}``")
else:
buf.append(f"**Default**: {cls.default}")
buf.blankline()
buf.blankline()
buf.append(f"**Type**: {get_yaml_type(cls.dtype)}")
if is_literal_type(cls.dtype):
valid_values = ", ".join(f"``{x}``" for x in cls.dtype.__args__)
buf.blankline()
buf.blankline()
buf.append(f"**Allowed values**: {valid_values}")
elif hasattr(cls.dtype, "__args__") and is_literal_type(
cls.dtype.__args__[0]):
valid_values = ", ".join(f"``{x}``"
for x in cls.dtype.__args__[0].__args__)
buf.blankline()
buf.blankline()
buf.append(f"**Allowed values**: {valid_values}")
buf.indent_size -= 1
return str(buf)
def _add_argument(self, *name_or_flags, **kwargs) -> None:
"""Adds an argument to self (i.e. the super class ArgumentParser).
Sets the following attributes of kwargs when not explicitly provided:
- type: Set to the type annotation of the argument.
- default: Set to the default value of the argument (if provided).
- required: True if a default value of the argument is not provided, False otherwise.
- action: Set to "store_true" if the argument is a required bool or a bool with default value False.
Set to "store_false" if the argument is a bool with default value True.
- nargs: Set to "*" if the type annotation is List[str], List[int], or List[float].
- help: Set to the argument documentation from the class docstring.
:param name_or_flags: Either a name or a list of option strings, e.g. foo or -f, --foo.
:param kwargs: Keyword arguments.
"""
# Get variable name
variable = get_dest(*name_or_flags, **kwargs)
# Get default if not specified
if hasattr(self, variable):
kwargs['default'] = kwargs.get('default', getattr(self, variable))
# Set required if option arg
if is_option_arg(*name_or_flags) and variable != 'help':
kwargs['required'] = kwargs.get('required',
not hasattr(self, variable))
# Set help if necessary
if 'help' not in kwargs:
kwargs['help'] = '('
# Type
if variable in self._annotations:
kwargs['help'] += type_to_str(
self._annotations[variable]) + ', '
# Required/default
if kwargs.get('required', False):
kwargs['help'] += 'required'
else:
kwargs['help'] += f'default={kwargs.get("default", None)}'
kwargs['help'] += ')'
# Description
if variable in self.class_variables:
kwargs[
'help'] += ' ' + self.class_variables[variable]['comment']
# Set other kwargs where not provided
if variable in self._annotations:
# Get type annotation
var_type = self._annotations[variable]
# If type is not explicitly provided, set it if it's one of our supported default types
if 'type' not in kwargs:
# First check whether it is a literal type or a boxed literal type
if is_literal_type(var_type):
var_type, kwargs['choices'] = get_literals(
var_type, variable)
elif (get_origin(var_type) in (List, list, Set, set)
and len(get_args(var_type)) > 0
and is_literal_type(get_args(var_type)[0])):
var_type, kwargs['choices'] = get_literals(
get_args(var_type)[0], variable)
kwargs['nargs'] = kwargs.get('nargs', '*')
# Handle Tuple type (with type args) by extracting types of Tuple elements and enforcing them
elif get_origin(var_type) in (Tuple, tuple) and len(
get_args(var_type)) > 0:
loop = False
types = get_args(var_type)
# Don't allow Tuple[()]
if len(types) == 1 and types[0] == tuple():
raise ValueError(
'Empty Tuples (i.e. Tuple[()]) are not a valid Tap type '
'because they have no arguments.')
# Handle Tuple[type, ...]
if len(types) == 2 and types[1] == Ellipsis:
types = types[0:1]
loop = True
kwargs['nargs'] = '*'
else:
kwargs['nargs'] = len(types)
var_type = TupleTypeEnforcer(types=types, loop=loop)
# To identify an Optional type, check if it's a union of a None and something else
elif (is_union_type(var_type) and len(get_args(var_type)) == 2
and isinstance(None,
get_args(var_type)[1])
and is_literal_type(get_args(var_type)[0])):
var_type, kwargs['choices'] = get_literals(
get_args(var_type)[0], variable)
elif var_type not in SUPPORTED_DEFAULT_TYPES:
raise ValueError(
f'Variable "{variable}" has type "{var_type}" which is not supported by default.\n'
f'Please explicitly add the argument to the parser by writing:\n\n'
f'def add_arguments(self) -> None:\n'
f' self.add_argument("--{variable}", type=func, {"required=True" if kwargs["required"] else f"default={getattr(self, variable)}"})\n\n'
f'where "func" maps from str to {var_type}.')
if var_type in SUPPORTED_DEFAULT_BOXED_TYPES:
# If List or Set type, set nargs
if var_type in SUPPORTED_DEFAULT_COLLECTION_TYPES:
kwargs['nargs'] = kwargs.get('nargs', '*')
# Extract boxed type for Optional, List, Set
arg_types = get_args(var_type)
# Set defaults type to str for Type and Type[()]
if len(arg_types) == 0 or arg_types[0] == EMPTY_TYPE:
var_type = str
else:
var_type = arg_types[0]
# Handle the cases of Optional[bool], List[bool], Set[bool]
if var_type == bool:
var_type = boolean_type
# If bool then set action, otherwise set type
if var_type == bool:
if self._explicit_bool:
kwargs['type'] = boolean_type
kwargs['choices'] = [
True, False
] # this makes the help message more helpful
else:
kwargs['action'] = kwargs.get(
'action',
f'store_{"true" if kwargs["required"] or not kwargs["default"] else "false"}'
)
else:
kwargs['type'] = var_type
super(Tap, self).add_argument(*name_or_flags, **kwargs)
def _analyze_http_code(
operation: oas.OASOperation,
rt_http_code: t.Optional[t.Type[t.Any]],
) -> t.Set[exceptions.Error]:
if rt_http_code is None:
# if there's no http_code in return Response
# this is permitted only if there's single response defined in
# OAS responses. User needs to set it otherwise how can we tell if
# everything is correct
if len(operation.responses) != 1:
logger.opt(lazy=True).error(
'Operation {id} handler skips return.http_code but it is impossible '
' with {count_of_ops} responses due to ambiguity.',
id=lambda: operation.id,
count_of_ops=lambda: len(operation.responses),
)
return {
exceptions.Error(
param_name='return.http_code',
reason='missing',
)
}
return set()
elif ti.is_literal_type(rt_http_code):
# this is acceptable. Literals hold particular values inside of them
# if user wants to have it that way -> go ahead.
# axion however will not validate a specific values in Literal.
# this is by design and due to:
# - error responses that axion implements via exceptions
literal_types = types.literal_types(rt_http_code)
if not all(
issubclass(lmt, model.HTTP_CODE_TYPE)
for lmt in literal_types):
return {
exceptions.Error(
param_name='return.http_code',
reason=exceptions.CustomReason(
f'expected {repr(te.Literal)}[int]'),
),
}
return set()
elif ti.is_new_type(rt_http_code):
# not quite sure why user would like to alias that
# but it is not a problem for axion as long `NewType` embedded type
# is fine
return _analyze_http_code(operation, rt_http_code.__supertype__)
elif issubclass(rt_http_code, bool):
# yeah, Python rocks -> bool is subclass of an int
# not quite sure wh that happens, perhaps someone sometime
# will answer that question
return {
exceptions.Error(
param_name='return.http_code',
reason=exceptions.IncorrectTypeReason(
expected=[model.HTTP_CODE_TYPE],
actual=bool,
),
),
}
else:
try:
assert issubclass(rt_http_code, model.HTTP_CODE_TYPE)
return set()
except (AssertionError, TypeError):
...
return {
exceptions.Error(
param_name='return.http_code',
reason=exceptions.IncorrectTypeReason(
actual=rt_http_code,
expected=[
type(None),
model.HTTP_CODE_TYPE,
t.NewType('HttpCode', model.HTTP_CODE_TYPE),
te.Literal,
],
),
),
}
