def visit_type_var(self, t: TypeVarType) -> Type:
if self.check_recursion(t):
return AnyType(TypeOfAny.from_error)
if t.upper_bound:
t.upper_bound = t.upper_bound.accept(self)
if t.values:
t.values = [v.accept(self) for v in t.values]
return t
def build_namedtuple_typeinfo(self, name: str, items: List[str], types: List[Type],
default_items: Mapping[str, Expression]) -> TypeInfo:
strtype = self.api.named_type('__builtins__.str')
implicit_any = AnyType(TypeOfAny.special_form)
basetuple_type = self.api.named_type('__builtins__.tuple', [implicit_any])
dictype = (self.api.named_type_or_none('builtins.dict', [strtype, implicit_any])
or self.api.named_type('__builtins__.object'))
# Actual signature should return OrderedDict[str, Union[types]]
ordereddictype = (self.api.named_type_or_none('builtins.dict', [strtype, implicit_any])
or self.api.named_type('__builtins__.object'))
fallback = self.api.named_type('__builtins__.tuple', [implicit_any])
# Note: actual signature should accept an invariant version of Iterable[UnionType[types]].
# but it can't be expressed. 'new' and 'len' should be callable types.
iterable_type = self.api.named_type_or_none('typing.Iterable', [implicit_any])
function_type = self.api.named_type('__builtins__.function')
info = self.api.basic_new_typeinfo(name, fallback)
info.is_named_tuple = True
info.tuple_type = TupleType(types, fallback)
def patch() -> None:
# Calculate the correct value type for the fallback tuple.
assert info.tuple_type, "TupleType type deleted before calling the patch"
fallback.args[0] = join.join_type_list(list(info.tuple_type.items))
# We can't calculate the complete fallback type until after semantic
# analysis, since otherwise MROs might be incomplete. Postpone a callback
# function that patches the fallback.
self.api.schedule_patch(PRIORITY_FALLBACKS, patch)
def add_field(var: Var, is_initialized_in_class: bool = False,
is_property: bool = False) -> None:
var.info = info
var.is_initialized_in_class = is_initialized_in_class
var.is_property = is_property
var._fullname = '%s.%s' % (info.fullname(), var.name())
info.names[var.name()] = SymbolTableNode(MDEF, var)
vars = [Var(item, typ) for item, typ in zip(items, types)]
for var in vars:
add_field(var, is_property=True)
tuple_of_strings = TupleType([strtype for _ in items], basetuple_type)
add_field(Var('_fields', tuple_of_strings), is_initialized_in_class=True)
add_field(Var('_field_types', dictype), is_initialized_in_class=True)
add_field(Var('_field_defaults', dictype), is_initialized_in_class=True)
add_field(Var('_source', strtype), is_initialized_in_class=True)
add_field(Var('__annotations__', ordereddictype), is_initialized_in_class=True)
add_field(Var('__doc__', strtype), is_initialized_in_class=True)
tvd = TypeVarDef('NT', 'NT', -1, [], info.tuple_type)
selftype = TypeVarType(tvd)
def add_method(funcname: str,
ret: Type,
args: List[Argument],
name: Optional[str] = None,
is_classmethod: bool = False,
is_new: bool = False,
) -> None:
if is_classmethod or is_new:
first = [Argument(Var('cls'), TypeType.make_normalized(selftype), None, ARG_POS)]
else:
first = [Argument(Var('self'), selftype, None, ARG_POS)]
args = first + args
types = [arg.type_annotation for arg in args]
items = [arg.variable.name() for arg in args]
arg_kinds = [arg.kind for arg in args]
assert None not in types
signature = CallableType(cast(List[Type], types), arg_kinds, items, ret,
function_type)
signature.variables = [tvd]
func = FuncDef(funcname, args, Block([]))
func.info = info
func.is_class = is_classmethod
func.type = set_callable_name(signature, func)
func._fullname = info.fullname() + '.' + funcname
if is_classmethod:
v = Var(funcname, func.type)
v.is_classmethod = True
v.info = info
v._fullname = func._fullname
dec = Decorator(func, [NameExpr('classmethod')], v)
info.names[funcname] = SymbolTableNode(MDEF, dec)
else:
info.names[funcname] = SymbolTableNode(MDEF, func)
add_method('_replace', ret=selftype,
args=[Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT) for var in vars])
def make_init_arg(var: Var) -> Argument:
default = default_items.get(var.name(), None)
kind = ARG_POS if default is None else ARG_OPT
return Argument(var, var.type, default, kind)
add_method('__new__', ret=selftype, name=info.name(),
args=[make_init_arg(var) for var in vars],
is_new=True)
add_method('_asdict', args=[], ret=ordereddictype)
special_form_any = AnyType(TypeOfAny.special_form)
add_method('_make', ret=selftype, is_classmethod=True,
args=[Argument(Var('iterable', iterable_type), iterable_type, None, ARG_POS),
Argument(Var('new'), special_form_any, EllipsisExpr(), ARG_NAMED_OPT),
Argument(Var('len'), special_form_any, EllipsisExpr(), ARG_NAMED_OPT)])
return info
def make_type_var(name: str, id: int, values: List[Type], upper_bound: Type,
variance: int) -> TypeVarType:
return TypeVarType(name, name, id, values, upper_bound, variance)
def transform(self) -> None:
"""Apply all the necessary transformations to the underlying
dataclass so as to ensure it is fully type checked according
to the rules in PEP 557.
"""
ctx = self._ctx
info = self._ctx.cls.info
attributes = self.collect_attributes()
if attributes is None:
# Some definitions are not ready, defer() should be already called.
return
for attr in attributes:
if info[attr.name].type is None:
ctx.api.defer()
return
decorator_arguments = {
'init': _get_decorator_bool_argument(self._ctx, 'init', True),
'eq': _get_decorator_bool_argument(self._ctx, 'eq', True),
'order': _get_decorator_bool_argument(self._ctx, 'order', False),
'frozen': _get_decorator_bool_argument(self._ctx, 'frozen', False),
}
# If there are no attributes, it may be that the new semantic analyzer has not
# processed them yet. In order to work around this, we can simply skip generating
# __init__ if there are no attributes, because if the user truly did not define any,
# then the object default __init__ with an empty signature will be present anyway.
if (decorator_arguments['init']
and ('__init__' not in info.names
or info.names['__init__'].plugin_generated)
and attributes):
add_method(
ctx,
'__init__',
args=[
attr.to_argument(info) for attr in attributes
if attr.is_in_init
],
return_type=NoneType(),
)
if (decorator_arguments['eq'] and info.get('__eq__') is None
or decorator_arguments['order']):
# Type variable for self types in generated methods.
obj_type = ctx.api.named_type('__builtins__.object')
self_tvar_expr = TypeVarExpr(
SELF_TVAR_NAME,
info.fullname() + '.' + SELF_TVAR_NAME, [], obj_type)
info.names[SELF_TVAR_NAME] = SymbolTableNode(MDEF, self_tvar_expr)
# Add an eq method, but only if the class doesn't already have one.
if decorator_arguments['eq'] and info.get('__eq__') is None:
for method_name in ['__eq__', '__ne__']:
# The TVar is used to enforce that "other" must have
# the same type as self (covariant). Note the
# "self_type" parameter to add_method.
obj_type = ctx.api.named_type('__builtins__.object')
cmp_tvar_def = TypeVarDef(
SELF_TVAR_NAME,
info.fullname() + '.' + SELF_TVAR_NAME, -1, [], obj_type)
cmp_other_type = TypeVarType(cmp_tvar_def)
cmp_return_type = ctx.api.named_type('__builtins__.bool')
add_method(
ctx,
method_name,
args=[
Argument(Var('other', cmp_other_type), cmp_other_type,
None, ARG_POS)
],
return_type=cmp_return_type,
self_type=cmp_other_type,
tvar_def=cmp_tvar_def,
)
# Add <, >, <=, >=, but only if the class has an eq method.
if decorator_arguments['order']:
if not decorator_arguments['eq']:
ctx.api.fail('eq must be True if order is True', ctx.cls)
for method_name in ['__lt__', '__gt__', '__le__', '__ge__']:
# Like for __eq__ and __ne__, we want "other" to match
# the self type.
obj_type = ctx.api.named_type('__builtins__.object')
order_tvar_def = TypeVarDef(
SELF_TVAR_NAME,
info.fullname() + '.' + SELF_TVAR_NAME, -1, [], obj_type)
order_other_type = TypeVarType(order_tvar_def)
order_return_type = ctx.api.named_type('__builtins__.bool')
order_args = [
Argument(Var('other', order_other_type), order_other_type,
None, ARG_POS)
]
existing_method = info.get(method_name)
if existing_method is not None and not existing_method.plugin_generated:
assert existing_method.node
ctx.api.fail(
'You may not have a custom %s method when order=True' %
method_name,
existing_method.node,
)
add_method(
ctx,
method_name,
args=order_args,
return_type=order_return_type,
self_type=order_other_type,
tvar_def=order_tvar_def,
)
if decorator_arguments['frozen']:
self._freeze(attributes)
self.reset_init_only_vars(info, attributes)
info.metadata['dataclass'] = {
'attributes':
OrderedDict((attr.name, attr.serialize()) for attr in attributes),
'frozen':
decorator_arguments['frozen'],
}
def visit_unbound_type(self, t: UnboundType) -> Type:
if t.optional:
t.optional = False
# We don't need to worry about double-wrapping Optionals or
# wrapping Anys: Union simplification will take care of that.
return UnionType.make_simplified_union(
[self.visit_unbound_type(t),
NoneTyp()])
sym = self.lookup(t.name, t)
if sym is not None:
if sym.node is None:
# UNBOUND_IMPORTED can happen if an unknown name was imported.
if sym.kind != UNBOUND_IMPORTED:
self.fail(
'Internal error (node is None, kind={})'.format(
sym.kind), t)
return AnyType()
fullname = sym.node.fullname()
if sym.kind == BOUND_TVAR:
if len(t.args) > 0:
self.fail(
'Type variable "{}" used with arguments'.format(
t.name), t)
assert sym.tvar_def is not None
return TypeVarType(sym.tvar_def, t.line)
elif fullname == 'builtins.None':
if experiments.STRICT_OPTIONAL:
if t.is_ret_type:
return Void()
else:
return NoneTyp()
else:
return Void()
elif fullname == 'typing.Any':
return AnyType()
elif fullname == 'typing.Tuple':
if len(t.args) == 2 and isinstance(t.args[1], EllipsisType):
# Tuple[T, ...] (uniform, variable-length tuple)
node = self.lookup_fqn_func('builtins.tuple')
tuple_info = cast(TypeInfo, node.node)
return Instance(tuple_info, [t.args[0].accept(self)],
t.line)
return self.tuple_type(self.anal_array(t.args))
elif fullname == 'typing.Union':
items = self.anal_array(t.args)
items = [item for item in items if not isinstance(item, Void)]
return UnionType.make_union(items)
elif fullname == 'typing.Optional':
if len(t.args) != 1:
self.fail(
'Optional[...] must have exactly one type argument', t)
return AnyType()
items = self.anal_array(t.args)
if experiments.STRICT_OPTIONAL:
return UnionType.make_simplified_union(
[items[0], NoneTyp()])
else:
# Without strict Optional checking Optional[t] is just an alias for t.
return items[0]
elif fullname == 'typing.Callable':
return self.analyze_callable_type(t)
elif fullname == 'typing.Type':
if len(t.args) == 0:
return TypeType(AnyType(), line=t.line)
if len(t.args) != 1:
self.fail('Type[...] must have exactly one type argument',
t)
items = self.anal_array(t.args)
item = items[0]
return TypeType(item, line=t.line)
elif sym.kind == TYPE_ALIAS:
# TODO: Generic type aliases.
return sym.type_override
elif not isinstance(sym.node, TypeInfo):
name = sym.fullname
if name is None:
name = sym.node.name()
if isinstance(sym.node, Var) and isinstance(
sym.node.type, AnyType):
# Something with an Any type -- make it an alias for Any in a type
# context. This is slightly problematic as it allows using the type 'Any'
# as a base class -- however, this will fail soon at runtime so the problem
# is pretty minor.
return AnyType()
self.fail('Invalid type "{}"'.format(name), t)
return t
info = sym.node # type: TypeInfo
if len(t.args) > 0 and info.fullname() == 'builtins.tuple':
return TupleType(self.anal_array(t.args),
Instance(info, [AnyType()], t.line), t.line)
else:
# Analyze arguments and construct Instance type. The
# number of type arguments and their values are
# checked only later, since we do not always know the
# valid count at this point. Thus we may construct an
# Instance with an invalid number of type arguments.
instance = Instance(info, self.anal_array(t.args), t.line)
if info.tuple_type is None:
return instance
else:
# The class has a Tuple[...] base class so it will be
# represented as a tuple type.
if t.args:
self.fail('Generic tuple types not supported', t)
return AnyType()
return TupleType(self.anal_array(info.tuple_type.items),
fallback=instance,
line=t.line)
else:
return AnyType()
def visit_unbound_type(self, t: UnboundType) -> Type:
if t.optional:
t.optional = False
# We don't need to worry about double-wrapping Optionals or
# wrapping Anys: Union simplification will take care of that.
return make_optional_type(self.visit_unbound_type(t))
sym = self.lookup(t.name, t, suppress_errors=self.third_pass)
if '.' in t.name:
# Handle indirect references to imported names.
#
# TODO: Do this for module-local references as well and remove ImportedName
# type check below.
sym = self.api.dereference_module_cross_ref(sym)
if sym is not None:
if isinstance(sym.node, ImportedName):
# Forward reference to an imported name that hasn't been processed yet.
# To maintain backward compatibility, these get translated to Any.
#
# TODO: Remove this special case.
return AnyType(TypeOfAny.implementation_artifact)
if sym.fullname in type_aliases:
# Resolve forward reference to type alias like 'typing.List'.
# TODO: Unify how type aliases are handled; currently we resolve them in two
# places (the other is in the semantic analyzer pass 2).
resolved = type_aliases[sym.fullname]
new = self.api.lookup_qualified(resolved, t)
if new:
sym = new.copy()
sym.normalized = True
if sym.node is None:
# UNBOUND_IMPORTED can happen if an unknown name was imported.
if sym.kind != UNBOUND_IMPORTED:
self.fail(
'Internal error (node is None, kind={})'.format(
sym.kind), t)
return AnyType(TypeOfAny.special_form)
fullname = sym.node.fullname()
hook = self.plugin.get_type_analyze_hook(fullname)
if hook:
return hook(AnalyzeTypeContext(t, t, self))
if (fullname in nongen_builtins and t.args and not sym.normalized
and not self.allow_unnormalized):
self.fail(no_subscript_builtin_alias(fullname), t)
if self.tvar_scope:
tvar_def = self.tvar_scope.get_binding(sym)
else:
tvar_def = None
if self.warn_bound_tvar and sym.kind == TVAR and tvar_def is not None:
self.fail(
'Can\'t use bound type variable "{}"'
' to define generic alias'.format(t.name), t)
return AnyType(TypeOfAny.from_error)
elif sym.kind == TVAR and tvar_def is not None:
if len(t.args) > 0:
self.fail(
'Type variable "{}" used with arguments'.format(
t.name), t)
return TypeVarType(tvar_def, t.line)
elif fullname == 'builtins.None':
return NoneTyp()
elif fullname == 'typing.Any' or fullname == 'builtins.Any':
return AnyType(TypeOfAny.explicit)
elif fullname == 'typing.Tuple':
if len(t.args) == 0 and not t.empty_tuple_index:
# Bare 'Tuple' is same as 'tuple'
if self.options.disallow_any_generics and not self.is_typeshed_stub:
self.fail(messages.BARE_GENERIC, t)
typ = self.named_type('builtins.tuple',
line=t.line,
column=t.column)
typ.from_generic_builtin = True
return typ
if len(t.args) == 2 and isinstance(t.args[1], EllipsisType):
# Tuple[T, ...] (uniform, variable-length tuple)
instance = self.named_type('builtins.tuple',
[self.anal_type(t.args[0])])
instance.line = t.line
return instance
return self.tuple_type(self.anal_array(t.args))
elif fullname == 'typing.Union':
items = self.anal_array(t.args)
return UnionType.make_union(items)
elif fullname == 'typing.Optional':
if len(t.args) != 1:
self.fail(
'Optional[...] must have exactly one type argument', t)
return AnyType(TypeOfAny.from_error)
item = self.anal_type(t.args[0])
return make_optional_type(item)
elif fullname == 'typing.Callable':
return self.analyze_callable_type(t)
elif fullname == 'typing.Type':
if len(t.args) == 0:
any_type = AnyType(TypeOfAny.from_omitted_generics,
line=t.line,
column=t.column)
return TypeType(any_type, line=t.line, column=t.column)
if len(t.args) != 1:
self.fail('Type[...] must have exactly one type argument',
t)
item = self.anal_type(t.args[0])
return TypeType.make_normalized(item, line=t.line)
elif fullname == 'typing.ClassVar':
if self.nesting_level > 0:
self.fail(
'Invalid type: ClassVar nested inside other type', t)
if len(t.args) == 0:
return AnyType(TypeOfAny.from_omitted_generics,
line=t.line,
column=t.column)
if len(t.args) != 1:
self.fail(
'ClassVar[...] must have at most one type argument', t)
return AnyType(TypeOfAny.from_error)
item = self.anal_type(t.args[0])
if isinstance(item, TypeVarType) or get_type_vars(item):
self.fail('Invalid type: ClassVar cannot be generic', t)
return AnyType(TypeOfAny.from_error)
return item
elif fullname in ('mypy_extensions.NoReturn', 'typing.NoReturn'):
return UninhabitedType(is_noreturn=True)
elif sym.kind == TYPE_ALIAS:
if sym.alias_name is not None:
self.aliases_used.add(sym.alias_name)
override = sym.type_override
all_vars = sym.alias_tvars
assert override is not None
an_args = self.anal_array(t.args)
if all_vars is not None:
exp_len = len(all_vars)
else:
exp_len = 0
act_len = len(an_args)
if exp_len > 0 and act_len == 0:
# Interpret bare Alias same as normal generic, i.e., Alias[Any, Any, ...]
assert all_vars is not None
return set_any_tvars(override, all_vars, t.line, t.column)
if exp_len == 0 and act_len == 0:
return override
if act_len != exp_len:
self.fail(
'Bad number of arguments for type alias, expected: %s, given: %s'
% (exp_len, act_len), t)
return set_any_tvars(override,
all_vars or [],
t.line,
t.column,
implicit=False)
assert all_vars is not None
return replace_alias_tvars(override, all_vars, an_args, t.line,
t.column)
elif not isinstance(sym.node, TypeInfo):
name = sym.fullname
if name is None:
name = sym.node.name()
if isinstance(sym.node, Var) and isinstance(
sym.node.type, AnyType):
# Something with an Any type -- make it an alias for Any in a type
# context. This is slightly problematic as it allows using the type 'Any'
# as a base class -- however, this will fail soon at runtime so the problem
# is pretty minor.
return AnyType(
TypeOfAny.from_unimported_type,
missing_import_name=sym.node.type.missing_import_name)
# Allow unbound type variables when defining an alias
if not (self.aliasing and sym.kind == TVAR and
(not self.tvar_scope
or self.tvar_scope.get_binding(sym) is None)):
if (not self.third_pass and not self.in_dynamic_func
and not (isinstance(sym.node, (FuncDef, Decorator))
or isinstance(sym.node, Var)
and sym.node.is_ready)
and not (sym.kind == TVAR and tvar_def is None)):
if t.args and not self.global_scope:
self.fail(
'Unsupported forward reference to "{}"'.format(
t.name), t)
return AnyType(TypeOfAny.from_error)
return ForwardRef(t)
self.fail('Invalid type "{}"'.format(name), t)
if self.third_pass and sym.kind == TVAR:
self.note_func(
"Forward references to type variables are prohibited",
t)
return t
info = sym.node # type: TypeInfo
if sym.is_aliasing:
if sym.alias_name is not None:
self.aliases_used.add(sym.alias_name)
if len(t.args) > 0 and info.fullname() == 'builtins.tuple':
fallback = Instance(info, [AnyType(TypeOfAny.special_form)],
t.line)
return TupleType(self.anal_array(t.args), fallback, t.line)
else:
# Analyze arguments and construct Instance type. The
# number of type arguments and their values are
# checked only later, since we do not always know the
# valid count at this point. Thus we may construct an
# Instance with an invalid number of type arguments.
instance = Instance(info, self.anal_array(t.args), t.line,
t.column)
instance.from_generic_builtin = sym.normalized
tup = info.tuple_type
if tup is not None:
# The class has a Tuple[...] base class so it will be
# represented as a tuple type.
if t.args:
self.fail('Generic tuple types not supported', t)
return AnyType(TypeOfAny.from_error)
return tup.copy_modified(items=self.anal_array(tup.items),
fallback=instance)
td = info.typeddict_type
if td is not None:
# The class has a TypedDict[...] base class so it will be
# represented as a typeddict type.
if t.args:
self.fail('Generic TypedDict types not supported', t)
return AnyType(TypeOfAny.from_error)
# Create a named TypedDictType
return td.copy_modified(item_types=self.anal_array(
list(td.items.values())),
fallback=instance)
return instance
else:
if self.third_pass:
self.fail('Invalid type "{}"'.format(t.name), t)
return AnyType(TypeOfAny.from_error)
return AnyType(TypeOfAny.special_form)
def build_namedtuple_typeinfo(self, name: str, items: List[str],
types: List[Type],
default_items: Mapping[str, Expression],
line: int) -> TypeInfo:
strtype = self.api.named_type('builtins.str')
implicit_any = AnyType(TypeOfAny.special_form)
basetuple_type = self.api.named_type('builtins.tuple', [implicit_any])
dictype = (self.api.named_type_or_none('builtins.dict',
[strtype, implicit_any])
or self.api.named_type('builtins.object'))
# Actual signature should return OrderedDict[str, Union[types]]
ordereddictype = (self.api.named_type_or_none('builtins.dict',
[strtype, implicit_any])
or self.api.named_type('builtins.object'))
fallback = self.api.named_type('builtins.tuple', [implicit_any])
# Note: actual signature should accept an invariant version of Iterable[UnionType[types]].
# but it can't be expressed. 'new' and 'len' should be callable types.
iterable_type = self.api.named_type_or_none('typing.Iterable',
[implicit_any])
function_type = self.api.named_type('builtins.function')
info = self.api.basic_new_typeinfo(name, fallback, line)
info.is_named_tuple = True
tuple_base = TupleType(types, fallback)
info.tuple_type = tuple_base
info.line = line
# For use by mypyc.
info.metadata['namedtuple'] = {'fields': items.copy()}
# We can't calculate the complete fallback type until after semantic
# analysis, since otherwise base classes might be incomplete. Postpone a
# callback function that patches the fallback.
self.api.schedule_patch(PRIORITY_FALLBACKS,
lambda: calculate_tuple_fallback(tuple_base))
def add_field(var: Var,
is_initialized_in_class: bool = False,
is_property: bool = False) -> None:
var.info = info
var.is_initialized_in_class = is_initialized_in_class
var.is_property = is_property
var._fullname = '%s.%s' % (info.fullname, var.name)
info.names[var.name] = SymbolTableNode(MDEF, var)
fields = [Var(item, typ) for item, typ in zip(items, types)]
for var in fields:
add_field(var, is_property=True)
# We can't share Vars between fields and method arguments, since they
# have different full names (the latter are normally used as local variables
# in functions, so their full names are set to short names when generated methods
# are analyzed).
vars = [Var(item, typ) for item, typ in zip(items, types)]
tuple_of_strings = TupleType([strtype for _ in items], basetuple_type)
add_field(Var('_fields', tuple_of_strings),
is_initialized_in_class=True)
add_field(Var('_field_types', dictype), is_initialized_in_class=True)
add_field(Var('_field_defaults', dictype),
is_initialized_in_class=True)
add_field(Var('_source', strtype), is_initialized_in_class=True)
add_field(Var('__annotations__', ordereddictype),
is_initialized_in_class=True)
add_field(Var('__doc__', strtype), is_initialized_in_class=True)
tvd = TypeVarType(SELF_TVAR_NAME, info.fullname + '.' + SELF_TVAR_NAME,
-1, [], info.tuple_type)
selftype = tvd
def add_method(
funcname: str,
ret: Type,
args: List[Argument],
is_classmethod: bool = False,
is_new: bool = False,
) -> None:
if is_classmethod or is_new:
first = [
Argument(Var('_cls'), TypeType.make_normalized(selftype),
None, ARG_POS)
]
else:
first = [Argument(Var('_self'), selftype, None, ARG_POS)]
args = first + args
types = [arg.type_annotation for arg in args]
items = [arg.variable.name for arg in args]
arg_kinds = [arg.kind for arg in args]
assert None not in types
signature = CallableType(cast(List[Type], types), arg_kinds, items,
ret, function_type)
signature.variables = [tvd]
func = FuncDef(funcname, args, Block([]))
func.info = info
func.is_class = is_classmethod
func.type = set_callable_name(signature, func)
func._fullname = info.fullname + '.' + funcname
func.line = line
if is_classmethod:
v = Var(funcname, func.type)
v.is_classmethod = True
v.info = info
v._fullname = func._fullname
func.is_decorated = True
dec = Decorator(func, [NameExpr('classmethod')], v)
dec.line = line
sym = SymbolTableNode(MDEF, dec)
else:
sym = SymbolTableNode(MDEF, func)
sym.plugin_generated = True
info.names[funcname] = sym
add_method('_replace',
ret=selftype,
args=[
Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT)
for var in vars
])
def make_init_arg(var: Var) -> Argument:
default = default_items.get(var.name, None)
kind = ARG_POS if default is None else ARG_OPT
return Argument(var, var.type, default, kind)
add_method('__new__',
ret=selftype,
args=[make_init_arg(var) for var in vars],
is_new=True)
add_method('_asdict', args=[], ret=ordereddictype)
special_form_any = AnyType(TypeOfAny.special_form)
add_method('_make',
ret=selftype,
is_classmethod=True,
args=[
Argument(Var('iterable', iterable_type), iterable_type,
None, ARG_POS),
Argument(Var('new'), special_form_any, EllipsisExpr(),
ARG_NAMED_OPT),
Argument(Var('len'), special_form_any, EllipsisExpr(),
ARG_NAMED_OPT)
])
self_tvar_expr = TypeVarExpr(SELF_TVAR_NAME,
info.fullname + '.' + SELF_TVAR_NAME, [],
info.tuple_type)
info.names[SELF_TVAR_NAME] = SymbolTableNode(MDEF, self_tvar_expr)
return info
def visit_unbound_type(self, t: UnboundType) -> Type:
if t.optional:
t.optional = False
# We don't need to worry about double-wrapping Optionals or
# wrapping Anys: Union simplification will take care of that.
return make_optional_type(self.visit_unbound_type(t))
sym = self.lookup(t.name, t)
if sym is not None:
if sym.node is None:
# UNBOUND_IMPORTED can happen if an unknown name was imported.
if sym.kind != UNBOUND_IMPORTED:
self.fail('Internal error (node is None, kind={})'.format(sym.kind), t)
return AnyType()
fullname = sym.node.fullname()
if (fullname in nongen_builtins and t.args and
not sym.normalized and not self.allow_unnormalized):
self.fail(no_subscript_builtin_alias(fullname), t)
tvar_def = self.tvar_scope.get_binding(sym)
if sym.kind == TVAR and tvar_def is not None:
if len(t.args) > 0:
self.fail('Type variable "{}" used with arguments'.format(
t.name), t)
return TypeVarType(tvar_def, t.line)
elif fullname == 'builtins.None':
return NoneTyp()
elif fullname == 'typing.Any' or fullname == 'builtins.Any':
return AnyType()
elif fullname == 'typing.Tuple':
if len(t.args) == 0 and not t.empty_tuple_index:
# Bare 'Tuple' is same as 'tuple'
return self.builtin_type('builtins.tuple')
if len(t.args) == 2 and isinstance(t.args[1], EllipsisType):
# Tuple[T, ...] (uniform, variable-length tuple)
instance = self.builtin_type('builtins.tuple', [self.anal_type(t.args[0])])
instance.line = t.line
return instance
return self.tuple_type(self.anal_array(t.args))
elif fullname == 'typing.Union':
items = self.anal_array(t.args)
if not experiments.STRICT_OPTIONAL:
items = [item for item in items if not isinstance(item, NoneTyp)]
return UnionType.make_union(items)
elif fullname == 'typing.Optional':
if len(t.args) != 1:
self.fail('Optional[...] must have exactly one type argument', t)
return AnyType()
item = self.anal_type(t.args[0])
return make_optional_type(item)
elif fullname == 'typing.Callable':
return self.analyze_callable_type(t)
elif fullname == 'typing.Type':
if len(t.args) == 0:
return TypeType(AnyType(), line=t.line)
if len(t.args) != 1:
self.fail('Type[...] must have exactly one type argument', t)
item = self.anal_type(t.args[0])
return TypeType(item, line=t.line)
elif fullname == 'typing.ClassVar':
if self.nesting_level > 0:
self.fail('Invalid type: ClassVar nested inside other type', t)
if len(t.args) == 0:
return AnyType(line=t.line)
if len(t.args) != 1:
self.fail('ClassVar[...] must have at most one type argument', t)
return AnyType()
item = self.anal_type(t.args[0])
if isinstance(item, TypeVarType) or get_type_vars(item):
self.fail('Invalid type: ClassVar cannot be generic', t)
return AnyType()
return item
elif fullname in ('mypy_extensions.NoReturn', 'typing.NoReturn'):
return UninhabitedType(is_noreturn=True)
elif sym.kind == TYPE_ALIAS:
override = sym.type_override
assert override is not None
an_args = self.anal_array(t.args)
all_vars = self.get_type_var_names(override)
exp_len = len(all_vars)
act_len = len(an_args)
if exp_len > 0 and act_len == 0:
# Interpret bare Alias same as normal generic, i.e., Alias[Any, Any, ...]
return self.replace_alias_tvars(override, all_vars, [AnyType()] * exp_len,
t.line, t.column)
if exp_len == 0 and act_len == 0:
return override
if act_len != exp_len:
self.fail('Bad number of arguments for type alias, expected: %s, given: %s'
% (exp_len, act_len), t)
return t
return self.replace_alias_tvars(override, all_vars, an_args, t.line, t.column)
elif not isinstance(sym.node, TypeInfo):
name = sym.fullname
if name is None:
name = sym.node.name()
if isinstance(sym.node, Var) and isinstance(sym.node.type, AnyType):
# Something with an Any type -- make it an alias for Any in a type
# context. This is slightly problematic as it allows using the type 'Any'
# as a base class -- however, this will fail soon at runtime so the problem
# is pretty minor.
return AnyType(from_unimported_type=True)
# Allow unbound type variables when defining an alias
if not (self.aliasing and sym.kind == TVAR and
self.tvar_scope.get_binding(sym) is None):
self.fail('Invalid type "{}"'.format(name), t)
return t
info = sym.node # type: TypeInfo
if len(t.args) > 0 and info.fullname() == 'builtins.tuple':
return TupleType(self.anal_array(t.args),
Instance(info, [AnyType()], t.line),
t.line)
else:
# Analyze arguments and construct Instance type. The
# number of type arguments and their values are
# checked only later, since we do not always know the
# valid count at this point. Thus we may construct an
# Instance with an invalid number of type arguments.
instance = Instance(info, self.anal_array(t.args), t.line, t.column)
tup = info.tuple_type
if tup is not None:
# The class has a Tuple[...] base class so it will be
# represented as a tuple type.
if t.args:
self.fail('Generic tuple types not supported', t)
return AnyType()
return tup.copy_modified(items=self.anal_array(tup.items),
fallback=instance)
td = info.typeddict_type
if td is not None:
# The class has a TypedDict[...] base class so it will be
# represented as a typeddict type.
if t.args:
self.fail('Generic TypedDict types not supported', t)
return AnyType()
# Create a named TypedDictType
return td.copy_modified(item_types=self.anal_array(list(td.items.values())),
fallback=instance)
return instance
else:
return AnyType()
def transform(self) -> None:
"""Apply all the necessary transformations to the underlying
dataclass so as to ensure it is fully type checked according
to the rules in PEP 557.
"""
ctx = self._ctx
info = self._ctx.cls.info
attributes = self.collect_attributes()
decorator_arguments = {
'init': _get_decorator_bool_argument(self._ctx, 'init', True),
'eq': _get_decorator_bool_argument(self._ctx, 'eq', True),
'order': _get_decorator_bool_argument(self._ctx, 'order', False),
'frozen': _get_decorator_bool_argument(self._ctx, 'frozen', False),
}
if decorator_arguments['init']:
_add_method(
ctx,
'__init__',
args=[attr.to_argument(info) for attr in attributes if attr.is_in_init],
return_type=NoneTyp(),
)
# Add an eq method, but only if the class doesn't already have one.
if decorator_arguments['eq'] and info.get('__eq__') is None:
for method_name in ['__eq__', '__ne__']:
# The TVar is used to enforce that "other" must have
# the same type as self (covariant). Note the
# "self_type" parameter to _add_method.
obj_type = ctx.api.named_type('__builtins__.object')
cmp_tvar_def = TypeVarDef('T', 'T', -1, [], obj_type)
cmp_other_type = TypeVarType(cmp_tvar_def)
cmp_return_type = ctx.api.named_type('__builtins__.bool')
_add_method(
ctx,
method_name,
args=[Argument(Var('other', cmp_other_type), cmp_other_type, None, ARG_POS)],
return_type=cmp_return_type,
self_type=cmp_other_type,
tvar_def=cmp_tvar_def,
)
# Add <, >, <=, >=, but only if the class has an eq method.
if decorator_arguments['order']:
if not decorator_arguments['eq']:
ctx.api.fail('eq must be True if order is True', ctx.cls)
for method_name in ['__lt__', '__gt__', '__le__', '__ge__']:
# Like for __eq__ and __ne__, we want "other" to match
# the self type.
obj_type = ctx.api.named_type('__builtins__.object')
order_tvar_def = TypeVarDef('T', 'T', -1, [], obj_type)
order_other_type = TypeVarType(order_tvar_def)
order_return_type = ctx.api.named_type('__builtins__.bool')
order_args = [
Argument(Var('other', order_other_type), order_other_type, None, ARG_POS)
]
existing_method = info.get(method_name)
if existing_method is not None:
assert existing_method.node
ctx.api.fail(
'You may not have a custom %s method when order=True' % method_name,
existing_method.node,
)
_add_method(
ctx,
method_name,
args=order_args,
return_type=order_return_type,
self_type=order_other_type,
tvar_def=order_tvar_def,
)
if decorator_arguments['frozen']:
self._freeze(attributes)
# Remove init-only vars from the class.
for attr in attributes:
if attr.is_init_var:
del info.names[attr.name]
info.metadata['dataclass'] = {
'attributes': OrderedDict((attr.name, attr.serialize()) for attr in attributes),
'frozen': decorator_arguments['frozen'],
}
def visit_unbound_type_nonoptional(self, t: UnboundType) -> Type:
sym = self.lookup(t.name, t, suppress_errors=self.third_pass)
if '.' in t.name:
# Handle indirect references to imported names.
#
# TODO: Do this for module-local references as well and remove ImportedName
# type check below.
sym = self.api.dereference_module_cross_ref(sym)
if sym is not None:
if isinstance(sym.node, ImportedName):
# Forward reference to an imported name that hasn't been processed yet.
# To maintain backward compatibility, these get translated to Any.
#
# TODO: Remove this special case.
return AnyType(TypeOfAny.implementation_artifact)
if sym.node is None:
# UNBOUND_IMPORTED can happen if an unknown name was imported.
if sym.kind != UNBOUND_IMPORTED:
self.fail(
'Internal error (node is None, kind={})'.format(
sym.kind), t)
return AnyType(TypeOfAny.special_form)
fullname = sym.node.fullname()
hook = self.plugin.get_type_analyze_hook(fullname)
if hook:
return hook(AnalyzeTypeContext(t, t, self))
if (fullname in nongen_builtins and t.args
and not self.allow_unnormalized):
self.fail(
no_subscript_builtin_alias(
fullname, propose_alt=not self.defining_alias), t)
if self.tvar_scope:
tvar_def = self.tvar_scope.get_binding(sym)
else:
tvar_def = None
if sym.kind == TVAR and tvar_def is not None and self.defining_alias:
self.fail(
'Can\'t use bound type variable "{}"'
' to define generic alias'.format(t.name), t)
return AnyType(TypeOfAny.from_error)
elif sym.kind == TVAR and tvar_def is not None:
if len(t.args) > 0:
self.fail(
'Type variable "{}" used with arguments'.format(
t.name), t)
return TypeVarType(tvar_def, t.line)
elif fullname == 'builtins.None':
return NoneTyp()
elif fullname == 'typing.Any' or fullname == 'builtins.Any':
return AnyType(TypeOfAny.explicit)
elif fullname == 'typing.Tuple':
if len(t.args) == 0 and not t.empty_tuple_index:
# Bare 'Tuple' is same as 'tuple'
if self.options.disallow_any_generics and not self.is_typeshed_stub:
self.fail(messages.BARE_GENERIC, t)
return self.named_type('builtins.tuple',
line=t.line,
column=t.column)
if len(t.args) == 2 and isinstance(t.args[1], EllipsisType):
# Tuple[T, ...] (uniform, variable-length tuple)
instance = self.named_type('builtins.tuple',
[self.anal_type(t.args[0])])
instance.line = t.line
return instance
return self.tuple_type(self.anal_array(t.args))
elif fullname == 'typing.Union':
items = self.anal_array(t.args)
return UnionType.make_union(items)
elif fullname == 'typing.Optional':
if len(t.args) != 1:
self.fail(
'Optional[...] must have exactly one type argument', t)
return AnyType(TypeOfAny.from_error)
item = self.anal_type(t.args[0])
return make_optional_type(item)
elif fullname == 'typing.Callable':
return self.analyze_callable_type(t)
elif fullname == 'typing.Type':
if len(t.args) == 0:
any_type = AnyType(TypeOfAny.from_omitted_generics,
line=t.line,
column=t.column)
return TypeType(any_type, line=t.line, column=t.column)
if len(t.args) != 1:
self.fail('Type[...] must have exactly one type argument',
t)
item = self.anal_type(t.args[0])
return TypeType.make_normalized(item, line=t.line)
elif fullname == 'typing.ClassVar':
if self.nesting_level > 0:
self.fail(
'Invalid type: ClassVar nested inside other type', t)
if len(t.args) == 0:
return AnyType(TypeOfAny.from_omitted_generics,
line=t.line,
column=t.column)
if len(t.args) != 1:
self.fail(
'ClassVar[...] must have at most one type argument', t)
return AnyType(TypeOfAny.from_error)
item = self.anal_type(t.args[0])
if isinstance(item, TypeVarType) or get_type_vars(item):
self.fail('Invalid type: ClassVar cannot be generic', t)
return AnyType(TypeOfAny.from_error)
return item
elif fullname in ('mypy_extensions.NoReturn', 'typing.NoReturn'):
return UninhabitedType(is_noreturn=True)
elif isinstance(sym.node, TypeAlias):
self.aliases_used.add(sym.node.fullname())
all_vars = sym.node.alias_tvars
target = sym.node.target
an_args = self.anal_array(t.args)
return expand_type_alias(target, all_vars, an_args, self.fail,
sym.node.no_args, t)
elif not isinstance(sym.node, TypeInfo):
# Something unusual. We try our best to find out what it is.
name = sym.fullname
if name is None:
name = sym.node.name()
# Option 1:
# Something with an Any type -- make it an alias for Any in a type
# context. This is slightly problematic as it allows using the type 'Any'
# as a base class -- however, this will fail soon at runtime so the problem
# is pretty minor.
if isinstance(sym.node, Var) and isinstance(
sym.node.type, AnyType):
return AnyType(
TypeOfAny.from_unimported_type,
missing_import_name=sym.node.type.missing_import_name)
# Option 2:
# Unbound type variable. Currently these may be still valid,
# for example when defining a generic type alias.
unbound_tvar = ((sym.kind == TVAR) and
(not self.tvar_scope
or self.tvar_scope.get_binding(sym) is None))
if self.allow_unbound_tvars and unbound_tvar and not self.third_pass:
return t
# Option 3:
# If it is not something clearly bad (like a known function, variable,
# type variable, or module), and it is still not too late, we try deferring
# this type using a forward reference wrapper. It will be revisited in
# the third pass.
allow_forward_ref = not (
self.third_pass or isinstance(sym.node,
(FuncDef, Decorator))
or isinstance(sym.node, Var) and sym.node.is_ready
or sym.kind in (MODULE_REF, TVAR))
if allow_forward_ref:
# We currently can't support subscripted forward refs in functions;
# see https://github.com/python/mypy/pull/3952#discussion_r139950690
# for discussion.
if t.args and not self.global_scope:
if not self.in_dynamic_func:
self.fail(
'Unsupported forward reference to "{}"'.format(
t.name), t)
return AnyType(TypeOfAny.from_error)
return ForwardRef(t)
# None of the above options worked, we give up.
self.fail('Invalid type "{}"'.format(name), t)
if self.third_pass and sym.kind == TVAR:
self.note_func(
"Forward references to type variables are prohibited",
t)
return AnyType(TypeOfAny.from_error)
# TODO: Would it be better to always return Any instead of UnboundType
# in case of an error? On one hand, UnboundType has a name so error messages
# are more detailed, on the other hand, some of them may be bogus.
return t
info = sym.node # type: TypeInfo
if len(t.args) > 0 and info.fullname() == 'builtins.tuple':
fallback = Instance(info, [AnyType(TypeOfAny.special_form)],
t.line)
return TupleType(self.anal_array(t.args), fallback, t.line)
else:
# Analyze arguments and construct Instance type. The
# number of type arguments and their values are
# checked only later, since we do not always know the
# valid count at this point. Thus we may construct an
# Instance with an invalid number of type arguments.
instance = Instance(info, self.anal_array(t.args), t.line,
t.column)
if not t.args and self.options.disallow_any_generics and not self.defining_alias:
# We report/patch invalid built-in instances already during second pass.
# This is done to avoid storing additional state on instances.
# All other (including user defined) generics will be patched/reported
# in the third pass.
if not self.is_typeshed_stub and info.fullname(
) in nongen_builtins:
alternative = nongen_builtins[info.fullname()]
self.fail(
messages.IMPLICIT_GENERIC_ANY_BUILTIN.format(
alternative), t)
any_type = AnyType(TypeOfAny.from_error, line=t.line)
else:
any_type = AnyType(TypeOfAny.from_omitted_generics,
line=t.line)
instance.args = [any_type] * len(info.type_vars)
tup = info.tuple_type
if tup is not None:
# The class has a Tuple[...] base class so it will be
# represented as a tuple type.
if t.args:
self.fail('Generic tuple types not supported', t)
return AnyType(TypeOfAny.from_error)
return tup.copy_modified(items=self.anal_array(tup.items),
fallback=instance)
td = info.typeddict_type
if td is not None:
# The class has a TypedDict[...] base class so it will be
# represented as a typeddict type.
if t.args:
self.fail('Generic TypedDict types not supported', t)
return AnyType(TypeOfAny.from_error)
# Create a named TypedDictType
return td.copy_modified(item_types=self.anal_array(
list(td.items.values())),
fallback=instance)
return instance
else:
if self.third_pass:
self.fail('Invalid type "{}"'.format(t.name), t)
return AnyType(TypeOfAny.from_error)
return AnyType(TypeOfAny.special_form)
def visit_type_var(self, t: TypeVarType) -> Type:
if t.id < 0:
return t
else:
return TypeVarType(t.name, -t.id - self.num_func_tvars, t.values,
t.upper_bound, t.variance)
def visit_unbound_type(self, t: UnboundType) -> Type:
sym = self.lookup(t.name, t)
if sym is not None:
fullname = sym.node.fullname()
if sym.kind == BOUND_TVAR:
if len(t.args) > 0:
self.fail(
'Type variable "{}" used with arguments'.format(
t.name), t)
tvar_expr = cast(TypeVarExpr, sym.node)
return TypeVarType(t.name, sym.tvar_id, tvar_expr.values,
self.builtin_type('builtins.object'),
tvar_expr.variance, t.line)
elif fullname == 'builtins.None':
return Void()
elif fullname == 'typing.Any':
return AnyType()
elif fullname == 'typing.Tuple':
if len(t.args) == 2 and isinstance(t.args[1], EllipsisType):
# Tuple[T, ...] (uniform, variable-length tuple)
node = self.lookup_fqn_func('builtins.tuple')
info = cast(TypeInfo, node.node)
return Instance(info, [t.args[0].accept(self)], t.line)
return TupleType(self.anal_array(t.args),
self.builtin_type('builtins.tuple'))
elif fullname == 'typing.Union':
items = self.anal_array(t.args)
items = [item for item in items if not isinstance(item, Void)]
return UnionType.make_union(items)
elif fullname == 'typing.Optional':
if len(t.args) != 1:
self.fail(
'Optional[...] must have exactly one type argument', t)
items = self.anal_array(t.args)
# Currently Optional[t] is just an alias for t.
return items[0]
elif fullname == 'typing.Callable':
return self.analyze_callable_type(t)
elif sym.kind == TYPE_ALIAS:
# TODO: Generic type aliases.
return sym.type_override
elif not isinstance(sym.node, TypeInfo):
name = sym.fullname
if name is None:
name = sym.node.name()
if isinstance(sym.node, Var) and isinstance(
sym.node.type, AnyType):
# Something with an Any type -- make it an alias for Any in a type
# context. This is slightly problematic as it allows using the type 'Any'
# as a base class -- however, this will fail soon at runtime so the problem
# is pretty minor.
return AnyType()
self.fail('Invalid type "{}"'.format(name), t)
return t
info = cast(TypeInfo, sym.node)
if len(t.args) > 0 and info.fullname() == 'builtins.tuple':
return TupleType(self.anal_array(t.args),
Instance(info, [AnyType()], t.line), t.line)
else:
# Analyze arguments and construct Instance type. The
# number of type arguments and their values are
# checked only later, since we do not always know the
# valid count at this point. Thus we may construct an
# Instance with an invalid number of type arguments.
instance = Instance(info, self.anal_array(t.args), t.line)
if info.tuple_type is None:
return instance
else:
# The class has a Tuple[...] base class so it will be
# represented as a tuple type.
if t.args:
self.fail('Generic tuple types not supported', t)
return AnyType()
return TupleType(self.anal_array(info.tuple_type.items),
fallback=instance,
line=t.line)
else:
return t
def snapshot_definition(node: Optional[SymbolNode],
common: Tuple[object, ...]) -> Tuple[object, ...]:
"""Create a snapshot description of a symbol table node.
The representation is nested tuples and dicts. Only externally
visible attributes are included.
"""
if isinstance(node, FuncBase):
# TODO: info
if node.type:
signature = snapshot_type(node.type)
else:
signature = snapshot_untyped_signature(node)
return ('Func', common, node.is_property, node.is_final, node.is_class,
node.is_static, signature)
elif isinstance(node, Var):
return ('Var', common, snapshot_optional_type(node.type),
node.is_final)
elif isinstance(node, Decorator):
# Note that decorated methods are represented by Decorator instances in
# a symbol table since we need to preserve information about the
# decorated function (whether it's a class function, for
# example). Top-level decorated functions, however, are represented by
# the corresponding Var node, since that happens to provide enough
# context.
return ('Decorator', node.is_overload,
snapshot_optional_type(node.var.type),
snapshot_definition(node.func, common))
elif isinstance(node, TypeInfo):
attrs = (
node.is_abstract,
node.is_enum,
node.is_protocol,
node.fallback_to_any,
node.is_named_tuple,
node.is_newtype,
# We need this to e.g. trigger metaclass calculation in subclasses.
snapshot_optional_type(node.metaclass_type),
snapshot_optional_type(node.tuple_type),
snapshot_optional_type(node.typeddict_type),
[base.fullname() for base in node.mro],
# Note that the structure of type variables is a part of the external interface,
# since creating instances might fail, for example:
# T = TypeVar('T', bound=int)
# class C(Generic[T]):
# ...
# x: C[str] <- this is invalid, and needs to be re-checked if `T` changes.
# An alternative would be to create both deps: <...> -> C, and <...> -> <C>,
# but this currently seems a bit ad hoc.
tuple(
snapshot_type(TypeVarType(tdef))
for tdef in node.defn.type_vars),
[snapshot_type(base) for base in node.bases],
snapshot_optional_type(node._promote))
prefix = node.fullname()
symbol_table = snapshot_symbol_table(prefix, node.names)
# Special dependency for abstract attribute handling.
symbol_table['(abstract)'] = ('Abstract',
tuple(sorted(node.abstract_attributes)))
return ('TypeInfo', common, attrs, symbol_table)
else:
# Other node types are handled elsewhere.
assert False, type(node)
