def do_func_def(self,
n: Union[ast35.FunctionDef, ast35.AsyncFunctionDef],
is_coroutine: bool = False) -> Union[FuncDef, Decorator]:
"""Helper shared between visit_FunctionDef and visit_AsyncFunctionDef."""
args = self.transform_args(n.args, n.lineno)
arg_kinds = [arg.kind for arg in args]
arg_names = [arg.variable.name() for arg in args]
arg_types = None # type: List[Type]
if n.type_comment is not None:
try:
func_type_ast = ast35.parse(n.type_comment, '<func_type>',
'func_type')
except SyntaxError:
raise TypeCommentParseError(TYPE_COMMENT_SYNTAX_ERROR,
n.lineno, n.col_offset)
assert isinstance(func_type_ast, ast35.FunctionType)
# for ellipsis arg
if (len(func_type_ast.argtypes) == 1
and isinstance(func_type_ast.argtypes[0], ast35.Ellipsis)):
arg_types = [
a.type_annotation
if a.type_annotation is not None else AnyType()
for a in args
]
else:
translated_args = (TypeConverter(
line=n.lineno).translate_expr_list(func_type_ast.argtypes))
arg_types = [
a if a is not None else AnyType() for a in translated_args
]
return_type = TypeConverter(line=n.lineno).visit(
func_type_ast.returns)
# add implicit self type
if self.in_class() and len(arg_types) < len(args):
arg_types.insert(0, AnyType())
else:
arg_types = [a.type_annotation for a in args]
return_type = TypeConverter(line=n.lineno).visit(n.returns)
for arg, arg_type in zip(args, arg_types):
self.set_type_optional(arg_type, arg.initializer)
if isinstance(return_type, UnboundType):
return_type.is_ret_type = True
func_type = None
if any(arg_types) or return_type:
func_type = CallableType(
[a if a is not None else AnyType()
for a in arg_types], arg_kinds, arg_names,
return_type if return_type is not None else AnyType(), None)
func_def = FuncDef(n.name, args, self.as_block(n.body, n.lineno),
func_type)
if is_coroutine:
# A coroutine is also a generator, mostly for internal reasons.
func_def.is_generator = func_def.is_coroutine = True
if func_type is not None:
func_type.definition = func_def
func_type.line = n.lineno
if n.decorator_list:
var = Var(func_def.name())
var.is_ready = False
var.set_line(n.decorator_list[0].lineno)
func_def.is_decorated = True
func_def.set_line(n.lineno + len(n.decorator_list))
func_def.body.set_line(func_def.get_line())
return Decorator(func_def,
self.translate_expr_list(n.decorator_list), var)
else:
return func_def
def get_assignment_target(self,
lvalue: Lvalue,
line: int = -1) -> AssignmentTarget:
if isinstance(lvalue, NameExpr):
# If we are visiting a decorator, then the SymbolNode we really want to be looking at
# is the function that is decorated, not the entire Decorator node itself.
symbol = lvalue.node
if isinstance(symbol, Decorator):
symbol = symbol.func
if symbol is None:
# New semantic analyzer doesn't create ad-hoc Vars for special forms.
assert lvalue.is_special_form
symbol = Var(lvalue.name)
if lvalue.kind == LDEF:
if symbol not in self.environment.symtable:
# If the function is a generator function, then first define a new variable
# in the current function's environment class. Next, define a target that
# refers to the newly defined variable in that environment class. Add the
# target to the table containing class environment variables, as well as the
# current environment.
if self.fn_info.is_generator:
return self.add_var_to_env_class(
symbol,
self.node_type(lvalue),
self.fn_info.generator_class,
reassign=False)
# Otherwise define a new local variable.
return self.environment.add_local_reg(
symbol, self.node_type(lvalue))
else:
# Assign to a previously defined variable.
return self.environment.lookup(symbol)
elif lvalue.kind == GDEF:
globals_dict = self.load_globals_dict()
name = self.load_static_unicode(lvalue.name)
return AssignmentTargetIndex(globals_dict, name)
else:
assert False, lvalue.kind
elif isinstance(lvalue, IndexExpr):
# Indexed assignment x[y] = e
base = self.accept(lvalue.base)
index = self.accept(lvalue.index)
return AssignmentTargetIndex(base, index)
elif isinstance(lvalue, MemberExpr):
# Attribute assignment x.y = e
obj = self.accept(lvalue.expr)
return AssignmentTargetAttr(obj, lvalue.name)
elif isinstance(lvalue, TupleExpr):
# Multiple assignment a, ..., b = e
star_idx = None # type: Optional[int]
lvalues = []
for idx, item in enumerate(lvalue.items):
targ = self.get_assignment_target(item)
lvalues.append(targ)
if isinstance(item, StarExpr):
if star_idx is not None:
self.error("Two starred expressions in assignment",
line)
star_idx = idx
return AssignmentTargetTuple(lvalues, star_idx)
elif isinstance(lvalue, StarExpr):
return self.get_assignment_target(lvalue.expr)
assert False, 'Unsupported lvalue: %r' % lvalue
def transform_args(
self,
n: ast27.arguments,
line: int,
) -> Tuple[List[Argument], List[Statement]]:
# TODO: remove the cast once https://github.com/python/typeshed/pull/522
# is accepted and synced
type_comments = cast(List[str], n.type_comments) # type: ignore
converter = TypeConverter(self.errors, line=line)
decompose_stmts = [] # type: List[Statement]
def extract_names(arg: ast27.expr) -> List[str]:
if isinstance(arg, ast27.Name):
return [arg.id]
elif isinstance(arg, ast27.Tuple):
return [
name for elt in arg.elts for name in extract_names(elt)
]
else:
return []
def convert_arg(index: int, arg: ast27.expr) -> Var:
if isinstance(arg, ast27.Name):
v = arg.id
elif isinstance(arg, ast27.Tuple):
v = '__tuple_arg_{}'.format(index + 1)
rvalue = NameExpr(v)
rvalue.set_line(line)
assignment = AssignmentStmt([self.visit(arg)], rvalue)
assignment.set_line(line)
decompose_stmts.append(assignment)
else:
raise RuntimeError("'{}' is not a valid argument.".format(
ast27.dump(arg)))
return Var(v)
def get_type(i: int) -> Optional[Type]:
if i < len(type_comments) and type_comments[i] is not None:
return converter.visit_raw_str(type_comments[i])
return None
args = [(convert_arg(i, arg), get_type(i))
for i, arg in enumerate(n.args)]
defaults = self.translate_expr_list(n.defaults)
names = [name for arg in n.args
for name in extract_names(arg)] # type: List[str]
new_args = [] # type: List[Argument]
num_no_defaults = len(args) - len(defaults)
# positional arguments without defaults
for a, annotation in args[:num_no_defaults]:
new_args.append(Argument(a, annotation, None, ARG_POS))
# positional arguments with defaults
for (a, annotation), d in zip(args[num_no_defaults:], defaults):
new_args.append(Argument(a, annotation, d, ARG_OPT))
# *arg
if n.vararg is not None:
new_args.append(
Argument(Var(n.vararg), get_type(len(args)), None, ARG_STAR))
names.append(n.vararg)
# **kwarg
if n.kwarg is not None:
typ = get_type(len(args) + (0 if n.vararg is None else 1))
new_args.append(Argument(Var(n.kwarg), typ, None, ARG_STAR2))
names.append(n.kwarg)
# We don't have any context object to give, but we have closed around the line num
def fail_arg(msg: str, arg: None) -> None:
self.fail(msg, line, 0)
check_arg_names(names, [None] * len(names), fail_arg)
return new_args, decompose_stmts
def to_var(self, info: TypeInfo) -> Var:
return Var(self.name, info[self.name].type)
def analyze_member_var_access(name: str, itype: Instance, info: TypeInfo,
node: Context, is_lvalue: bool, is_super: bool,
builtin_type: Callable[[str], Instance],
not_ready_callback: Callable[[str, Context], None],
msg: MessageBuilder,
original_type: Type,
chk: 'mypy.checker.TypeChecker') -> Type:
"""Analyse attribute access that does not target a method.
This is logically part of analyze_member_access and the arguments are similar.
original_type is the type of E in the expression E.var
"""
# It was not a method. Try looking up a variable.
v = lookup_member_var_or_accessor(info, name, is_lvalue)
vv = v
if isinstance(vv, Decorator):
# The associated Var node of a decorator contains the type.
v = vv.var
if isinstance(vv, TypeInfo):
# If the associated variable is a TypeInfo synthesize a Var node for
# the purposes of type checking. This enables us to type check things
# like accessing class attributes on an inner class.
v = Var(name, type=type_object_type(vv, builtin_type))
v.info = info
if isinstance(vv, TypeAlias) and isinstance(vv.target, Instance):
# Similar to the above TypeInfo case, we allow using
# qualified type aliases in runtime context if it refers to an
# instance type. For example:
# class C:
# A = List[int]
# x = C.A() <- this is OK
typ = instance_alias_type(vv, builtin_type)
v = Var(name, type=typ)
v.info = info
if isinstance(v, Var):
implicit = info[name].implicit
# An assignment to final attribute is always an error,
# independently of types.
if is_lvalue and not chk.get_final_context():
check_final_member(name, info, msg, node)
return analyze_var(name, v, itype, info, node, is_lvalue, msg,
original_type, builtin_type, not_ready_callback,
chk=chk, implicit=implicit)
elif isinstance(v, FuncDef):
assert False, "Did not expect a function"
elif not v and name not in ['__getattr__', '__setattr__', '__getattribute__']:
if not is_lvalue:
for method_name in ('__getattribute__', '__getattr__'):
method = info.get_method(method_name)
# __getattribute__ is defined on builtins.object and returns Any, so without
# the guard this search will always find object.__getattribute__ and conclude
# that the attribute exists
if method and method.info.fullname() != 'builtins.object':
function = function_type(method, builtin_type('builtins.function'))
bound_method = bind_self(function, original_type)
typ = map_instance_to_supertype(itype, method.info)
getattr_type = expand_type_by_instance(bound_method, typ)
if isinstance(getattr_type, CallableType):
return getattr_type.ret_type
else:
setattr_meth = info.get_method('__setattr__')
if setattr_meth and setattr_meth.info.fullname() != 'builtins.object':
setattr_func = function_type(setattr_meth, builtin_type('builtins.function'))
bound_type = bind_self(setattr_func, original_type)
typ = map_instance_to_supertype(itype, setattr_meth.info)
setattr_type = expand_type_by_instance(bound_type, typ)
if isinstance(setattr_type, CallableType) and len(setattr_type.arg_types) > 0:
return setattr_type.arg_types[-1]
if itype.type.fallback_to_any:
return AnyType(TypeOfAny.special_form)
# Could not find the member.
if is_super:
msg.undefined_in_superclass(name, node)
return AnyType(TypeOfAny.from_error)
else:
if chk and chk.should_suppress_optional_error([itype]):
return AnyType(TypeOfAny.from_error)
return msg.has_no_attr(original_type, itype, name, node)
def setUp(self) -> None:
self.env = Environment()
self.n = self.env.add_local(Var('n'), int_rprimitive)
self.context = EmitterContext(['mod'])
self.emitter = Emitter(self.context, self.env)
def do_func_def(self,
n: Union[ast3.FunctionDef, ast3.AsyncFunctionDef],
is_coroutine: bool = False) -> Union[FuncDef, Decorator]:
"""Helper shared between visit_FunctionDef and visit_AsyncFunctionDef."""
no_type_check = bool(
n.decorator_list
and any(is_no_type_check_decorator(d) for d in n.decorator_list))
args = self.transform_args(n.args,
n.lineno,
no_type_check=no_type_check)
arg_kinds = [arg.kind for arg in args]
arg_names = [arg.variable.name()
for arg in args] # type: List[Optional[str]]
arg_names = [
None if argument_elide_name(name) else name for name in arg_names
]
if special_function_elide_names(n.name):
arg_names = [None] * len(arg_names)
arg_types = None # type: List[Type]
if no_type_check:
arg_types = [None] * len(args)
return_type = None
elif n.type_comment is not None:
try:
func_type_ast = ast3.parse(n.type_comment, '<func_type>',
'func_type')
assert isinstance(func_type_ast, ast3.FunctionType)
# for ellipsis arg
if (len(func_type_ast.argtypes) == 1 and isinstance(
func_type_ast.argtypes[0], ast3.Ellipsis)):
if n.returns:
# PEP 484 disallows both type annotations and type comments
self.fail(messages.DUPLICATE_TYPE_SIGNATURES, n.lineno,
n.col_offset)
arg_types = [
a.type_annotation
if a.type_annotation is not None else AnyType()
for a in args
]
else:
# PEP 484 disallows both type annotations and type comments
if n.returns or any(a.type_annotation is not None
for a in args):
self.fail(messages.DUPLICATE_TYPE_SIGNATURES, n.lineno,
n.col_offset)
translated_args = (TypeConverter(
self.errors, line=n.lineno).translate_expr_list(
func_type_ast.argtypes))
arg_types = [
a if a is not None else AnyType()
for a in translated_args
]
return_type = TypeConverter(self.errors, line=n.lineno).visit(
func_type_ast.returns)
# add implicit self type
if self.in_class() and len(arg_types) < len(args):
arg_types.insert(0, AnyType())
except SyntaxError:
self.fail(TYPE_COMMENT_SYNTAX_ERROR, n.lineno, n.col_offset)
arg_types = [AnyType()] * len(args)
return_type = AnyType()
else:
arg_types = [a.type_annotation for a in args]
return_type = TypeConverter(self.errors,
line=n.lineno).visit(n.returns)
for arg, arg_type in zip(args, arg_types):
self.set_type_optional(arg_type, arg.initializer)
func_type = None
if any(arg_types) or return_type:
if len(arg_types) != 1 and any(
isinstance(t, EllipsisType) for t in arg_types):
self.fail(
"Ellipses cannot accompany other argument types "
"in function type signature.", n.lineno, 0)
elif len(arg_types) > len(arg_kinds):
self.fail('Type signature has too many arguments', n.lineno, 0)
elif len(arg_types) < len(arg_kinds):
self.fail('Type signature has too few arguments', n.lineno, 0)
else:
func_type = CallableType(
[
a if a is not None else AnyType(implicit=True)
for a in arg_types
], arg_kinds, arg_names,
return_type if return_type is not None else AnyType(
implicit=True), None)
func_def = FuncDef(n.name, args, self.as_block(n.body, n.lineno),
func_type)
if is_coroutine:
# A coroutine is also a generator, mostly for internal reasons.
func_def.is_generator = func_def.is_coroutine = True
if func_type is not None:
func_type.definition = func_def
func_type.line = n.lineno
if n.decorator_list:
var = Var(func_def.name())
var.is_ready = False
var.set_line(n.decorator_list[0].lineno)
func_def.is_decorated = True
func_def.set_line(n.lineno + len(n.decorator_list))
func_def.body.set_line(func_def.get_line())
return Decorator(func_def,
self.translate_expr_list(n.decorator_list), var)
else:
return func_def
def setUp(self) -> None:
self.env = Environment()
self.n = self.env.add_local(Var('n'), int_rprimitive)
self.m = self.env.add_local(Var('m'), int_rprimitive)
self.k = self.env.add_local(Var('k'), int_rprimitive)
self.l = self.env.add_local(Var('l'), list_rprimitive) # noqa
self.ll = self.env.add_local(Var('ll'), list_rprimitive)
self.o = self.env.add_local(Var('o'), object_rprimitive)
self.o2 = self.env.add_local(Var('o2'), object_rprimitive)
self.d = self.env.add_local(Var('d'), dict_rprimitive)
self.b = self.env.add_local(Var('b'), bool_rprimitive)
self.s1 = self.env.add_local(Var('s1'), short_int_rprimitive)
self.s2 = self.env.add_local(Var('s2'), short_int_rprimitive)
self.i32 = self.env.add_local(Var('i32'), int32_rprimitive)
self.i32_1 = self.env.add_local(Var('i32_1'), int32_rprimitive)
self.i64 = self.env.add_local(Var('i64'), int64_rprimitive)
self.i64_1 = self.env.add_local(Var('i64_1'), int64_rprimitive)
self.t = self.env.add_local(Var('t'),
RTuple([int_rprimitive, bool_rprimitive]))
self.tt = self.env.add_local(
Var('tt'),
RTuple(
[RTuple([int_rprimitive, bool_rprimitive]), bool_rprimitive]))
ir = ClassIR('A', 'mod')
ir.attributes = OrderedDict([('x', bool_rprimitive),
('y', int_rprimitive)])
compute_vtable(ir)
ir.mro = [ir]
self.r = self.env.add_local(Var('r'), RInstance(ir))
self.context = EmitterContext(NameGenerator([['mod']]))
self.emitter = Emitter(self.context, self.env)
self.declarations = Emitter(self.context, self.env)
const_int_regs = {} # type: Dict[str, int]
self.visitor = FunctionEmitterVisitor(self.emitter, self.declarations,
'prog.py', 'prog',
const_int_regs)
def add_self_to_env(environment: Environment,
cls: ClassIR) -> AssignmentTargetRegister:
return environment.add_local_reg(Var(SELF_NAME),
RInstance(cls),
is_arg=True)
def transform(self) -> None:
"""Apply all the necessary transformations to the underlying
dataclass so as to ensure it is fully type checked according
to the rules in PEP 557.
"""
ctx = self._ctx
info = self._ctx.cls.info
attributes = self.collect_attributes()
decorator_arguments = {
'init': _get_decorator_bool_argument(self._ctx, 'init', True),
'eq': _get_decorator_bool_argument(self._ctx, 'eq', True),
'order': _get_decorator_bool_argument(self._ctx, 'order', False),
'frozen': _get_decorator_bool_argument(self._ctx, 'frozen', False),
}
if decorator_arguments['init']:
_add_method(
ctx,
'__init__',
args=[attr.to_argument(info) for attr in attributes if attr.is_in_init],
return_type=NoneTyp(),
)
for stmt in self._ctx.cls.defs.body:
# Fix up the types of classmethods since, by default,
# they will be based on the parent class' init.
if isinstance(stmt, Decorator) and stmt.func.is_class:
func_type = stmt.func.type
if isinstance(func_type, CallableType):
func_type.arg_types[0] = self._ctx.api.class_type(self._ctx.cls.info)
if isinstance(stmt, OverloadedFuncDef) and stmt.is_class:
func_type = stmt.type
if isinstance(func_type, Overloaded):
class_type = ctx.api.class_type(ctx.cls.info)
for item in func_type.items():
item.arg_types[0] = class_type
if stmt.impl is not None:
assert isinstance(stmt.impl, Decorator)
if isinstance(stmt.impl.func.type, CallableType):
stmt.impl.func.type.arg_types[0] = class_type
# Add an eq method, but only if the class doesn't already have one.
if decorator_arguments['eq'] and info.get('__eq__') is None:
for method_name in ['__eq__', '__ne__']:
# The TVar is used to enforce that "other" must have
# the same type as self (covariant). Note the
# "self_type" parameter to _add_method.
obj_type = ctx.api.named_type('__builtins__.object')
cmp_tvar_def = TypeVarDef('T', 'T', 1, [], obj_type)
cmp_other_type = TypeVarType(cmp_tvar_def)
cmp_return_type = ctx.api.named_type('__builtins__.bool')
_add_method(
ctx,
method_name,
args=[Argument(Var('other', cmp_other_type), cmp_other_type, None, ARG_POS)],
return_type=cmp_return_type,
self_type=cmp_other_type,
tvar_def=cmp_tvar_def,
)
# Add <, >, <=, >=, but only if the class has an eq method.
if decorator_arguments['order']:
if not decorator_arguments['eq']:
ctx.api.fail('eq must be True if order is True', ctx.cls)
for method_name in ['__lt__', '__gt__', '__le__', '__ge__']:
# Like for __eq__ and __ne__, we want "other" to match
# the self type.
obj_type = ctx.api.named_type('__builtins__.object')
order_tvar_def = TypeVarDef('T', 'T', 1, [], obj_type)
order_other_type = TypeVarType(order_tvar_def)
order_return_type = ctx.api.named_type('__builtins__.bool')
order_args = [
Argument(Var('other', order_other_type), order_other_type, None, ARG_POS)
]
existing_method = info.get(method_name)
if existing_method is not None:
assert existing_method.node
ctx.api.fail(
'You may not have a custom %s method when order=True' % method_name,
existing_method.node,
)
_add_method(
ctx,
method_name,
args=order_args,
return_type=order_return_type,
self_type=order_other_type,
tvar_def=order_tvar_def,
)
if decorator_arguments['frozen']:
self._freeze(attributes)
# Remove init-only vars from the class.
for attr in attributes:
if attr.is_init_var:
del info.names[attr.name]
info.metadata['dataclass'] = {
'attributes': OrderedDict((attr.name, attr.serialize()) for attr in attributes),
'frozen': decorator_arguments['frozen'],
}
def setUp(self) -> None:
self.var = Var('arg')
self.arg = RuntimeArg('arg', int_rprimitive)
self.env = Environment()
self.reg = self.env.add_local(self.var, int_rprimitive)
self.block = BasicBlock(0)
def _add_method(
ctx: ClassDefContext,
name: str,
args: List[Argument],
return_type: mypy.types.Type,
self_type: Optional[mypy.types.Type] = None,
tvar_def: Optional[mypy.types.TypeVarDef] = None,
is_classmethod: bool = False,
) -> None:
"""Adds a new method to a class.
"""
info = ctx.cls.info
# First remove any previously generated methods with the same name
# to avoid clashes and problems in new semantic analyzer.
if name in info.names:
sym = info.names[name]
if sym.plugin_generated and isinstance(sym.node, FuncDef):
ctx.cls.defs.body.remove(sym.node)
if is_classmethod:
first = Argument(
Var('cls'),
# Working around python/mypy#5416.
# This should be: mypy.types.TypeType.make_normalized(self_type)
mypy.types.AnyType(mypy.types.TypeOfAny.implementation_artifact),
None,
ARG_POS)
else:
self_type = self_type or fill_typevars(info)
first = Argument(Var('self'), self_type, None, ARG_POS)
args = [first] + args
function_type = ctx.api.named_type('__builtins__.function')
arg_types, arg_names, arg_kinds = [], [], []
for arg in args:
assert arg.type_annotation, 'All arguments must be fully typed.'
arg_types.append(arg.type_annotation)
arg_names.append(get_name(arg.variable))
arg_kinds.append(arg.kind)
signature = mypy.types.CallableType(arg_types, arg_kinds, arg_names,
return_type, function_type)
if tvar_def:
signature.variables = [tvar_def]
func = FuncDef(name, args, Block([PassStmt()]))
func.info = info
func.is_class = is_classmethod
func.type = set_callable_name(signature, func)
func._fullname = get_fullname(info) + '.' + name
func.line = info.line
# NOTE: we would like the plugin generated node to dominate, but we still
# need to keep any existing definitions so they get semantically analyzed.
if name in info.names:
# Get a nice unique name instead.
r_name = get_unique_redefinition_name(name, info.names)
info.names[r_name] = info.names[name]
info.defn.defs.body.append(func)
info.names[name] = SymbolTableNode(MDEF, func, plugin_generated=True)
def transform(self) -> bool:
"""Apply all the necessary transformations to the underlying
dataclass so as to ensure it is fully type checked according
to the rules in PEP 557.
"""
ctx = self._ctx
info = self._ctx.cls.info
attributes = self.collect_attributes()
if attributes is None:
# Some definitions are not ready. We need another pass.
return False
for attr in attributes:
if attr.type is None:
return False
decorator_arguments = {
'init': _get_decorator_bool_argument(self._ctx, 'init', True),
'eq': _get_decorator_bool_argument(self._ctx, 'eq', True),
'order': _get_decorator_bool_argument(self._ctx, 'order', False),
'frozen': _get_decorator_bool_argument(self._ctx, 'frozen', False),
'slots': _get_decorator_bool_argument(self._ctx, 'slots', False),
'match_args': _get_decorator_bool_argument(self._ctx, 'match_args', True),
}
py_version = self._ctx.api.options.python_version
# If there are no attributes, it may be that the semantic analyzer has not
# processed them yet. In order to work around this, we can simply skip generating
# __init__ if there are no attributes, because if the user truly did not define any,
# then the object default __init__ with an empty signature will be present anyway.
if (decorator_arguments['init'] and
('__init__' not in info.names or info.names['__init__'].plugin_generated) and
attributes):
args = [attr.to_argument() for attr in attributes if attr.is_in_init
and not self._is_kw_only_type(attr.type)]
if info.fallback_to_any:
# Make positional args optional since we don't know their order.
# This will at least allow us to typecheck them if they are called
# as kwargs
for arg in args:
if arg.kind == ARG_POS:
arg.kind = ARG_OPT
nameless_var = Var('')
args = [Argument(nameless_var, AnyType(TypeOfAny.explicit), None, ARG_STAR),
*args,
Argument(nameless_var, AnyType(TypeOfAny.explicit), None, ARG_STAR2),
]
add_method(
ctx,
'__init__',
args=args,
return_type=NoneType(),
)
if (decorator_arguments['eq'] and info.get('__eq__') is None or
decorator_arguments['order']):
# Type variable for self types in generated methods.
obj_type = ctx.api.named_type('builtins.object')
self_tvar_expr = TypeVarExpr(SELF_TVAR_NAME, info.fullname + '.' + SELF_TVAR_NAME,
[], obj_type)
info.names[SELF_TVAR_NAME] = SymbolTableNode(MDEF, self_tvar_expr)
# Add <, >, <=, >=, but only if the class has an eq method.
if decorator_arguments['order']:
if not decorator_arguments['eq']:
ctx.api.fail('eq must be True if order is True', ctx.cls)
for method_name in ['__lt__', '__gt__', '__le__', '__ge__']:
# Like for __eq__ and __ne__, we want "other" to match
# the self type.
obj_type = ctx.api.named_type('builtins.object')
order_tvar_def = TypeVarType(SELF_TVAR_NAME, info.fullname + '.' + SELF_TVAR_NAME,
-1, [], obj_type)
order_return_type = ctx.api.named_type('builtins.bool')
order_args = [
Argument(Var('other', order_tvar_def), order_tvar_def, None, ARG_POS)
]
existing_method = info.get(method_name)
if existing_method is not None and not existing_method.plugin_generated:
assert existing_method.node
ctx.api.fail(
f'You may not have a custom {method_name} method when order=True',
existing_method.node,
)
add_method(
ctx,
method_name,
args=order_args,
return_type=order_return_type,
self_type=order_tvar_def,
tvar_def=order_tvar_def,
)
if decorator_arguments['frozen']:
self._propertize_callables(attributes, settable=False)
self._freeze(attributes)
else:
self._propertize_callables(attributes)
if decorator_arguments['slots']:
self.add_slots(info, attributes, correct_version=py_version >= (3, 10))
self.reset_init_only_vars(info, attributes)
if (decorator_arguments['match_args'] and
('__match_args__' not in info.names or
info.names['__match_args__'].plugin_generated) and
attributes):
str_type = ctx.api.named_type("builtins.str")
literals: List[Type] = [LiteralType(attr.name, str_type)
for attr in attributes if attr.is_in_init]
match_args_type = TupleType(literals, ctx.api.named_type("builtins.tuple"))
add_attribute_to_class(ctx.api, ctx.cls, "__match_args__", match_args_type)
self._add_dataclass_fields_magic_attribute()
info.metadata['dataclass'] = {
'attributes': [attr.serialize() for attr in attributes],
'frozen': decorator_arguments['frozen'],
}
return True
def make_argument(arg: ast35.arg, default: Optional[ast35.expr],
kind: int) -> Argument:
arg_type = TypeConverter(line=line).visit(arg.annotation)
return Argument(Var(arg.arg), arg_type, self.visit(default), kind)
def argument(self, ctx: 'mypy.plugin.ClassDefContext') -> Argument:
"""Return this attribute as an argument to __init__."""
assert self.init
init_type = self.init_type or self.info[self.name].type
if self.converter.name:
# When a converter is set the init_type is overridden by the first argument
# of the converter method.
converter = lookup_fully_qualified(self.converter.name,
ctx.api.modules,
raise_on_missing=False)
if not converter:
# The converter may be a local variable. Check there too.
converter = ctx.api.lookup_qualified(self.converter.name,
self.info, True)
# Get the type of the converter.
converter_type: Optional[Type] = None
if converter and isinstance(converter.node, TypeInfo):
from mypy.checkmember import type_object_type # To avoid import cycle.
converter_type = type_object_type(converter.node,
ctx.api.named_type)
elif converter and isinstance(converter.node, OverloadedFuncDef):
converter_type = converter.node.type
elif converter and converter.type:
converter_type = converter.type
init_type = None
converter_type = get_proper_type(converter_type)
if isinstance(converter_type,
CallableType) and converter_type.arg_types:
init_type = ctx.api.anal_type(converter_type.arg_types[0])
elif isinstance(converter_type, Overloaded):
types: List[Type] = []
for item in converter_type.items:
# Walk the overloads looking for methods that can accept one argument.
num_arg_types = len(item.arg_types)
if not num_arg_types:
continue
if num_arg_types > 1 and any(
kind == ARG_POS for kind in item.arg_kinds[1:]):
continue
types.append(item.arg_types[0])
# Make a union of all the valid types.
if types:
args = make_simplified_union(types)
init_type = ctx.api.anal_type(args)
if self.converter.is_attr_converters_optional and init_type:
# If the converter was attr.converter.optional(type) then add None to
# the allowed init_type.
init_type = UnionType.make_union([init_type, NoneType()])
if not init_type:
ctx.api.fail("Cannot determine __init__ type from converter",
self.context)
init_type = AnyType(TypeOfAny.from_error)
elif self.converter.name == '':
# This means we had a converter but it's not of a type we can infer.
# Error was shown in _get_converter_name
init_type = AnyType(TypeOfAny.from_error)
if init_type is None:
if ctx.api.options.disallow_untyped_defs:
# This is a compromise. If you don't have a type here then the
# __init__ will be untyped. But since the __init__ is added it's
# pointing at the decorator. So instead we also show the error in the
# assignment, which is where you would fix the issue.
node = self.info[self.name].node
assert node is not None
ctx.api.msg.need_annotation_for_var(node, self.context)
# Convert type not set to Any.
init_type = AnyType(TypeOfAny.unannotated)
if self.kw_only:
arg_kind = ARG_NAMED_OPT if self.has_default else ARG_NAMED
else:
arg_kind = ARG_OPT if self.has_default else ARG_POS
# Attrs removes leading underscores when creating the __init__ arguments.
return Argument(Var(self.name.lstrip("_"), init_type), init_type, None,
arg_kind)
def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement:
converter = TypeConverter(line=n.lineno)
args = self.transform_args(n.args, n.lineno)
arg_kinds = [arg.kind for arg in args]
arg_names = [arg.variable.name() for arg in args] # type: List[Optional[str]]
arg_names = [None if argument_elide_name(name) else name for name in arg_names]
if special_function_elide_names(n.name):
arg_names = [None] * len(arg_names)
arg_types = None # type: List[Type]
if n.type_comment is not None and len(n.type_comment) > 0:
try:
func_type_ast = ast35.parse(n.type_comment, '<func_type>', 'func_type')
except SyntaxError:
raise TypeCommentParseError(TYPE_COMMENT_SYNTAX_ERROR, n.lineno, n.col_offset)
assert isinstance(func_type_ast, ast35.FunctionType)
# for ellipsis arg
if (len(func_type_ast.argtypes) == 1 and
isinstance(func_type_ast.argtypes[0], ast35.Ellipsis)):
arg_types = [a.type_annotation if a.type_annotation is not None else AnyType()
for a in args]
else:
arg_types = [a if a is not None else AnyType() for
a in converter.translate_expr_list(func_type_ast.argtypes)]
return_type = converter.visit(func_type_ast.returns)
# add implicit self type
if self.in_class() and len(arg_types) < len(args):
arg_types.insert(0, AnyType())
else:
arg_types = [a.type_annotation for a in args]
return_type = converter.visit(None)
for arg, arg_type in zip(args, arg_types):
self.set_type_optional(arg_type, arg.initializer)
if isinstance(return_type, UnboundType):
return_type.is_ret_type = True
func_type = None
if any(arg_types) or return_type:
if len(arg_types) > len(arg_kinds):
raise FastParserError('Type signature has too many arguments', n.lineno, offset=0)
if len(arg_types) < len(arg_kinds):
raise FastParserError('Type signature has too few arguments', n.lineno, offset=0)
func_type = CallableType([a if a is not None else AnyType() for a in arg_types],
arg_kinds,
arg_names,
return_type if return_type is not None else AnyType(),
None)
func_def = FuncDef(n.name,
args,
self.as_block(n.body, n.lineno),
func_type)
if func_type is not None:
func_type.definition = func_def
func_type.line = n.lineno
if n.decorator_list:
var = Var(func_def.name())
var.is_ready = False
var.set_line(n.decorator_list[0].lineno)
func_def.is_decorated = True
func_def.set_line(n.lineno + len(n.decorator_list))
func_def.body.set_line(func_def.get_line())
return Decorator(func_def, self.translate_expr_list(n.decorator_list), var)
else:
return func_def
def visit_file(self, file: MypyFile, fnam: str, mod_id: str,
options: Options) -> None:
"""Perform the first analysis pass.
Populate module global table. Resolve the full names of
definitions not nested within functions and construct type
info structures, but do not resolve inter-definition
references such as base classes.
Also add implicit definitions such as __name__.
In this phase we don't resolve imports. For 'from ... import',
we generate dummy symbol table nodes for the imported names,
and these will get resolved in later phases of semantic
analysis.
"""
sem = self.sem
self.sem.options = options # Needed because we sometimes call into it
self.pyversion = options.python_version
self.platform = options.platform
sem.cur_mod_id = mod_id
sem.cur_mod_node = file
sem.errors.set_file(fnam, mod_id, scope=sem.scope)
sem.globals = SymbolTable()
sem.global_decls = [set()]
sem.nonlocal_decls = [set()]
sem.block_depth = [0]
sem.scope.enter_file(mod_id)
defs = file.defs
with experiments.strict_optional_set(options.strict_optional):
# Add implicit definitions of module '__name__' etc.
for name, t in implicit_module_attrs.items():
# unicode docstrings should be accepted in Python 2
if name == '__doc__':
if self.pyversion >= (3, 0):
typ = UnboundType('__builtins__.str') # type: Type
else:
typ = UnionType([
UnboundType('__builtins__.str'),
UnboundType('__builtins__.unicode')
])
else:
assert t is not None, 'type should be specified for {}'.format(
name)
typ = UnboundType(t)
v = Var(name, typ)
v._fullname = self.sem.qualified_name(name)
self.sem.globals[name] = SymbolTableNode(GDEF, v)
for d in defs:
d.accept(self)
# Add implicit definition of literals/keywords to builtins, as we
# cannot define a variable with them explicitly.
if mod_id == 'builtins':
literal_types = [
('None', NoneTyp()),
# reveal_type is a mypy-only function that gives an error with
# the type of its arg.
('reveal_type', AnyType(TypeOfAny.special_form)),
# reveal_locals is a mypy-only function that gives an error with the types of
# locals
('reveal_locals', AnyType(TypeOfAny.special_form)),
] # type: List[Tuple[str, Type]]
# TODO(ddfisher): This guard is only needed because mypy defines
# fake builtins for its tests which often don't define bool. If
# mypy is fast enough that we no longer need those, this
# conditional check should be removed.
if 'bool' in self.sem.globals:
bool_type = self.sem.named_type('bool')
literal_types.extend([
('True', bool_type),
('False', bool_type),
('__debug__', bool_type),
])
else:
# We are running tests without 'bool' in builtins.
# TODO: Find a permanent solution to this problem.
# Maybe add 'bool' to all fixtures?
literal_types.append(
('True', AnyType(TypeOfAny.special_form)))
for name, typ in literal_types:
v = Var(name, typ)
v._fullname = self.sem.qualified_name(name)
self.sem.globals[name] = SymbolTableNode(GDEF, v)
del self.sem.options
sem.scope.leave()
def gen_glue_method(
builder: IRBuilder,
sig: FuncSignature,
target: FuncIR,
cls: ClassIR,
base: ClassIR,
line: int,
do_pycall: bool,
) -> FuncIR:
"""Generate glue methods that mediate between different method types in subclasses.
For example, if we have:
class A:
def f(builder: IRBuilder, x: int) -> object: ...
then it is totally permissible to have a subclass
class B(A):
def f(builder: IRBuilder, x: object) -> int: ...
since '(object) -> int' is a subtype of '(int) -> object' by the usual
contra/co-variant function subtyping rules.
The trickiness here is that int and object have different
runtime representations in mypyc, so A.f and B.f have
different signatures at the native C level. To deal with this,
we need to generate glue methods that mediate between the
different versions by coercing the arguments and return
values.
If do_pycall is True, then make the call using the C API
instead of a native call.
"""
builder.enter()
builder.ret_types[-1] = sig.ret_type
rt_args = list(sig.args)
if target.decl.kind == FUNC_NORMAL:
rt_args[0] = RuntimeArg(sig.args[0].name, RInstance(cls))
# The environment operates on Vars, so we make some up
fake_vars = [(Var(arg.name), arg.type) for arg in rt_args]
args = [
builder.read(builder.environment.add_local_reg(var, type, is_arg=True),
line) for var, type in fake_vars
]
arg_names = [arg.name for arg in rt_args]
arg_kinds = [concrete_arg_kind(arg.kind) for arg in rt_args]
if do_pycall:
retval = builder.builder.py_method_call(args[0], target.name, args[1:],
line, arg_kinds[1:],
arg_names[1:])
else:
retval = builder.builder.call(target.decl, args, arg_kinds, arg_names,
line)
retval = builder.coerce(retval, sig.ret_type, line)
builder.add(Return(retval))
blocks, env, ret_type, _ = builder.leave()
return FuncIR(
FuncDecl(target.name + '__' + base.name + '_glue', cls.name,
builder.module_name, FuncSignature(rt_args, ret_type),
target.decl.kind), blocks, env)
def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement:
self.class_and_function_stack.append('F')
lineno = n.lineno
converter = TypeConverter(self.errors,
line=lineno,
override_column=n.col_offset,
assume_str_is_unicode=self.unicode_literals)
args, decompose_stmts = self.transform_args(n.args, lineno)
arg_kinds = [arg.kind for arg in args]
arg_names = [arg.variable.name()
for arg in args] # type: List[Optional[str]]
arg_names = [
None if argument_elide_name(name) else name for name in arg_names
]
if special_function_elide_names(n.name):
arg_names = [None] * len(arg_names)
arg_types = [] # type: List[Optional[Type]]
type_comment = n.type_comment
if (n.decorator_list and any(
is_no_type_check_decorator(d) for d in n.decorator_list)):
arg_types = [None] * len(args)
return_type = None
elif type_comment is not None and len(type_comment) > 0:
try:
func_type_ast = ast3_parse(type_comment, '<func_type>',
'func_type')
assert isinstance(func_type_ast, ast3.FunctionType)
# for ellipsis arg
if (len(func_type_ast.argtypes) == 1 and isinstance(
func_type_ast.argtypes[0], ast3.Ellipsis)):
arg_types = [
a.type_annotation if a.type_annotation is not None else
AnyType(TypeOfAny.unannotated) for a in args
]
else:
# PEP 484 disallows both type annotations and type comments
if any(a.type_annotation is not None for a in args):
self.fail(message_registry.DUPLICATE_TYPE_SIGNATURES,
lineno, n.col_offset)
arg_types = [
a if a is not None else AnyType(TypeOfAny.unannotated)
for a in converter.translate_expr_list(
func_type_ast.argtypes)
]
return_type = converter.visit(func_type_ast.returns)
# add implicit self type
if self.in_method_scope() and len(arg_types) < len(args):
arg_types.insert(0, AnyType(TypeOfAny.special_form))
except SyntaxError:
stripped_type = type_comment.split("#", 2)[0].strip()
err_msg = "{} '{}'".format(TYPE_COMMENT_SYNTAX_ERROR,
stripped_type)
self.fail(err_msg, lineno, n.col_offset)
arg_types = [AnyType(TypeOfAny.from_error)] * len(args)
return_type = AnyType(TypeOfAny.from_error)
else:
arg_types = [a.type_annotation for a in args]
return_type = converter.visit(None)
for arg, arg_type in zip(args, arg_types):
self.set_type_optional(arg_type, arg.initializer)
func_type = None
if any(arg_types) or return_type:
if len(arg_types) != 1 and any(
isinstance(t, EllipsisType) # type: ignore
for t in arg_types):
self.fail(
"Ellipses cannot accompany other argument types "
"in function type signature", lineno, n.col_offset)
elif len(arg_types) > len(arg_kinds):
self.fail('Type signature has too many arguments',
lineno,
n.col_offset,
blocker=False)
elif len(arg_types) < len(arg_kinds):
self.fail('Type signature has too few arguments',
lineno,
n.col_offset,
blocker=False)
else:
any_type = AnyType(TypeOfAny.unannotated)
func_type = CallableType(
[a if a is not None else any_type
for a in arg_types], arg_kinds, arg_names,
return_type if return_type is not None else any_type,
_dummy_fallback)
body = self.as_required_block(n.body, lineno)
if decompose_stmts:
body.body = decompose_stmts + body.body
func_def = FuncDef(n.name, args, body, func_type)
if isinstance(func_def.type, CallableType):
# semanal.py does some in-place modifications we want to avoid
func_def.unanalyzed_type = func_def.type.copy_modified()
if func_type is not None:
func_type.definition = func_def
func_type.line = lineno
if n.decorator_list:
var = Var(func_def.name())
var.is_ready = False
var.set_line(n.decorator_list[0].lineno)
func_def.is_decorated = True
func_def.set_line(lineno + len(n.decorator_list))
func_def.body.set_line(func_def.get_line())
dec = Decorator(func_def,
self.translate_expr_list(n.decorator_list), var)
dec.set_line(lineno, n.col_offset)
retval = dec # type: Statement
else:
# Overrides set_line -- can't use self.set_line
func_def.set_line(lineno, n.col_offset)
retval = func_def
self.class_and_function_stack.pop()
return retval
def to_var(self, info: TypeInfo, use_alias: bool) -> Var:
name = self.name
if use_alias and self.alias is not None:
name = self.alias
return Var(name, info[self.name].type)
def build_namedtuple_typeinfo(
self, name: str, items: List[str], types: List[Type],
default_items: Mapping[str, Expression]) -> TypeInfo:
strtype = self.api.named_type('__builtins__.str')
implicit_any = AnyType(TypeOfAny.special_form)
basetuple_type = self.api.named_type('__builtins__.tuple',
[implicit_any])
dictype = (self.api.named_type_or_none('builtins.dict',
[strtype, implicit_any])
or self.api.named_type('__builtins__.object'))
# Actual signature should return OrderedDict[str, Union[types]]
ordereddictype = (self.api.named_type_or_none('builtins.dict',
[strtype, implicit_any])
or self.api.named_type('__builtins__.object'))
fallback = self.api.named_type('__builtins__.tuple', [implicit_any])
# Note: actual signature should accept an invariant version of Iterable[UnionType[types]].
# but it can't be expressed. 'new' and 'len' should be callable types.
iterable_type = self.api.named_type_or_none('typing.Iterable',
[implicit_any])
function_type = self.api.named_type('__builtins__.function')
info = self.api.basic_new_typeinfo(name, fallback)
info.is_named_tuple = True
info.tuple_type = TupleType(types, fallback)
def patch() -> None:
# Calculate the correct value type for the fallback tuple.
assert info.tuple_type, "TupleType type deleted before calling the patch"
fallback.args[0] = join.join_type_list(list(info.tuple_type.items))
# We can't calculate the complete fallback type until after semantic
# analysis, since otherwise MROs might be incomplete. Postpone a callback
# function that patches the fallback.
self.api.schedule_patch(PRIORITY_FALLBACKS, patch)
def add_field(var: Var,
is_initialized_in_class: bool = False,
is_property: bool = False) -> None:
var.info = info
var.is_initialized_in_class = is_initialized_in_class
var.is_property = is_property
var._fullname = '%s.%s' % (info.fullname(), var.name())
info.names[var.name()] = SymbolTableNode(MDEF, var)
vars = [Var(item, typ) for item, typ in zip(items, types)]
for var in vars:
add_field(var, is_property=True)
tuple_of_strings = TupleType([strtype for _ in items], basetuple_type)
add_field(Var('_fields', tuple_of_strings),
is_initialized_in_class=True)
add_field(Var('_field_types', dictype), is_initialized_in_class=True)
add_field(Var('_field_defaults', dictype),
is_initialized_in_class=True)
add_field(Var('_source', strtype), is_initialized_in_class=True)
add_field(Var('__annotations__', ordereddictype),
is_initialized_in_class=True)
add_field(Var('__doc__', strtype), is_initialized_in_class=True)
tvd = TypeVarDef('NT', 'NT', -1, [], info.tuple_type)
selftype = TypeVarType(tvd)
def add_method(
funcname: str,
ret: Type,
args: List[Argument],
name: Optional[str] = None,
is_classmethod: bool = False,
is_new: bool = False,
) -> None:
if is_classmethod or is_new:
first = [
Argument(Var('cls'), TypeType.make_normalized(selftype),
None, ARG_POS)
]
else:
first = [Argument(Var('self'), selftype, None, ARG_POS)]
args = first + args
types = [arg.type_annotation for arg in args]
items = [arg.variable.name() for arg in args]
arg_kinds = [arg.kind for arg in args]
assert None not in types
signature = CallableType(cast(List[Type], types), arg_kinds, items,
ret, function_type)
signature.variables = [tvd]
func = FuncDef(funcname, args, Block([]))
func.info = info
func.is_class = is_classmethod
func.type = set_callable_name(signature, func)
func._fullname = info.fullname() + '.' + funcname
if is_classmethod:
v = Var(funcname, func.type)
v.is_classmethod = True
v.info = info
v._fullname = func._fullname
dec = Decorator(func, [NameExpr('classmethod')], v)
info.names[funcname] = SymbolTableNode(MDEF, dec)
else:
info.names[funcname] = SymbolTableNode(MDEF, func)
add_method('_replace',
ret=selftype,
args=[
Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT)
for var in vars
])
def make_init_arg(var: Var) -> Argument:
default = default_items.get(var.name(), None)
kind = ARG_POS if default is None else ARG_OPT
return Argument(var, var.type, default, kind)
add_method('__new__',
ret=selftype,
name=info.name(),
args=[make_init_arg(var) for var in vars],
is_new=True)
add_method('_asdict', args=[], ret=ordereddictype)
special_form_any = AnyType(TypeOfAny.special_form)
add_method('_make',
ret=selftype,
is_classmethod=True,
args=[
Argument(Var('iterable', iterable_type), iterable_type,
None, ARG_POS),
Argument(Var('new'), special_form_any, EllipsisExpr(),
ARG_NAMED_OPT),
Argument(Var('len'), special_form_any, EllipsisExpr(),
ARG_NAMED_OPT)
])
return info
def add_method(
ctx: ClassDefContext,
name: str,
args: List[Argument],
return_type: Type,
self_type: Optional[Type] = None,
tvar_def: Optional[TypeVarDef] = None,
is_classmethod: bool = False,
is_new: bool = False,
# is_staticmethod: bool = False,
) -> None:
"""
Adds a new method to a class.
This can be dropped if/when https://github.com/python/mypy/issues/7301 is merged
"""
info = ctx.cls.info
# First remove any previously generated methods with the same name
# to avoid clashes and problems in the semantic analyzer.
if name in info.names:
sym = info.names[name]
if sym.plugin_generated and isinstance(sym.node, FuncDef):
ctx.cls.defs.body.remove(sym.node)
self_type = self_type or fill_typevars(info)
if is_classmethod or is_new:
first = [
Argument(Var('_cls'), TypeType.make_normalized(self_type), None,
ARG_POS)
]
# elif is_staticmethod:
# first = []
else:
self_type = self_type or fill_typevars(info)
first = [Argument(Var('self'), self_type, None, ARG_POS)]
args = first + args
arg_types, arg_names, arg_kinds = [], [], []
for arg in args:
assert arg.type_annotation, 'All arguments must be fully typed.'
arg_types.append(arg.type_annotation)
arg_names.append(get_name(arg.variable))
arg_kinds.append(arg.kind)
function_type = ctx.api.named_type('__builtins__.function')
signature = CallableType(arg_types, arg_kinds, arg_names, return_type,
function_type)
if tvar_def:
signature.variables = [tvar_def]
func = FuncDef(name, args, Block([PassStmt()]))
func.info = info
func.type = set_callable_name(signature, func)
func.is_class = is_classmethod
# func.is_static = is_staticmethod
func._fullname = get_fullname(info) + '.' + name
func.line = info.line
# NOTE: we would like the plugin generated node to dominate, but we still
# need to keep any existing definitions so they get semantically analyzed.
if name in info.names:
# Get a nice unique name instead.
r_name = get_unique_redefinition_name(name, info.names)
info.names[r_name] = info.names[name]
if is_classmethod: # or is_staticmethod:
func.is_decorated = True
v = Var(name, func.type)
v.info = info
v._fullname = func._fullname
# if is_classmethod:
v.is_classmethod = True
dec = Decorator(func, [NameExpr('classmethod')], v)
# else:
# v.is_staticmethod = True
# dec = Decorator(func, [NameExpr('staticmethod')], v)
dec.line = info.line
sym = SymbolTableNode(MDEF, dec)
else:
sym = SymbolTableNode(MDEF, func)
sym.plugin_generated = True
info.names[name] = sym
info.defn.defs.body.append(func)
def transform(self) -> None:
"""Apply all the necessary transformations to the underlying
dataclass so as to ensure it is fully type checked according
to the rules in PEP 557.
"""
ctx = self._ctx
info = self._ctx.cls.info
attributes = self.collect_attributes()
# Check if attribute types are ready.
for attr in attributes:
if info[attr.name].type is None:
ctx.api.defer()
return
decorator_arguments = {
'init': _get_decorator_bool_argument(self._ctx, 'init', True),
'eq': _get_decorator_bool_argument(self._ctx, 'eq', True),
'order': _get_decorator_bool_argument(self._ctx, 'order', False),
'frozen': _get_decorator_bool_argument(self._ctx, 'frozen', False),
}
# If there are no attributes, it may be that the new semantic analyzer has not
# processed them yet. In order to work around this, we can simply skip generating
# __init__ if there are no attributes, because if the user truly did not define any,
# then the object default __init__ with an empty signature will be present anyway.
if decorator_arguments['init'] and '__init__' not in info.names and attributes:
add_method(
ctx,
'__init__',
args=[attr.to_argument(info) for attr in attributes if attr.is_in_init],
return_type=NoneType(),
)
if (decorator_arguments['eq'] and info.get('__eq__') is None or
decorator_arguments['order']):
# Type variable for self types in generated methods.
obj_type = ctx.api.named_type('__builtins__.object')
self_tvar_expr = TypeVarExpr(SELF_TVAR_NAME, info.fullname() + '.' + SELF_TVAR_NAME,
[], obj_type)
info.names[SELF_TVAR_NAME] = SymbolTableNode(MDEF, self_tvar_expr)
# Add an eq method, but only if the class doesn't already have one.
if decorator_arguments['eq'] and info.get('__eq__') is None:
for method_name in ['__eq__', '__ne__']:
# The TVar is used to enforce that "other" must have
# the same type as self (covariant). Note the
# "self_type" parameter to add_method.
obj_type = ctx.api.named_type('__builtins__.object')
cmp_tvar_def = TypeVarDef(SELF_TVAR_NAME, info.fullname() + '.' + SELF_TVAR_NAME,
-1, [], obj_type)
cmp_other_type = TypeVarType(cmp_tvar_def)
cmp_return_type = ctx.api.named_type('__builtins__.bool')
add_method(
ctx,
method_name,
args=[Argument(Var('other', cmp_other_type), cmp_other_type, None, ARG_POS)],
return_type=cmp_return_type,
self_type=cmp_other_type,
tvar_def=cmp_tvar_def,
)
# Add <, >, <=, >=, but only if the class has an eq method.
if decorator_arguments['order']:
if not decorator_arguments['eq']:
ctx.api.fail('eq must be True if order is True', ctx.cls)
for method_name in ['__lt__', '__gt__', '__le__', '__ge__']:
# Like for __eq__ and __ne__, we want "other" to match
# the self type.
obj_type = ctx.api.named_type('__builtins__.object')
order_tvar_def = TypeVarDef(SELF_TVAR_NAME, info.fullname() + '.' + SELF_TVAR_NAME,
-1, [], obj_type)
order_other_type = TypeVarType(order_tvar_def)
order_return_type = ctx.api.named_type('__builtins__.bool')
order_args = [
Argument(Var('other', order_other_type), order_other_type, None, ARG_POS)
]
existing_method = info.get(method_name)
if existing_method is not None and not existing_method.plugin_generated:
assert existing_method.node
ctx.api.fail(
'You may not have a custom %s method when order=True' % method_name,
existing_method.node,
)
add_method(
ctx,
method_name,
args=order_args,
return_type=order_return_type,
self_type=order_other_type,
tvar_def=order_tvar_def,
)
if decorator_arguments['frozen']:
self._freeze(attributes)
self.reset_init_only_vars(info, attributes)
info.metadata['dataclass'] = {
'attributes': OrderedDict((attr.name, attr.serialize()) for attr in attributes),
'frozen': decorator_arguments['frozen'],
}
def analyze_member_var_access(name: str,
itype: Instance,
info: TypeInfo,
mx: MemberContext) -> Type:
"""Analyse attribute access that does not target a method.
This is logically part of analyze_member_access and the arguments are similar.
original_type is the type of E in the expression E.var
"""
# It was not a method. Try looking up a variable.
v = lookup_member_var_or_accessor(info, name, mx.is_lvalue)
vv = v
if isinstance(vv, Decorator):
# The associated Var node of a decorator contains the type.
v = vv.var
if isinstance(vv, TypeInfo):
# If the associated variable is a TypeInfo synthesize a Var node for
# the purposes of type checking. This enables us to type check things
# like accessing class attributes on an inner class.
v = Var(name, type=type_object_type(vv, mx.builtin_type))
v.info = info
if isinstance(vv, TypeAlias) and isinstance(get_proper_type(vv.target), Instance):
# Similar to the above TypeInfo case, we allow using
# qualified type aliases in runtime context if it refers to an
# instance type. For example:
# class C:
# A = List[int]
# x = C.A() <- this is OK
typ = instance_alias_type(vv, mx.builtin_type)
v = Var(name, type=typ)
v.info = info
if isinstance(v, Var):
implicit = info[name].implicit
# An assignment to final attribute is always an error,
# independently of types.
if mx.is_lvalue and not mx.chk.get_final_context():
check_final_member(name, info, mx.msg, mx.context)
return analyze_var(name, v, itype, info, mx, implicit=implicit)
elif isinstance(v, FuncDef):
assert False, "Did not expect a function"
elif (not v and name not in ['__getattr__', '__setattr__', '__getattribute__'] and
not mx.is_operator):
if not mx.is_lvalue:
for method_name in ('__getattribute__', '__getattr__'):
method = info.get_method(method_name)
# __getattribute__ is defined on builtins.object and returns Any, so without
# the guard this search will always find object.__getattribute__ and conclude
# that the attribute exists
if method and method.info.fullname != 'builtins.object':
function = function_type(method, mx.builtin_type('builtins.function'))
bound_method = bind_self(function, mx.self_type)
typ = map_instance_to_supertype(itype, method.info)
getattr_type = get_proper_type(expand_type_by_instance(bound_method, typ))
if isinstance(getattr_type, CallableType):
result = getattr_type.ret_type
# Call the attribute hook before returning.
fullname = '{}.{}'.format(method.info.fullname, name)
hook = mx.chk.plugin.get_attribute_hook(fullname)
if hook:
result = hook(AttributeContext(get_proper_type(mx.original_type),
result, mx.context, mx.chk))
return result
else:
setattr_meth = info.get_method('__setattr__')
if setattr_meth and setattr_meth.info.fullname != 'builtins.object':
setattr_func = function_type(setattr_meth, mx.builtin_type('builtins.function'))
bound_type = bind_self(setattr_func, mx.self_type)
typ = map_instance_to_supertype(itype, setattr_meth.info)
setattr_type = get_proper_type(expand_type_by_instance(bound_type, typ))
if isinstance(setattr_type, CallableType) and len(setattr_type.arg_types) > 0:
return setattr_type.arg_types[-1]
if itype.type.fallback_to_any:
return AnyType(TypeOfAny.special_form)
# Could not find the member.
if mx.is_super:
mx.msg.undefined_in_superclass(name, mx.context)
return AnyType(TypeOfAny.from_error)
else:
if mx.chk and mx.chk.should_suppress_optional_error([itype]):
return AnyType(TypeOfAny.from_error)
return mx.msg.has_no_attr(
mx.original_type, itype, name, mx.context, mx.module_symbol_table
)
def make_argument(self, name: str, type: Type) -> Argument:
return Argument(Var(name), type, None, ARG_POS)
def copy_method_to_another_class(
ctx: ClassDefContext,
self_type: Instance,
new_method_name: str,
method_node: FuncDef,
return_type: Optional[MypyType] = None,
original_module_name: Optional[str] = None,
) -> None:
semanal_api = get_semanal_api(ctx)
if method_node.type is None:
if not semanal_api.final_iteration:
semanal_api.defer()
return
arguments, return_type = build_unannotated_method_args(method_node)
add_method(ctx,
new_method_name,
args=arguments,
return_type=return_type,
self_type=self_type)
return
method_type = method_node.type
if not isinstance(method_type, CallableType):
if not semanal_api.final_iteration:
semanal_api.defer()
return
if return_type is None:
return_type = bind_or_analyze_type(method_type.ret_type, semanal_api,
original_module_name)
if return_type is None:
return
try:
original_arguments = method_node.arguments[1:]
except AttributeError:
original_arguments = []
arguments = []
for arg_name, arg_type, original_argument in zip(method_type.arg_names[1:],
method_type.arg_types[1:],
original_arguments):
bound_arg_type = bind_or_analyze_type(arg_type, semanal_api,
original_module_name)
if bound_arg_type is None:
return
var = Var(name=original_argument.variable.name, type=arg_type)
var.line = original_argument.variable.line
var.column = original_argument.variable.column
argument = Argument(
variable=var,
type_annotation=bound_arg_type,
initializer=original_argument.initializer,
kind=original_argument.kind,
)
argument.set_line(original_argument)
arguments.append(argument)
add_method(ctx,
new_method_name,
args=arguments,
return_type=return_type,
self_type=self_type)
def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement:
converter = TypeConverter(self.errors, line=n.lineno)
args, decompose_stmts = self.transform_args(n.args, n.lineno)
arg_kinds = [arg.kind for arg in args]
arg_names = [arg.variable.name()
for arg in args] # type: List[Optional[str]]
arg_names = [
None if argument_elide_name(name) else name for name in arg_names
]
if special_function_elide_names(n.name):
arg_names = [None] * len(arg_names)
arg_types = None # type: List[Type]
if (n.decorator_list and any(
is_no_type_check_decorator(d) for d in n.decorator_list)):
arg_types = [None] * len(args)
return_type = None
elif n.type_comment is not None and len(n.type_comment) > 0:
try:
func_type_ast = ast3.parse(n.type_comment, '<func_type>',
'func_type')
assert isinstance(func_type_ast, ast3.FunctionType)
# for ellipsis arg
if (len(func_type_ast.argtypes) == 1 and isinstance(
func_type_ast.argtypes[0], ast3.Ellipsis)):
arg_types = [
a.type_annotation
if a.type_annotation is not None else AnyType()
for a in args
]
else:
# PEP 484 disallows both type annotations and type comments
if any(a.type_annotation is not None for a in args):
self.fail(messages.DUPLICATE_TYPE_SIGNATURES, n.lineno,
n.col_offset)
arg_types = [
a if a is not None else AnyType() for a in
converter.translate_expr_list(func_type_ast.argtypes)
]
return_type = converter.visit(func_type_ast.returns)
# add implicit self type
if self.in_class() and len(arg_types) < len(args):
arg_types.insert(0, AnyType())
except SyntaxError:
self.fail(TYPE_COMMENT_SYNTAX_ERROR, n.lineno, n.col_offset)
arg_types = [AnyType()] * len(args)
return_type = AnyType()
else:
arg_types = [a.type_annotation for a in args]
return_type = converter.visit(None)
for arg, arg_type in zip(args, arg_types):
self.set_type_optional(arg_type, arg.initializer)
func_type = None
if any(arg_types) or return_type:
if len(arg_types) != 1 and any(
isinstance(t, EllipsisType) for t in arg_types):
self.fail(
"Ellipses cannot accompany other argument types "
"in function type signature.", n.lineno, 0)
elif len(arg_types) > len(arg_kinds):
self.fail('Type signature has too many arguments', n.lineno, 0)
elif len(arg_types) < len(arg_kinds):
self.fail('Type signature has too few arguments', n.lineno, 0)
else:
func_type = CallableType(
[a if a is not None else AnyType()
for a in arg_types], arg_kinds, arg_names,
return_type if return_type is not None else AnyType(),
None)
body = self.as_block(n.body, n.lineno)
if decompose_stmts:
body.body = decompose_stmts + body.body
func_def = FuncDef(n.name, args, body, func_type)
if func_type is not None:
func_type.definition = func_def
func_type.line = n.lineno
if n.decorator_list:
var = Var(func_def.name())
var.is_ready = False
var.set_line(n.decorator_list[0].lineno)
func_def.is_decorated = True
func_def.set_line(n.lineno + len(n.decorator_list))
func_def.body.set_line(func_def.get_line())
return Decorator(func_def,
self.translate_expr_list(n.decorator_list), var)
else:
return func_def
def to_var(self) -> Var:
return Var(self.name, self.type)
def transform_type_info(context: mypy.plugin.ClassDefContext,
proto_type_node: TypeInfo,
attribute_type_node: TypeInfo, init: bool) -> TypeInfo:
transformed_info: TypeInfo = context.cls.info
# Get the list of proto_type class members that are not dunders or private
names = [
name for name in proto_type_node.names if not name.startswith('_')
]
transformedNames = []
for name in names:
proto_node = proto_type_node.names[name] # SymbolTableNode
if isinstance(proto_node.node, FuncDef):
# Ignore methods
continue
transformedNames.append(name)
copied_node = proto_node.copy() # SymbolTableNode
copied_node.node = copy.copy(proto_node.node) # Var
copied_node.node._fullname = "{}.{}".format(transformed_info.fullname,
copied_node.node.name)
copied_node.plugin_generated = True
try:
nestedTypeInfo = typeInfoByProtoTypeName.get(
proto_node.node.type.type.fullname, None)
except AttributeError:
nestedTypeInfo = None
if isinstance(proto_node.node.type, mypy.types.AnyType):
# AnyType is not a transformable class anyway.
pass
else:
print("Warning: Failed to check fullname of {}: {}".format(
name, proto_node.node.type))
if nestedTypeInfo is not None:
# This member's type has been transformed.
copied_node.node.type = \
mypy.types.Instance(nestedTypeInfo, [])
else:
if attribute_type_node.is_generic():
typeArgs = [proto_node.node.type]
else:
typeArgs = []
copied_node.node.type = \
mypy.types.Instance(attribute_type_node, typeArgs)
transformed_info.names[name] = copied_node
protoTypeInstance = mypy.types.Instance(proto_type_node, [])
if init:
argument = Argument(
Var(proto_type_node.name.lower(), protoTypeInstance),
protoTypeInstance, None, ARG_POS)
add_method_to_class(context.api, context.cls, "__init__", [
argument,
], mypy.types.NoneType())
add_method_to_class(context.api, context.cls, "GetProtoType", [],
protoTypeInstance)
# Now that the class is built, update the info
for name in transformedNames:
transformed_info.names[name].node.info = transformed_info
return transformed_info
def build_namedtuple_typeinfo(self, name: str, items: List[str],
types: List[Type],
default_items: Mapping[str, Expression],
line: int) -> TypeInfo:
strtype = self.api.named_type('__builtins__.str')
implicit_any = AnyType(TypeOfAny.special_form)
basetuple_type = self.api.named_type('__builtins__.tuple',
[implicit_any])
dictype = (self.api.named_type_or_none('builtins.dict',
[strtype, implicit_any])
or self.api.named_type('__builtins__.object'))
# Actual signature should return OrderedDict[str, Union[types]]
ordereddictype = (self.api.named_type_or_none('builtins.dict',
[strtype, implicit_any])
or self.api.named_type('__builtins__.object'))
fallback = self.api.named_type('__builtins__.tuple', [implicit_any])
# Note: actual signature should accept an invariant version of Iterable[UnionType[types]].
# but it can't be expressed. 'new' and 'len' should be callable types.
iterable_type = self.api.named_type_or_none('typing.Iterable',
[implicit_any])
function_type = self.api.named_type('__builtins__.function')
info = self.api.basic_new_typeinfo(name, fallback, line)
info.is_named_tuple = True
tuple_base = TupleType(types, fallback)
info.tuple_type = tuple_base
info.line = line
# For use by mypyc.
info.metadata['namedtuple'] = {'fields': items.copy()}
# We can't calculate the complete fallback type until after semantic
# analysis, since otherwise base classes might be incomplete. Postpone a
# callback function that patches the fallback.
self.api.schedule_patch(PRIORITY_FALLBACKS,
lambda: calculate_tuple_fallback(tuple_base))
def add_field(var: Var,
is_initialized_in_class: bool = False,
is_property: bool = False) -> None:
var.info = info
var.is_initialized_in_class = is_initialized_in_class
var.is_property = is_property
var._fullname = '%s.%s' % (info.fullname, var.name)
info.names[var.name] = SymbolTableNode(MDEF, var)
fields = [Var(item, typ) for item, typ in zip(items, types)]
for var in fields:
add_field(var, is_property=True)
# We can't share Vars between fields and method arguments, since they
# have different full names (the latter are normally used as local variables
# in functions, so their full names are set to short names when generated methods
# are analyzed).
vars = [Var(item, typ) for item, typ in zip(items, types)]
tuple_of_strings = TupleType([strtype for _ in items], basetuple_type)
add_field(Var('_fields', tuple_of_strings),
is_initialized_in_class=True)
add_field(Var('_field_types', dictype), is_initialized_in_class=True)
add_field(Var('_field_defaults', dictype),
is_initialized_in_class=True)
add_field(Var('_source', strtype), is_initialized_in_class=True)
add_field(Var('__annotations__', ordereddictype),
is_initialized_in_class=True)
add_field(Var('__doc__', strtype), is_initialized_in_class=True)
tvd = TypeVarDef(SELF_TVAR_NAME, info.fullname + '.' + SELF_TVAR_NAME,
-1, [], info.tuple_type)
selftype = TypeVarType(tvd)
def add_method(
funcname: str,
ret: Type,
args: List[Argument],
is_classmethod: bool = False,
is_new: bool = False,
) -> None:
if is_classmethod or is_new:
first = [
Argument(Var('_cls'), TypeType.make_normalized(selftype),
None, ARG_POS)
]
else:
first = [Argument(Var('_self'), selftype, None, ARG_POS)]
args = first + args
types = [arg.type_annotation for arg in args]
items = [arg.variable.name for arg in args]
arg_kinds = [arg.kind for arg in args]
assert None not in types
signature = CallableType(cast(List[Type], types), arg_kinds, items,
ret, function_type)
signature.variables = [tvd]
func = FuncDef(funcname, args, Block([]))
func.info = info
func.is_class = is_classmethod
func.type = set_callable_name(signature, func)
func._fullname = info.fullname + '.' + funcname
func.line = line
if is_classmethod:
v = Var(funcname, func.type)
v.is_classmethod = True
v.info = info
v._fullname = func._fullname
func.is_decorated = True
dec = Decorator(func, [NameExpr('classmethod')], v)
dec.line = line
sym = SymbolTableNode(MDEF, dec)
else:
sym = SymbolTableNode(MDEF, func)
sym.plugin_generated = True
info.names[funcname] = sym
add_method('_replace',
ret=selftype,
args=[
Argument(var, var.type, EllipsisExpr(), ARG_NAMED_OPT)
for var in vars
])
def make_init_arg(var: Var) -> Argument:
default = default_items.get(var.name, None)
kind = ARG_POS if default is None else ARG_OPT
return Argument(var, var.type, default, kind)
add_method('__new__',
ret=selftype,
args=[make_init_arg(var) for var in vars],
is_new=True)
add_method('_asdict', args=[], ret=ordereddictype)
special_form_any = AnyType(TypeOfAny.special_form)
add_method('_make',
ret=selftype,
is_classmethod=True,
args=[
Argument(Var('iterable', iterable_type), iterable_type,
None, ARG_POS),
Argument(Var('new'), special_form_any, EllipsisExpr(),
ARG_NAMED_OPT),
Argument(Var('len'), special_form_any, EllipsisExpr(),
ARG_NAMED_OPT)
])
self_tvar_expr = TypeVarExpr(SELF_TVAR_NAME,
info.fullname + '.' + SELF_TVAR_NAME, [],
info.tuple_type)
info.names[SELF_TVAR_NAME] = SymbolTableNode(MDEF, self_tvar_expr)
return info
