def test_getitem__concrete_index(self):
t = self._ctx.convert.tuple_to_value((self._var, ))
index = self._convert.constant_to_var(0)
node, var = t.cls.getitem_slot(self._node, index)
self.assertIs(node, self._node)
self.assertIs(abstract_utils.get_atomic_value(var),
abstract_utils.get_atomic_value(self._var))
def make_class(self, node, bases, f_locals):
# If BuildClass.call() hits max depth, f_locals will be [unsolvable]
# Since we don't support defining NamedTuple subclasses in a nested scope
# anyway, we can just return unsolvable here to prevent a crash, and let the
# invalid namedtuple error get raised later.
if isinstance(f_locals.data[0], abstract.Unsolvable):
return node, self.ctx.new_unsolvable(node)
f_locals = abstract_utils.get_atomic_python_constant(f_locals)
# retrieve __qualname__ to get the name of class
name = f_locals["__qualname__"]
nameval = abstract_utils.get_atomic_python_constant(name)
if "." in nameval:
nameval = nameval.rsplit(".", 1)[-1]
name = self.ctx.convert.constant_to_var(nameval)
# assemble the arguments that are compatible with NamedTupleFuncBuilder.call
field_list = []
defaults = []
cls_locals = classgen.get_class_locals(
nameval,
allow_methods=True,
ordering=classgen.Ordering.FIRST_ANNOTATE,
ctx=self.ctx)
for k, local in cls_locals.items():
assert local.typ
if k in f_locals:
defaults.append(f_locals[k])
k = self.ctx.convert.constant_to_var(k, node=node)
field_list.append(self.ctx.convert.build_tuple(node, (k, local.typ)))
anno = self.ctx.convert.build_list(node, field_list)
posargs = (name, anno)
args = function.Args(posargs=posargs)
node, cls_var = self.namedtuple.call(node, None, args, bases)
cls_val = abstract_utils.get_atomic_value(cls_var)
if not isinstance(cls_val, abstract.Unsolvable):
# set __new__.__defaults__
defaults = self.ctx.convert.build_tuple(node, defaults)
node, new_attr = self.ctx.attribute_handler.get_attribute(
node, cls_val, "__new__")
new_attr = abstract_utils.get_atomic_value(new_attr)
node = self.ctx.attribute_handler.set_attribute(node, new_attr,
"__defaults__", defaults)
# set the attribute without overriding special namedtuple attributes
node, fields = self.ctx.attribute_handler.get_attribute(
node, cls_val, "_fields")
fields = abstract_utils.get_atomic_python_constant(fields, tuple)
fields = [abstract_utils.get_atomic_python_constant(field, str)
for field in fields]
for key in f_locals:
if key in self._prohibited:
self.ctx.errorlog.not_writable(self.ctx.vm.frames, cls_val, key)
if key not in abstract_utils.CLASS_LEVEL_IGNORE and key not in fields:
node = self.ctx.attribute_handler.set_attribute(
node, cls_val, key, f_locals[key])
return node, cls_var
def test_instantiate_interpreter_class(self):
cls = abstract.InterpreterClass("X", [], {}, None, self._ctx)
# When there is no current frame, create a new instance every time.
v1 = abstract_utils.get_atomic_value(cls.instantiate(self._node))
v2 = abstract_utils.get_atomic_value(cls.instantiate(self._node))
self.assertIsNot(v1, v2)
# Create one instance per opcode.
fake_opcode = object()
self._ctx.vm.push_frame(frame_state.SimpleFrame(fake_opcode))
v3 = abstract_utils.get_atomic_value(cls.instantiate(self._node))
v4 = abstract_utils.get_atomic_value(cls.instantiate(self._node))
self.assertIsNot(v1, v3)
self.assertIsNot(v2, v3)
self.assertIs(v3, v4)
def generator():
"""Yields mutations."""
if (not (self.is_attribute_of_class or self.name == "__new__")
or not first_arg or not substs):
return
try:
inst = abstract_utils.get_atomic_value(first_arg,
_instance_base.Instance)
except abstract_utils.ConversionError:
return
if inst.cls.template:
for subst in substs:
for k, v in subst.items():
if k in inst.instance_type_parameters:
value = inst.instance_type_parameters[
k].AssignToNewVariable(node)
if all(
isinstance(val, _singletons.Unknown)
for val in v.data):
for param in inst.cls.template:
if subst.same_name(k, param.full_name):
value.PasteVariable(
param.instantiate(node), node)
break
else:
# See GenericFeatureTest.test_reinherit_generic in
# tests/test_generic2. This can happen if one generic class
# inherits from another and separately reuses a TypeVar.
value.PasteVariable(v, node)
else:
value.PasteVariable(v, node)
yield function.Mutation(inst, k, value)
def getitem_slot(self, node, index_var):
"""Implementation of tuple.__getitem__."""
try:
index = self.ctx.convert.value_to_constant(
abstract_utils.get_atomic_value(index_var), (int, slice))
except abstract_utils.ConversionError:
pass
else:
if isinstance(index, slice):
if self._instance:
slice_content = self._instance.pyval[index]
return node, self.ctx.convert.build_tuple(
node, slice_content)
else:
# Constructing the tuple directly is faster than calling call_pytd.
instance = _instance_base.Instance(
self.ctx.convert.tuple_type, self.ctx)
node, contained_type = self.ctx.vm.init_class(
node, self.formal_type_parameters[abstract_utils.T])
instance.merge_instance_type_parameter(
node, abstract_utils.T, contained_type)
return node, instance.to_variable(node)
if -self.tuple_length <= index < self.tuple_length:
# Index out of bounds is not a pytype error because of the high
# likelihood of false positives, e.g.,
# tup = []
# idx = 0
# if idx < len(tup):
# tup[idx]
return node, self._instantiate_index(node, index)
return self.call_pytd(node, "__getitem__", self.instantiate(node),
index_var)
def get_decorated_method(name, value, func_slot):
fvar = getattr(value, func_slot)
func = abstract_utils.get_atomic_value(fvar, abstract.Function)
defn = self.value_to_pytd_def(node, func, name)
defn = defn.Visit(visitors.DropMutableParameters())
defn = add_final(defn, value)
return defn
def __init__(self, callself, underlying):
super().__init__(underlying.name, underlying.ctx)
self.cls = underlying.cls
self._callself = callself
self.underlying = underlying
self.is_attribute_of_class = False
self.is_class_builder = False
# If the function belongs to `ParameterizedClass`, we will annotate the
# `self` when do argument matching
self.replace_self_annot = None
inst = abstract_utils.get_atomic_value(
self._callself, default=self.ctx.convert.unsolvable)
if self._should_replace_self_annot():
if (isinstance(inst.cls, class_mixin.Class)
and inst.cls.full_name != "builtins.type"):
for cls in inst.cls.mro:
if isinstance(cls, _classes.ParameterizedClass):
base_cls = cls.base_cls
else:
base_cls = cls
if isinstance(base_cls,
class_mixin.Class) and base_cls.template:
self.replace_self_annot = (
_classes.ParameterizedClass.
get_generic_instance_type(base_cls))
break
if isinstance(inst, _instance_base.SimpleValue):
self.alias_map = inst.instance_type_parameters.uf
elif isinstance(inst, _typing.TypeParameterInstance):
self.alias_map = inst.instance.instance_type_parameters.uf
else:
self.alias_map = None
def call(self, node, unused_func, args):
"""Adds a metaclass."""
self.match_args(node, args)
meta = abstract_utils.get_atomic_value(
args.posargs[0], default=self.ctx.convert.unsolvable)
return node, AddMetaclassInstance(meta, self.ctx,
self.module_name).to_variable(node)
def _inner_cls_check(self, last_frame):
"""Check if the function and its nested class use same type parameter."""
# get all type parameters from function annotations
all_type_parameters = []
for annot in self.signature.annotations.values():
params = self.ctx.annotation_utils.get_type_parameters(annot)
all_type_parameters.extend(itm.with_module(None) for itm in params)
if all_type_parameters:
for key, value in last_frame.f_locals.pyval.items():
value = abstract_utils.get_atomic_value(
value, default=self.ctx.convert.unsolvable)
if (isinstance(value, _classes.InterpreterClass)
and value.template and key == value.name):
# `value` is a nested class definition.
inner_cls_types = value.collect_inner_cls_types()
inner_cls_types.update([(value, item.with_module(None))
for item in value.template])
# Report errors in a deterministic order.
for cls, item in sorted(inner_cls_types,
key=lambda typ: typ[1].name):
if item in all_type_parameters:
self.ctx.errorlog.invalid_annotation(
self.ctx.vm.simple_stack(
self.get_first_opcode()), item,
("Function [%s] and its nested generic class [%s] cannot use "
"the same type variable %s") %
(self.full_name, cls.full_name, item.name))
def call(self, node, func, args):
args = args.simplify(node, self.ctx)
self.match_args(node, args, match_all_views=True)
# As long as the types match we do not really care about the actual
# class name. But, if we have a string literal value as the name arg,
# we will use it.
name_arg = args.namedargs.get(self._name_arg_name) or args.posargs[0]
try:
_ = abstract_utils.get_atomic_python_constant(name_arg, str)
except abstract_utils.ConversionError:
name_arg = self.ctx.convert.constant_to_var(
f"_NewType_Internal_Class_Name_{self.internal_name_counter}_")
type_arg = args.namedargs.get(self._type_arg_name) or args.posargs[1]
try:
type_value = abstract_utils.get_atomic_value(type_arg)
except abstract_utils.ConversionError:
# We need the type arg to be an atomic value. If not, we just
# silently return unsolvable.
return node, self.ctx.new_unsolvable(node)
value_arg_name = "val"
constructor = overlay_utils.make_method(
self.ctx,
node,
name="__init__",
params=[Param(value_arg_name, type_value)])
members = abstract.Dict(self.ctx)
members.set_str_item(node, "__init__", constructor)
return self.ctx.make_class(node, name_arg, (type_arg, ),
members.to_variable(node), None)
def test_call_wrong_argcount(self):
self._ctx.vm.push_frame(frame_state.SimpleFrame())
node, result = self._is_instance.call(
self._node, None, function.Args((), self.new_dict(), None, None))
self.assertEqual(self._node, node)
self.assertIsInstance(abstract_utils.get_atomic_value(result),
abstract.Unsolvable)
self.assertRegex(str(self._ctx.errorlog), "missing-parameter")
def call(self, node, unused_func, args):
"""Creates an anonymous class to act as a metaclass."""
self.match_args(node, args)
meta = abstract_utils.get_atomic_value(
args.posargs[0], default=self.ctx.convert.unsolvable)
bases = args.posargs[1:]
result = WithMetaclassInstance(self.ctx, meta, bases).to_variable(node)
return node, result
def _eval_expr_as_tuple(self, node, expr, stack):
"""Evaluate an expression as a tuple."""
if not expr:
return (), None
f_globals = self.ctx.vm.frame.f_globals
f_locals = self.ctx.vm.frame.f_locals
with self.ctx.vm.generate_late_annotations(stack):
result_var, errorlog = abstract_utils.eval_expr(self.ctx, node, f_globals,
f_locals, expr)
result = abstract_utils.get_atomic_value(result_var)
# If the result is a tuple, expand it.
if (isinstance(result, mixin.PythonConstant) and
isinstance(result.pyval, tuple)):
return (tuple(abstract_utils.get_atomic_value(x) for x in result.pyval),
errorlog)
else:
return (result,), errorlog
def get_inner_classes(self):
"""Return the list of top-level nested classes."""
values = [
abstract_utils.get_atomic_value(
mbr, default=self.ctx.convert.unsolvable)
for mbr in self.members.values()
]
return [
x for x in values if isinstance(x, InterpreterClass) and x != self
]
def test_call_wrong_keywords(self):
self._ctx.vm.push_frame(frame_state.SimpleFrame())
x = self.new_var(abstract.Unknown(self._ctx))
node, result = self._is_instance.call(
self._node, None, function.Args(
(x, x), self.new_dict(foo=x), None, None))
self.assertEqual(self._node, node)
self.assertIsInstance(abstract_utils.get_atomic_value(result),
abstract.Unsolvable)
self.assertRegex(
str(self._ctx.errorlog), r"foo.*isinstance.*\[wrong-keyword-args\]")
def to_annotation_container(self):
if _isinstance(self,
"PyTDClass") and self.full_name == "builtins.tuple":
# If we are parameterizing builtins.tuple, replace it with typing.Tuple so
# that heterogeneous tuple annotations work. We need the isinstance()
# check to distinguish PyTDClass(tuple) from ParameterizedClass(tuple);
# the latter appears here when a generic type alias is being substituted.
typing = self.ctx.vm.import_module("typing", "typing",
0).get_module("Tuple")
typing.load_lazy_attribute("Tuple")
return abstract_utils.get_atomic_value(typing.members["Tuple"])
return _make("AnnotationContainer", self.name, self.ctx, self)
def collect_inner_cls_types(self, max_depth=5):
"""Collect all the type parameters from nested classes."""
templates = set()
if max_depth > 0:
for mbr in self.members.values():
mbr = abstract_utils.get_atomic_value(
mbr, default=self.ctx.convert.unsolvable)
if isinstance(mbr, InterpreterClass) and mbr.template:
templates.update([(mbr, item.with_module(None))
for item in mbr.template])
templates.update(mbr.collect_inner_cls_types(max_depth -
1))
return templates
def _check_str_key_value(self, node, name, value_var):
if not self._check_str_key(name):
return
typ = abstract_utils.get_atomic_value(self.fields[name])
bad = self.ctx.matcher(node).bad_matches(value_var, typ)
for view, error_details in bad:
binding = view[value_var]
self.ctx.errorlog.annotation_type_mismatch(self.ctx.vm.frames,
typ,
binding,
name,
error_details,
typed_dict=self)
def set_class(self, node, var):
"""Set the __class__ of an instance, for code that does "x.__class__ = y."""
# Simplification: Setting __class__ is done rarely, and supporting this
# action would complicate pytype considerably by forcing us to track the
# class in a variable, so we instead fall back to Any.
try:
new_cls = abstract_utils.get_atomic_value(var)
except abstract_utils.ConversionError:
self.cls = self.ctx.convert.unsolvable
else:
if self.cls != new_cls:
self.cls = self.ctx.convert.unsolvable
return node
def _get_scopes(
state, names: Sequence[str], ctx,
) -> Sequence[Union[abstract.InterpreterClass, abstract.InterpreterFunction]]:
"""Gets the class or function objects for a sequence of nested scope names.
For example, if the code under analysis is:
class Foo:
def f(self):
def g(): ...
then when called with ['Foo', 'f', 'g'], this method returns
[InterpreterClass(Foo), InterpreterFunction(f), InterpreterFunction(g)].
Arguments:
state: The current state.
names: A sequence of names for consecutive nested scopes in the module
under analysis. Must start with a module-level name.
ctx: The current context.
Returns:
The class or function object corresponding to each name in 'names'.
"""
scopes = []
for name in names:
prev = scopes[-1] if scopes else None
if not prev:
try:
_, var = ctx.vm.load_global(state, name)
except KeyError:
break
elif isinstance(prev, abstract.InterpreterClass):
if name in prev.members:
var = prev.members[name]
else:
break
else:
assert isinstance(prev, abstract.InterpreterFunction)
# For last_frame to be populated, 'prev' has to have been called at
# least once. This has to be true for all functions except the innermost
# one, since pytype cannot detect a nested function without analyzing
# the code that defines the nested function.
if prev.last_frame and name in prev.last_frame.f_locals.pyval:
var = prev.last_frame.f_locals.pyval[name]
else:
break
try:
scopes.append(abstract_utils.get_atomic_value(
var, (abstract.InterpreterClass, abstract.InterpreterFunction)))
except abstract_utils.ConversionError:
break
return scopes
def _update_signature_scope(self):
# If this is a nested function in an instance method and the nested function
# accesses 'self', then the first variable in the closure is 'self'. We use
# 'self' to update the scopes of any type parameters in the nested method's
# signature to the containing class.
if not self.closure:
return
maybe_instance = self.closure[0]
try:
instance = abstract_utils.get_atomic_value(maybe_instance,
_instance_base.Instance)
except abstract_utils.ConversionError:
return
if isinstance(instance.cls, _classes.InterpreterClass):
instance.cls.update_signature_scope(self)
def _load_all_formal_type_parameters(self):
"""Load _all_formal_type_parameters."""
if self._all_formal_type_parameters_loaded:
return
bases = [
abstract_utils.get_atomic_value(
base, default=self.ctx.convert.unsolvable)
for base in self.bases()
]
for base in bases:
abstract_utils.parse_formal_type_parameters(
base, self.full_name, self._all_formal_type_parameters)
self._all_formal_type_parameters_loaded = True
def test_callable_no_args(self):
ast = self._load_ast("a", """
from typing import Callable
x = ... # type: Callable[[], ...]
""")
x = ast.Lookup("a.x").type
cls = self._ctx.convert.constant_to_value(x, {}, self._ctx.root_node)
instance = self._ctx.convert.constant_to_value(
abstract_utils.AsInstance(x), {}, self._ctx.root_node)
self.assertIsInstance(
cls.get_formal_type_parameter(abstract_utils.ARGS), abstract.Empty)
self.assertEqual(
abstract_utils.get_atomic_value(
instance.get_instance_type_parameter(abstract_utils.ARGS)),
self._ctx.convert.empty)
def property_get(self, callself, is_class=False):
if self.kind == pytd.MethodKind.STATICMETHOD:
if is_class:
# Binding the function to None rather than not binding it tells
# output.py to infer the type as a Callable rather than reproducing the
# signature, including the @staticmethod decorator, which is
# undesirable for module-level aliases.
callself = None
return _function_base.StaticMethod(self.name, self, callself, self.ctx)
elif self.kind == pytd.MethodKind.CLASSMETHOD:
if not is_class:
callself = abstract_utils.get_atomic_value(
callself, default=self.ctx.convert.unsolvable)
if isinstance(callself, _typing.TypeParameterInstance):
callself = abstract_utils.get_atomic_value(
callself.instance.get_instance_type_parameter(callself.name),
default=self.ctx.convert.unsolvable)
# callself is the instance, and we want to bind to its class.
callself = callself.cls.to_variable(self.ctx.root_node)
return _function_base.ClassMethod(self.name, self, callself, self.ctx)
elif self.kind == pytd.MethodKind.PROPERTY and not is_class:
return _function_base.Property(self.name, self, callself, self.ctx)
else:
return super().property_get(callself, is_class)
def _get_template(val: Any):
"""Get the value's class template."""
if _isinstance(val, "Class"):
res = {t.full_name for t in val.template}
if _isinstance(val, "ParameterizedClass"):
res.update(_get_template(val.base_cls))
elif _isinstance(val, ("PyTDClass", "InterpreterClass")):
for base in val.bases():
base = abstract_utils.get_atomic_value(
base, default=val.ctx.convert.unsolvable)
res.update(_get_template(base))
return res
elif val.cls != val:
return _get_template(val.cls)
else:
return set()
def _getargs(self, node, args, functional):
self.match_args(node, args)
sig, = self.signatures
callargs = {name: var for name, var, _ in sig.signature.iter_args(args)}
# typing.NamedTuple doesn't support rename or verbose
name_var = callargs["typename"]
fields_var = callargs["fields"]
fields = abstract_utils.get_atomic_python_constant(fields_var)
if isinstance(fields, str):
# Since str matches Sequence, we have to manually check for it.
raise function.WrongArgTypes(sig.signature, args, self.ctx,
self._fields_param)
# The fields is a list of tuples, so we need to deeply unwrap them.
fields = [abstract_utils.get_atomic_python_constant(t) for t in fields]
# We need the actual string for the field names and the BaseValue
# for the field types.
names = []
types = []
for field in fields:
if isinstance(field, str):
# Since str matches Sequence, we have to manually check for it.
raise function.WrongArgTypes(sig.signature, args, self.ctx,
self._fields_param)
if (len(field) != 2 or
any(not self._is_str_instance(v) for v in field[0].data)):
# Note that we don't need to check field[1] because both 'str'
# (forward reference) and 'type' are valid for it.
raise function.WrongArgTypes(sig.signature, args, self.ctx,
self._fields_param)
name, typ = field
name_py_constant = abstract_utils.get_atomic_python_constant(name)
names.append(name_py_constant)
if functional:
allowed_type_params = (
self.ctx.annotation_utils.get_callable_type_parameter_names(typ))
annot = self.ctx.annotation_utils.extract_annotation(
node,
typ,
name_py_constant,
self.ctx.vm.simple_stack(),
allowed_type_params=allowed_type_params)
else:
# This NamedTuple was constructed with the class syntax. The field
# annotations were already processed when added to __annotations__.
annot = abstract_utils.get_atomic_value(typ)
types.append(annot)
return name_var, names, types
def _make_init(self, props):
# __init__ method for type checking signatures.
# We construct this here and pass it to TypedDictClass because we need
# access to abstract.SignedFunction.
sig = function.Signature.from_param_names(
f"{props.name}.__init__",
props.fields.keys(),
kind=pytd.ParameterKind.KWONLY)
sig.annotations = {
k: abstract_utils.get_atomic_value(v)
for k, v in props.fields.items()
}
sig.defaults = {
k: self.ctx.new_unsolvable(self.ctx.root_node)
for k in props.optional
}
return abstract.SignedFunction(sig, self.ctx)
def update_method_type_params(self):
if self.template:
# For function type parameters check
for mbr in self.members.values():
m = abstract_utils.get_atomic_value(
mbr, default=self.ctx.convert.unsolvable)
if _isinstance(m, "SignedFunction"):
self.update_signature_scope(m)
elif mbr.data and all(
x.__class__.__name__ == "PropertyInstance"
for x in mbr.data):
# We generate a new variable every time we add a property slot, so we
# take the last one (which contains bindings for all defined slots).
prop = mbr.data[-1]
for slot in (prop.fget, prop.fset, prop.fdel):
if slot:
for d in slot.data:
if _isinstance(d, "SignedFunction"):
self.update_signature_scope(d)
def extract_annotation(
self, node, var, name, stack, allowed_type_params=None):
"""Returns an annotation extracted from 'var'.
Args:
node: The current node.
var: The variable to extract from.
name: The annotated name.
stack: The frame stack.
allowed_type_params: Type parameters that are allowed to appear in the
annotation. 'None' means all are allowed.
"""
try:
typ = abstract_utils.get_atomic_value(var)
except abstract_utils.ConversionError:
self.ctx.errorlog.ambiguous_annotation(self.ctx.vm.frames, None, name)
return self.ctx.convert.unsolvable
typ = self._process_one_annotation(node, typ, name, stack)
if not typ:
return self.ctx.convert.unsolvable
if typ.formal and allowed_type_params is not None:
illegal_params = [x.name for x in self.get_type_parameters(typ)
if x.name not in allowed_type_params]
if illegal_params:
details = "TypeVar(s) %s not in scope" % ", ".join(
repr(p) for p in utils.unique_list(illegal_params))
if self.ctx.vm.frame.func:
method = self.ctx.vm.frame.func.data
if isinstance(method, abstract.BoundFunction):
desc = "class"
frame_name = method.name.rsplit(".", 1)[0]
else:
desc = "class" if method.is_class_builder else "method"
frame_name = method.name
details += f" for {desc} {frame_name!r}"
if "AnyStr" in illegal_params:
str_type = "Union[str, bytes]"
details += (f"\nNote: For all string types, use {str_type}.")
self.ctx.errorlog.invalid_annotation(stack, typ, details, name)
return self.ctx.convert.unsolvable
return typ
def _constant_to_value(self, pyval, subst, get_node):
"""Create a BaseValue that represents a python constant.
This supports both constant from code constant pools and PyTD constants such
as classes. This also supports builtin python objects such as int and float.
Args:
pyval: The python or PyTD value to convert.
subst: The current type parameters.
get_node: A getter function for the current node.
Returns:
A Value that represents the constant, or None if we couldn't convert.
Raises:
NotImplementedError: If we don't know how to convert a value.
TypeParameterError: If we can't find a substitution for a type parameter.
"""
if isinstance(pyval, str):
return abstract.ConcreteValue(pyval, self.str_type, self.ctx)
elif isinstance(pyval, bytes):
return abstract.ConcreteValue(pyval, self.bytes_type, self.ctx)
elif isinstance(pyval, bool):
return self.true if pyval else self.false
elif isinstance(pyval, int) and -1 <= pyval <= _MAX_IMPORT_DEPTH:
# For small integers, preserve the actual value (for things like the
# level in IMPORT_NAME).
return abstract.ConcreteValue(pyval, self.int_type, self.ctx)
elif pyval.__class__ in self.primitive_classes:
return self.primitive_class_instances[pyval.__class__]
elif pyval.__class__ is frozenset:
instance = abstract.Instance(self.frozenset_type, self.ctx)
for element in pyval:
instance.merge_instance_type_parameter(
self.ctx.root_node, abstract_utils.T,
self.constant_to_var(element, subst, self.ctx.root_node))
return instance
elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)):
return abstract.ConcreteValue(pyval,
self.primitive_classes[types.CodeType],
self.ctx)
elif pyval is super:
return special_builtins.Super(self.ctx)
elif pyval is object:
return special_builtins.Object(self.ctx)
elif pyval.__class__ is type:
try:
return self.name_to_value(self._type_to_name(pyval), subst)
except (KeyError, AttributeError):
log.debug("Failed to find pytd", exc_info=True)
raise
elif isinstance(pyval, pytd.LateType):
actual = self._load_late_type(pyval)
return self._constant_to_value(actual, subst, get_node)
elif isinstance(pyval, pytd.TypeDeclUnit):
return self._create_module(pyval)
elif isinstance(pyval, pytd.Module):
mod = self.ctx.loader.import_name(pyval.module_name)
return self._create_module(mod)
elif isinstance(pyval, pytd.Class):
if pyval.name == "builtins.super":
return self.ctx.special_builtins["super"]
elif pyval.name == "builtins.object":
return self.object_type
elif pyval.name == "types.ModuleType":
return self.module_type
elif pyval.name == "_importlib_modulespec.ModuleType":
# Python 3's typeshed uses a stub file indirection to define ModuleType
# even though it is exported via types.pyi.
return self.module_type
elif pyval.name == "types.FunctionType":
return self.function_type
else:
module, dot, base_name = pyval.name.rpartition(".")
# typing.TypingContainer intentionally loads the underlying pytd types.
if (module not in ("typing", "typing_extensions") and
module in overlay_dict.overlays):
overlay = self.ctx.vm.import_module(module, module, 0)
if overlay.get_module(base_name) is overlay:
overlay.load_lazy_attribute(base_name)
return abstract_utils.get_atomic_value(overlay.members[base_name])
try:
cls = abstract.PyTDClass.make(base_name, pyval, self.ctx)
except mro.MROError as e:
self.ctx.errorlog.mro_error(self.ctx.vm.frames, base_name, e.mro_seqs)
cls = self.unsolvable
else:
if dot:
cls.module = module
cls.call_metaclass_init(get_node())
return cls
elif isinstance(pyval, pytd.Function):
signatures = [
abstract.PyTDSignature(pyval.name, sig, self.ctx)
for sig in pyval.signatures
]
type_new = self.ctx.loader.lookup_builtin("builtins.type").Lookup(
"__new__")
if pyval is type_new:
f_cls = special_builtins.TypeNew
else:
f_cls = abstract.PyTDFunction
f = f_cls(pyval.name, signatures, pyval.kind, self.ctx)
f.is_abstract = pyval.is_abstract
return f
elif isinstance(pyval, pytd.ClassType):
if pyval.cls:
cls = pyval.cls
else:
# If pyval is a reference to a class in builtins or typing, we can fill
# in the class ourselves. lookup_builtin raises a KeyError if the name
# is not found.
cls = self.ctx.loader.lookup_builtin(pyval.name)
assert isinstance(cls, pytd.Class)
return self.constant_to_value(cls, subst, self.ctx.root_node)
elif isinstance(pyval, pytd.NothingType):
return self.empty
elif isinstance(pyval, pytd.AnythingType):
return self.unsolvable
elif (isinstance(pyval, pytd.Constant) and
isinstance(pyval.type, pytd.AnythingType)):
# We allow "X = ... # type: Any" to declare X as a type.
return self.unsolvable
elif (isinstance(pyval, pytd.Constant) and
isinstance(pyval.type, pytd.GenericType) and
pyval.type.name == "builtins.type"):
# `X: Type[other_mod.X]` is equivalent to `X = other_mod.X`.
param, = pyval.type.parameters
return self.constant_to_value(param, subst, self.ctx.root_node)
elif isinstance(pyval, pytd.UnionType):
options = [
self.constant_to_value(t, subst, self.ctx.root_node)
for t in pyval.type_list
]
if len(options) > 1:
return abstract.Union(options, self.ctx)
else:
return options[0]
elif isinstance(pyval, pytd.TypeParameter):
constraints = tuple(
self.constant_to_value(c, {}, self.ctx.root_node)
for c in pyval.constraints)
bound = (
pyval.bound and
self.constant_to_value(pyval.bound, {}, self.ctx.root_node))
return abstract.TypeParameter(
pyval.name,
self.ctx,
constraints=constraints,
bound=bound,
module=pyval.scope)
elif isinstance(pyval, abstract_utils.AsInstance):
cls = pyval.cls
if isinstance(cls, pytd.LateType):
actual = self._load_late_type(cls)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
cls = actual.cls
if isinstance(cls, pytd.ClassType):
cls = cls.cls
if isinstance(cls, pytd.GenericType) and cls.name == "typing.ClassVar":
param, = cls.parameters
return self.constant_to_value(
abstract_utils.AsInstance(param), subst, self.ctx.root_node)
elif isinstance(cls, pytd.GenericType) or (isinstance(cls, pytd.Class) and
cls.template):
# If we're converting a generic Class, need to create a new instance of
# it. See test_classes.testGenericReinstantiated.
if isinstance(cls, pytd.Class):
params = tuple(t.type_param.upper_value for t in cls.template)
cls = pytd.GenericType(base_type=pytd.ClassType(cls.name, cls),
parameters=params)
if isinstance(cls.base_type, pytd.LateType):
actual = self._load_late_type(cls.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base_cls = actual.cls
else:
base_type = cls.base_type
assert isinstance(base_type, pytd.ClassType)
base_cls = base_type.cls
assert isinstance(base_cls, pytd.Class), base_cls
if base_cls.name == "builtins.type":
c, = cls.parameters
if isinstance(c, pytd.TypeParameter):
if not subst or c.full_name not in subst:
raise self.TypeParameterError(c.full_name)
# deformalize gets rid of any unexpected TypeVars, which can appear
# if something is annotated as Type[T].
return self.ctx.annotation_utils.deformalize(
self.merge_classes(subst[c.full_name].data))
else:
return self.constant_to_value(c, subst, self.ctx.root_node)
elif isinstance(cls, pytd.TupleType):
content = tuple(self.constant_to_var(abstract_utils.AsInstance(p),
subst, get_node())
for p in cls.parameters)
return self.tuple_to_value(content)
elif isinstance(cls, pytd.CallableType):
clsval = self.constant_to_value(cls, subst, self.ctx.root_node)
return abstract.Instance(clsval, self.ctx)
else:
clsval = self.constant_to_value(base_cls, subst, self.ctx.root_node)
instance = abstract.Instance(clsval, self.ctx)
num_params = len(cls.parameters)
assert num_params <= len(base_cls.template)
for i, formal in enumerate(base_cls.template):
if i < num_params:
node = get_node()
p = self.constant_to_var(
abstract_utils.AsInstance(cls.parameters[i]), subst, node)
else:
# An omitted type parameter implies `Any`.
node = self.ctx.root_node
p = self.unsolvable.to_variable(node)
instance.merge_instance_type_parameter(node, formal.name, p)
return instance
elif isinstance(cls, pytd.Class):
assert not cls.template
# This key is also used in __init__
key = (abstract.Instance, cls)
if key not in self._convert_cache:
if cls.name in ["builtins.type", "builtins.property"]:
# An instance of "type" or of an anonymous property can be anything.
instance = self._create_new_unknown_value("type")
else:
mycls = self.constant_to_value(cls, subst, self.ctx.root_node)
instance = abstract.Instance(mycls, self.ctx)
log.info("New pytd instance for %s: %r", cls.name, instance)
self._convert_cache[key] = instance
return self._convert_cache[key]
elif isinstance(cls, pytd.Literal):
return self.constant_to_value(
self._get_literal_value(cls.value), subst, self.ctx.root_node)
else:
return self.constant_to_value(cls, subst, self.ctx.root_node)
elif (isinstance(pyval, pytd.GenericType) and
pyval.name == "typing.ClassVar"):
param, = pyval.parameters
return self.constant_to_value(param, subst, self.ctx.root_node)
elif isinstance(pyval, pytd.GenericType):
if isinstance(pyval.base_type, pytd.LateType):
actual = self._load_late_type(pyval.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base = actual.cls
else:
assert isinstance(pyval.base_type, pytd.ClassType), pyval
base = pyval.base_type.cls
assert isinstance(base, pytd.Class), base
base_cls = self.constant_to_value(base, subst, self.ctx.root_node)
if not isinstance(base_cls, abstract.Class):
# base_cls can be, e.g., an unsolvable due to an mro error.
return self.unsolvable
if isinstance(pyval, pytd.TupleType):
abstract_class = abstract.TupleClass
template = list(range(len(pyval.parameters))) + [abstract_utils.T]
combined_parameter = pytd_utils.JoinTypes(pyval.parameters)
parameters = pyval.parameters + (combined_parameter,)
elif isinstance(pyval, pytd.CallableType):
abstract_class = abstract.CallableClass
template = list(range(len(pyval.args))) + [abstract_utils.ARGS,
abstract_utils.RET]
parameters = pyval.args + (pytd_utils.JoinTypes(pyval.args), pyval.ret)
else:
abstract_class = abstract.ParameterizedClass
if pyval.name == "typing.Generic":
pyval_template = pyval.parameters
else:
pyval_template = base.template
template = tuple(t.name for t in pyval_template)
parameters = pyval.parameters
assert (pyval.name in ("typing.Generic", "typing.Protocol") or
len(parameters) <= len(template))
# Delay type parameter loading to handle recursive types.
# See the ParameterizedClass.formal_type_parameters() property.
type_parameters = abstract_utils.LazyFormalTypeParameters(
template, parameters, subst)
return abstract_class(base_cls, type_parameters, self.ctx)
elif isinstance(pyval, pytd.Literal):
value = self.constant_to_value(
self._get_literal_value(pyval.value), subst, self.ctx.root_node)
return abstract.LiteralClass(value, self.ctx)
elif isinstance(pyval, pytd.Annotated):
typ = self.constant_to_value(pyval.base_type, subst, self.ctx.root_node)
if pyval.annotations[0] == "'pytype_metadata'":
try:
md = metadata.from_string(pyval.annotations[1])
if md["tag"] == "attr.ib":
ret = attr_overlay.AttribInstance.from_metadata(
self.ctx, self.ctx.root_node, typ, md)
return ret
elif md["tag"] == "attr.s":
ret = attr_overlay.Attrs.from_metadata(self.ctx, md)
return ret
except (IndexError, ValueError, TypeError, KeyError):
details = "Wrong format for pytype_metadata."
self.ctx.errorlog.invalid_annotation(self.ctx.vm.frames,
pyval.annotations[1], details)
return typ
else:
return typ
elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node
return self.tuple_to_value([
self.constant_to_var(item, subst, self.ctx.root_node)
for i, item in enumerate(pyval)
])
else:
raise NotImplementedError("Can't convert constant %s %r" %
(type(pyval), pyval))
