def make_class(self, node, f_locals):
# If BuildClass.call() hits max depth, f_locals will be [unsolvable]
# Since we don't support defining NamedTuple subclasses in a nested scope
# anyway, we can just return unsolvable here to prevent a crash, and let the
# invalid namedtuple error get raised later.
if f_locals.data[0].isinstance_Unsolvable():
return node, self.vm.new_unsolvable(node)
f_locals = abstract_utils.get_atomic_python_constant(f_locals)
# retrieve __qualname__ to get the name of class
name = f_locals["__qualname__"]
# assemble the arguments that are compatible with NamedTupleFuncBuilder.call
field_list = []
defaults = []
cls_locals = classgen.get_class_locals(
abstract_utils.get_atomic_python_constant(name),
allow_methods=True,
ordering=classgen.Ordering.FIRST_ANNOTATE,
vm=self.vm)
for k, local in cls_locals.items():
assert local.typ
if k in f_locals:
defaults.append(f_locals[k])
k = self.vm.convert.constant_to_var(k, node=node)
field_list.append(self.vm.convert.build_tuple(
node, (k, local.typ)))
anno = self.vm.convert.build_list(node, field_list)
posargs = (name, anno)
args = function.Args(posargs=posargs)
node, cls_var = self.namedtuple.call(node, None, args)
cls_val = abstract_utils.get_atomic_value(cls_var)
if not isinstance(cls_val, abstract.Unsolvable):
# set __new__.__defaults__
defaults = abstract.Tuple(tuple(defaults),
self.vm).to_variable(node)
node, new_attr = self.vm.attribute_handler.get_attribute(
node, cls_val, "__new__")
new_attr = abstract_utils.get_atomic_value(new_attr)
node = self.vm.attribute_handler.set_attribute(
node, new_attr, "__defaults__", defaults)
# set the attribute without overriding special namedtuple attributes
node, fields = self.vm.attribute_handler.get_attribute(
node, cls_val, "_fields")
fields = abstract_utils.get_atomic_python_constant(fields, tuple)
fields = [
abstract_utils.get_atomic_python_constant(field, str)
for field in fields
]
for key in f_locals:
if key in self._prohibited:
self.vm.errorlog.not_writable(self.vm.frames, cls_val, key)
if key not in self._special and key not in fields:
node = self.vm.attribute_handler.set_attribute(
node, cls_val, key, f_locals[key])
return node, cls_var
def test_getitem__concrete_index(self):
t = abstract.Tuple((self._var, ), self._vm)
index = self._vm.convert.constant_to_var(0)
node, var = t.cls.data[0].getitem_slot(self._node, index)
self.assertIs(node, self._node)
self.assertIs(abstract.get_atomic_value(var),
abstract.get_atomic_value(self._var))
def call(self, node, _, args):
posargs = args.posargs
if isinstance(args.namedargs, dict):
namedargs = args.namedargs
else:
namedargs = self.vm.convert.value_to_constant(args.namedargs, dict)
if namedargs and self.vm.python_version < (3, 6):
errmsg = "Keyword syntax for NamedTuple is only supported in Python 3.6+"
self.vm.errorlog.invalid_namedtuple_arg(self.vm.frames,
err_msg=errmsg)
if namedargs and len(posargs) == 1:
namedargs = [
abstract.Tuple((self.vm.convert.build_string(node, k), v),
self.vm).to_variable(node)
for k, v in namedargs.items()
]
namedargs = abstract.List(namedargs, self.vm).to_variable(node)
posargs += (namedargs, )
args = function.Args(posargs)
elif namedargs:
errmsg = ("Either list of fields or keywords can be provided to "
"NamedTuple, not both")
self.vm.errorlog.invalid_namedtuple_arg(self.vm.frames,
err_msg=errmsg)
return self.namedtuple.call(node, None, args)
def test_call_with_all_args(self):
f = self._make_func(
param_names=("a", "b", "c"),
varargs_name="arg",
kwargs_name="kwarg",
defaults=(self._vm.convert.build_int(self._vm.root_cfg_node),),
annotations={
"a": self._vm.convert.str_type,
"b": self._vm.convert.int_type,
"c": self._vm.convert.int_type,
"arg": self._vm.convert.primitive_classes[float],
"kwarg": self._vm.convert.primitive_classes[bool]
}
)
posargs = (self._vm.convert.build_string(self._vm.root_cfg_node, "1"),
self._vm.convert.build_int(self._vm.root_cfg_node))
float_inst = self._vm.convert.primitive_class_instances[float]
stararg = abstract.Tuple((float_inst.to_variable(self._vm.root_cfg_node),),
self._vm).to_variable(self._vm.root_cfg_node)
namedargs = abstract.Dict(self._vm)
kwarg = abstract.Dict(self._vm)
kwarg.update(self._vm.root_cfg_node,
{"x": self._vm.convert.build_bool(self._vm.root_cfg_node),
"y": self._vm.convert.build_bool(self._vm.root_cfg_node)})
kwarg = kwarg.to_variable(self._vm.root_cfg_node)
args = function.Args(posargs, namedargs, stararg, kwarg)
f.call(self._vm.root_cfg_node, f, args)
def test_getitem__abstract_index(self):
t = abstract.Tuple((self._var, ), self._vm)
index = self._vm.convert.build_int(self._node)
node, var = t.cls.data[0].getitem_slot(self._node, index)
self.assertIs(node, self._node)
self.assertIs(abstract.get_atomic_value(var),
abstract.get_atomic_value(self._var))
def test_cmp_rel__prefix_unknown(self):
tup1 = abstract.Tuple(
(self._convert.constant_to_value(3).to_variable(self._node),
self._convert.primitive_class_instances[int].to_variable(
self._node)), self._vm)
tup2 = self._convert.constant_to_value((3, 1))
for op in (slots.LT, slots.LE, slots.EQ, slots.NE, slots.GE, slots.GT):
self.assertIsNone(compare.cmp_rel(self._vm, op, tup1, tup2))
self.assertIsNone(compare.cmp_rel(self._vm, op, tup2, tup1))
def test_to_type_with_view2(self):
# to_type(<tuple (int or str,)>, view={0: str})
param1 = self._vm.convert.primitive_class_instances[int]
param2 = self._vm.convert.primitive_class_instances[str]
param_var = param1.to_variable(self._vm.root_cfg_node)
str_binding = param_var.AddBinding(param2, [], self._vm.root_cfg_node)
instance = abstract.Tuple((param_var,), self._vm)
view = {param_var: str_binding}
pytd_type = instance.to_type(self._vm.root_cfg_node, seen=None, view=view)
self.assertEqual(pytd_type.parameters[0],
pytd.NamedType("__builtin__.str"))
def test_call_with_bad_varargs(self):
f = self._make_func(
varargs_name="arg",
annotations={"arg": self._vm.convert.str_type})
starargs = abstract.Tuple(
(self._vm.convert.build_string(self._vm.root_node, ""),
self._vm.convert.build_int(self._vm.root_node)),
self._vm).to_variable(self._vm.root_node)
args = function.Args(posargs=(), starargs=starargs)
self.assertRaises(function.WrongArgTypes, f.call, self._vm.root_node, f,
args)
def make_class(self, node, f_locals):
f_locals = abstract_utils.get_atomic_python_constant(f_locals)
# retrieve __qualname__ to get the name of class
name = f_locals["__qualname__"]
# retrieve __annotations__ to get the dict
# with key-value pair of (variable, type)
anno = f_locals.get("__annotations__", {})
if anno:
anno = abstract_utils.get_atomic_value(anno)
# assemble the arguments that are compatible with NamedTupleFuncBuilder.call
field_list = []
defaults = []
for k, v in anno.items():
if k in f_locals:
defaults.append(f_locals.get(k))
# TODO(ahxun): check if the value matches the declared type
k = self.vm.convert.constant_to_var(k, node=node)
field_list.append(self.vm.convert.build_tuple(node, (k, v)))
anno = self.vm.convert.build_list(node, field_list)
posargs = (name, anno)
args = function.Args(posargs=posargs)
node, cls_var = self.namedtuple.call(node, None, args)
cls_val = abstract_utils.get_atomic_value(cls_var)
if not isinstance(cls_val, abstract.Unsolvable):
# set __new__.__defaults__
defaults = abstract.Tuple(tuple(defaults),
self.vm).to_variable(node)
node, new_attr = self.vm.attribute_handler.get_attribute(
node, cls_val, "__new__")
new_attr = abstract_utils.get_atomic_value(new_attr)
node = self.vm.attribute_handler.set_attribute(
node, new_attr, "__defaults__", defaults)
# set the attribute without overriding special namedtuple attributes
node, fields = self.vm.attribute_handler.get_attribute(
node, cls_val, "_fields")
fields = abstract_utils.get_atomic_python_constant(fields, tuple)
fields = [
abstract_utils.get_atomic_python_constant(field, str)
for field in fields
]
for key in f_locals:
if key in self._prohibited:
self.vm.errorlog.not_writable(self.vm.frames, cls_val, key)
if key not in self._special and key not in fields:
node = self.vm.attribute_handler.set_attribute(
node, cls_val, key, f_locals[key])
return node, cls_var
def test_call_with_multiple_varargs_bindings(self):
f = self._make_func(
varargs_name="arg",
annotations={"arg": self._vm.convert.str_type})
arg = self._vm.program.NewVariable()
arg.AddBinding(self._vm.convert.primitive_class_instances[str], [],
self._vm.root_cfg_node)
arg.AddBinding(self._vm.convert.primitive_class_instances[int], [],
self._vm.root_cfg_node)
starargs = abstract.Tuple((arg,), self._vm)
starargs = starargs.to_variable(self._vm.root_cfg_node)
args = function.Args(posargs=(), starargs=starargs)
f.call(self._vm.root_cfg_node, f, args)
def test_call_with_varargs(self):
f = self._make_func(
varargs_name="arg",
annotations={"arg": self._vm.convert.str_type,
"return": self._vm.convert.str_type}
)
starargs = abstract.Tuple(
(self._vm.convert.build_string(self._vm.root_cfg_node, ""),),
self._vm).to_variable(self._vm.root_cfg_node)
args = function.Args(posargs=(), starargs=starargs)
node, ret = f.call(self._vm.root_cfg_node, f, args)
self.assertIs(node, self._vm.root_cfg_node)
self.assertIs(ret.data[0].cls, self._vm.convert.str_type)
def test_change_defaults(self):
f = self._make_func(
param_names=("a", "b", "c"),
defaults=(self._vm.convert.build_int(self._vm.root_node),))
args = function.Args(
posargs=(self._vm.convert.build_int(self._vm.root_node),
self._vm.convert.build_int(self._vm.root_node)))
f.call(self._vm.root_node, f, args)
new_defaults = abstract.Tuple((self._vm.convert.build_int(
self._vm.root_node), self._vm.convert.build_int(self._vm.root_node)),
self._vm).to_variable(self._vm.root_node)
f.set_function_defaults(self._vm.root_node, new_defaults)
f.call(self._vm.root_node, f, args)
args = function.Args(
posargs=(self._vm.convert.build_int(self._vm.root_node),))
f.call(self._vm.root_node, f, args)
def test_cmp_rel__prefix_not_equal(self):
tup1 = abstract.Tuple(
(self._convert.constant_to_value(3).to_variable(self._node),
self._convert.constant_to_value(1).to_variable(self._node),
self._convert.primitive_class_instances[int].to_variable(
self._node)), self._vm)
tup2 = self._convert.constant_to_value((4, 2))
self.assertIs(True, compare.cmp_rel(self._vm, slots.LT, tup1, tup2))
self.assertIs(False, compare.cmp_rel(self._vm, slots.LT, tup2, tup1))
self.assertIs(True, compare.cmp_rel(self._vm, slots.LE, tup1, tup2))
self.assertIs(False, compare.cmp_rel(self._vm, slots.LE, tup2, tup1))
self.assertIs(False, compare.cmp_rel(self._vm, slots.EQ, tup1, tup2))
self.assertIs(False, compare.cmp_rel(self._vm, slots.EQ, tup2, tup1))
self.assertIs(True, compare.cmp_rel(self._vm, slots.NE, tup1, tup2))
self.assertIs(True, compare.cmp_rel(self._vm, slots.NE, tup2, tup1))
self.assertIs(False, compare.cmp_rel(self._vm, slots.GE, tup1, tup2))
self.assertIs(True, compare.cmp_rel(self._vm, slots.GE, tup2, tup1))
self.assertIs(False, compare.cmp_rel(self._vm, slots.GT, tup1, tup2))
self.assertIs(True, compare.cmp_rel(self._vm, slots.GT, tup2, tup1))
def _constant_to_value(self, pyval, subst, get_node):
"""Create a AtomicAbstractValue that represents a python constant.
This supports both constant from code constant pools and PyTD constants such
as classes. This also supports builtin python objects such as int and float.
Args:
pyval: The python or PyTD value to convert.
subst: The current type parameters.
get_node: A getter function for the current node.
Returns:
A Value that represents the constant, or None if we couldn't convert.
Raises:
NotImplementedError: If we don't know how to convert a value.
TypeParameterError: If we can't find a substitution for a type parameter.
"""
if pyval.__class__ is str:
# We use a subclass of str, compat.BytesPy3, to mark Python 3
# bytestrings, which are converted to abstract bytes instances.
# compat.BytesType dispatches to this when appropriate.
return abstract.AbstractOrConcreteValue(pyval, self.str_type,
self.vm)
elif isinstance(pyval, compat.UnicodeType):
return abstract.AbstractOrConcreteValue(pyval, self.unicode_type,
self.vm)
elif isinstance(pyval, compat.BytesType):
return abstract.AbstractOrConcreteValue(pyval, self.bytes_type,
self.vm)
elif isinstance(pyval, bool):
return self.true if pyval is True else self.false
elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH:
# For small integers, preserve the actual value (for things like the
# level in IMPORT_NAME).
return abstract.AbstractOrConcreteValue(pyval, self.int_type,
self.vm)
elif isinstance(pyval, compat.LongType):
# long is aliased to int
return self.primitive_class_instances[int]
elif pyval.__class__ in self.primitive_classes:
return self.primitive_class_instances[pyval.__class__]
elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)):
return abstract.AbstractOrConcreteValue(
pyval, self.primitive_classes[types.CodeType], self.vm)
elif pyval is super:
return special_builtins.Super(self.vm)
elif pyval is object:
return special_builtins.Object(self.vm)
elif pyval.__class__ is type:
try:
return self.name_to_value(self._type_to_name(pyval), subst)
except (KeyError, AttributeError):
log.debug("Failed to find pytd", exc_info=True)
raise
elif isinstance(pyval, pytd.LateType):
actual = self._load_late_type(pyval)
return self._constant_to_value(actual, subst, get_node)
elif isinstance(pyval, pytd.TypeDeclUnit):
return self._create_module(pyval)
elif isinstance(pyval, pytd.Module):
mod = self.vm.loader.import_name(pyval.module_name)
return self._create_module(mod)
elif isinstance(pyval, pytd.Class):
if pyval.name == "__builtin__.super":
return self.vm.special_builtins["super"]
elif pyval.name == "__builtin__.object":
return self.object_type
elif pyval.name == "types.ModuleType":
return self.module_type
elif pyval.name == "_importlib_modulespec.ModuleType":
# Python 3's typeshed uses a stub file indirection to define ModuleType
# even though it is exported via types.pyi.
return self.module_type
else:
module, dot, base_name = pyval.name.rpartition(".")
try:
cls = abstract.PyTDClass(base_name, pyval, self.vm)
except mro.MROError as e:
self.vm.errorlog.mro_error(self.vm.frames, base_name,
e.mro_seqs)
cls = self.unsolvable
else:
if dot:
cls.module = module
return cls
elif isinstance(pyval, pytd.Function):
signatures = [
abstract.PyTDSignature(pyval.name, sig, self.vm)
for sig in pyval.signatures
]
type_new = self.vm.lookup_builtin("__builtin__.type").Lookup(
"__new__")
if pyval is type_new:
f_cls = special_builtins.TypeNew
else:
f_cls = abstract.PyTDFunction
f = f_cls(pyval.name, signatures, pyval.kind, self.vm)
f.is_abstract = pyval.is_abstract
return f
elif isinstance(pyval, pytd.ClassType):
assert pyval.cls
return self.constant_to_value(pyval.cls, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.NothingType):
return self.empty
elif isinstance(pyval, pytd.AnythingType):
return self.unsolvable
elif (isinstance(pyval, pytd.Constant)
and isinstance(pyval.type, pytd.AnythingType)):
# We allow "X = ... # type: Any" to declare X as a type.
return self.unsolvable
elif isinstance(pyval, pytd.FunctionType):
return self.constant_to_value(pyval.function, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.UnionType):
options = [
self.constant_to_value(t, subst, self.vm.root_cfg_node)
for t in pyval.type_list
]
if len(options) > 1:
return abstract.Union(options, self.vm)
else:
return options[0]
elif isinstance(pyval, pytd.TypeParameter):
constraints = tuple(
self.constant_to_value(c, {}, self.vm.root_cfg_node)
for c in pyval.constraints)
bound = (pyval.bound and self.constant_to_value(
pyval.bound, {}, self.vm.root_cfg_node))
return abstract.TypeParameter(pyval.name,
self.vm,
constraints=constraints,
bound=bound)
elif isinstance(pyval, abstract.AsInstance):
cls = pyval.cls
if isinstance(cls, pytd.LateType):
actual = self._load_late_type(cls)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
cls = actual.cls
if isinstance(cls, pytd.ClassType):
cls = cls.cls
if (isinstance(cls, pytd.GenericType)
and cls.base_type.name == "typing.ClassVar"):
param, = cls.parameters
return self.constant_to_value(abstract.AsInstance(param),
subst, self.vm.root_cfg_node)
elif isinstance(cls,
pytd.GenericType) or (isinstance(cls, pytd.Class)
and cls.template):
# If we're converting a generic Class, need to create a new instance of
# it. See test_classes.testGenericReinstantiated.
if isinstance(cls, pytd.Class):
params = tuple(t.type_param.upper_value
for t in cls.template)
cls = pytd.GenericType(base_type=pytd.ClassType(
cls.name, cls),
parameters=params)
if isinstance(cls.base_type, pytd.LateType):
actual = self._load_late_type(cls.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base_cls = actual.cls
else:
assert isinstance(cls.base_type, pytd.ClassType)
base_cls = cls.base_type.cls
assert isinstance(base_cls, pytd.Class), base_cls
if base_cls.name == "__builtin__.type":
c, = cls.parameters
if isinstance(c, pytd.TypeParameter):
if not subst or c.name not in subst:
raise self.TypeParameterError(c.name)
return self.merge_classes(get_node(),
subst[c.name].data)
else:
return self.constant_to_value(c, subst,
self.vm.root_cfg_node)
elif isinstance(cls, pytd.TupleType):
content = tuple(
self.constant_to_var(abstract.AsInstance(p), subst,
get_node())
for p in cls.parameters)
return abstract.Tuple(content, self.vm)
elif isinstance(cls, pytd.CallableType):
clsval = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
return abstract.Instance(clsval, self.vm)
else:
clsval = self.constant_to_value(base_cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(clsval, self.vm)
num_params = len(cls.parameters)
assert num_params <= len(base_cls.template)
for i, formal in enumerate(base_cls.template):
if i < num_params:
node = get_node()
p = self.constant_to_var(
abstract.AsInstance(cls.parameters[i]), subst,
node)
else:
# An omitted type parameter implies `Any`.
node = self.vm.root_cfg_node
p = self.unsolvable.to_variable(node)
instance.merge_type_parameter(node, formal.name, p)
return instance
elif isinstance(cls, pytd.Class):
assert not cls.template
# This key is also used in __init__
key = (abstract.Instance, cls)
if key not in self._convert_cache:
if cls.name in [
"__builtin__.type", "__builtin__.property"
]:
# An instance of "type" or of an anonymous property can be anything.
instance = self._create_new_unknown_value("type")
else:
mycls = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(mycls, self.vm)
log.info("New pytd instance for %s: %r", cls.name,
instance)
self._convert_cache[key] = instance
return self._convert_cache[key]
else:
return self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
elif (isinstance(pyval, pytd.GenericType)
and pyval.base_type.name == "typing.ClassVar"):
param, = pyval.parameters
return self.constant_to_value(param, subst, self.vm.root_cfg_node)
elif isinstance(pyval, pytd.GenericType):
if isinstance(pyval.base_type, pytd.LateType):
actual = self._load_late_type(pyval.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base = actual.cls
else:
assert isinstance(pyval.base_type, pytd.ClassType)
base = pyval.base_type.cls
assert isinstance(base, pytd.Class), base
base_cls = self.constant_to_value(base, subst,
self.vm.root_cfg_node)
if not isinstance(base_cls, abstract.Class):
# base_cls can be, e.g., an unsolvable due to an mro error.
return self.unsolvable
if isinstance(pyval, pytd.TupleType):
abstract_class = abstract.TupleClass
template = list(range(len(pyval.parameters))) + [abstract.T]
parameters = pyval.parameters + (pytd.UnionType(
pyval.parameters), )
elif isinstance(pyval, pytd.CallableType):
abstract_class = abstract.Callable
template = list(range(len(
pyval.args))) + [abstract.ARGS, abstract.RET]
parameters = pyval.args + (pytd_utils.JoinTypes(
pyval.args), pyval.ret)
else:
abstract_class = abstract.ParameterizedClass
template = tuple(t.name for t in base.template)
parameters = pyval.parameters
assert (pyval.base_type.name == "typing.Generic"
or len(parameters) <= len(template))
type_parameters = datatypes.LazyDict()
for i, name in enumerate(template):
if i < len(parameters):
type_parameters.add_lazy_item(name, self.constant_to_value,
parameters[i], subst,
self.vm.root_cfg_node)
else:
type_parameters[name] = self.unsolvable
return abstract_class(base_cls, type_parameters, self.vm)
elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node
return self.tuple_to_value([
self.constant_to_var(item, subst, self.vm.root_cfg_node)
for i, item in enumerate(pyval)
])
else:
raise NotImplementedError("Can't convert constant %s %r" %
(type(pyval), pyval))
def test_compatible_with__empty(self):
t = abstract.Tuple((), self._vm)
self.assertFalsy(t)
def tuple_to_value(self, content):
"""Create a VM tuple from the given sequence."""
content = tuple(content) # content might be a generator
value = abstract.Tuple(content, self.vm)
return value
def _constant_to_value(self, pyval, subst, get_node):
"""Create a AtomicAbstractValue that represents a python constant.
This supports both constant from code constant pools and PyTD constants such
as classes. This also supports builtin python objects such as int and float.
Args:
pyval: The python or PyTD value to convert.
subst: The current type parameters.
get_node: A getter function for the current node.
Returns:
A Value that represents the constant, or None if we couldn't convert.
Raises:
NotImplementedError: If we don't know how to convert a value.
TypeParameterError: If we can't find a substitution for a type parameter.
"""
if isinstance(pyval, str):
return abstract.AbstractOrConcreteValue(pyval, self.str_type,
self.vm)
elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH:
# For small integers, preserve the actual value (for things like the
# level in IMPORT_NAME).
return abstract.AbstractOrConcreteValue(pyval, self.int_type,
self.vm)
elif isinstance(pyval, long):
# long is aliased to int
return self.primitive_class_instances[int]
elif pyval.__class__ in self.primitive_classes:
return self.primitive_class_instances[pyval.__class__]
elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)):
return abstract.AbstractOrConcreteValue(
pyval, self.primitive_classes[types.CodeType], self.vm)
elif pyval is super:
return special_builtins.Super(self.vm)
elif pyval is object:
return special_builtins.Object(self.vm)
elif pyval.__class__ is type:
if pyval is types.FunctionType:
classname = "typing.Callable"
else:
classname = "__builtin__." + pyval.__name__
try:
return self.name_to_value(classname, subst)
except (KeyError, AttributeError):
log.debug("Failed to find pytd", exc_info=True)
raise
elif isinstance(pyval, pytd.TypeDeclUnit):
data = (pyval.constants + pyval.type_params + pyval.classes +
pyval.functions + pyval.aliases)
members = {val.name.rsplit(".")[-1]: val for val in data}
return abstract.Module(self.vm, pyval.name, members, pyval)
elif isinstance(pyval,
pytd.Class) and pyval.name == "__builtin__.super":
return self.vm.special_builtins["super"]
elif isinstance(pyval,
pytd.Class) and pyval.name == "__builtin__.object":
return self.object_type
elif isinstance(pyval,
pytd.Class) and pyval.name == "types.ModuleType":
return self.module_type
elif isinstance(pyval, pytd.Class):
module, dot, base_name = pyval.name.rpartition(".")
try:
cls = abstract.PyTDClass(base_name, pyval, self.vm)
except mro.MROError as e:
self.vm.errorlog.mro_error(self.vm.frames, base_name,
e.mro_seqs)
cls = self.unsolvable
else:
if dot:
cls.module = module
return cls
elif isinstance(pyval, pytd.ExternalFunction):
module, _, name = pyval.name.partition(".")
assert module == "__builtin__", "PYTHONCODE allowed only in __builtin__"
return abstract.merge_values(
self.vm.frame.f_globals.members[name].data, self.vm)
elif isinstance(pyval, pytd.Function):
signatures = [
abstract.PyTDSignature(pyval.name, sig, self.vm)
for sig in pyval.signatures
]
type_new = self.vm.lookup_builtin("__builtin__.type").Lookup(
"__new__")
if pyval is type_new:
f_cls = special_builtins.TypeNew
else:
f_cls = abstract.PyTDFunction
f = f_cls(pyval.name, signatures, pyval.kind, self.vm)
f.is_abstract = pyval.is_abstract
return f
elif isinstance(pyval, pytd.ClassType):
assert pyval.cls
return self.constant_to_value(pyval.cls, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.NothingType):
return self.nothing
elif isinstance(pyval, pytd.AnythingType):
return self.unsolvable
elif isinstance(pyval, pytd.FunctionType):
return self.constant_to_value(pyval.function, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.UnionType):
options = [
self.constant_to_value(t, subst, self.vm.root_cfg_node)
for t in pyval.type_list
]
if len(options) > 1:
return abstract.Union(options, self.vm)
else:
return options[0]
elif isinstance(pyval, pytd.TypeParameter):
constraints = tuple(
self.constant_to_value(c, {}, self.vm.root_cfg_node)
for c in pyval.constraints)
bound = (pyval.bound and self.constant_to_value(
pyval.bound, {}, self.vm.root_cfg_node))
return abstract.TypeParameter(pyval.name,
self.vm,
constraints=constraints,
bound=bound)
elif isinstance(pyval, abstract.AsInstance):
cls = pyval.cls
if isinstance(cls, pytd.ClassType):
cls = cls.cls
if isinstance(cls, pytd.Class):
# This key is also used in __init__
key = (abstract.Instance, cls)
if key not in self._convert_cache:
if cls.name in [
"__builtin__.type", "__builtin__.property"
]:
# An instance of "type" or of an anonymous property can be anything.
instance = self._create_new_unknown_value("type")
else:
mycls = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(mycls, self.vm)
instance.make_template_unsolvable(
cls.template, self.vm.root_cfg_node)
log.info("New pytd instance for %s: %r", cls.name,
instance)
self._convert_cache[key] = instance
return self._convert_cache[key]
elif isinstance(cls, pytd.GenericType):
assert isinstance(cls.base_type, pytd.ClassType)
base_cls = cls.base_type.cls
if base_cls.name == "__builtin__.type":
c, = cls.parameters
if isinstance(c, pytd.TypeParameter):
if not subst or c.name not in subst:
raise self.TypeParameterError(c.name)
return self.merge_classes(get_node(),
subst[c.name].data)
else:
return self.constant_to_value(c, subst,
self.vm.root_cfg_node)
elif isinstance(cls, pytd.TupleType):
content = tuple(
self.constant_to_var(abstract.AsInstance(p), subst,
get_node())
for p in cls.parameters)
return abstract.Tuple(content, self.vm)
elif isinstance(cls, pytd.CallableType):
clsval = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
return abstract.Instance(clsval, self.vm)
else:
clsval = self.constant_to_value(base_cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(clsval, self.vm)
assert len(cls.parameters) <= len(base_cls.template)
for formal, actual in zip(base_cls.template,
cls.parameters):
p = self.constant_to_var(abstract.AsInstance(actual),
subst, self.vm.root_cfg_node)
instance.initialize_type_parameter(
get_node(), formal.name, p)
return instance
else:
return self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.GenericType):
assert isinstance(pyval.base_type, pytd.ClassType)
base_cls = self.constant_to_value(pyval.base_type.cls, subst,
self.vm.root_cfg_node)
if not isinstance(base_cls, abstract.Class):
# base_cls can be, e.g., an unsolvable due to an mro error.
return self.unsolvable
if isinstance(pyval, pytd.TupleType):
abstract_class = abstract.TupleClass
template = range(len(pyval.parameters)) + [abstract.T]
parameters = pyval.parameters + (pytd.UnionType(
pyval.parameters), )
elif isinstance(pyval, pytd.CallableType):
abstract_class = abstract.Callable
template = range(len(
pyval.args)) + [abstract.ARGS, abstract.RET]
parameters = pyval.args + (pytd_utils.JoinTypes(
pyval.args), pyval.ret)
else:
abstract_class = abstract.ParameterizedClass
template = tuple(t.name for t in pyval.base_type.cls.template)
parameters = pyval.parameters
assert (pyval.base_type.name == "typing.Generic"
or len(parameters) <= len(template))
type_parameters = utils.LazyDict()
for i, name in enumerate(template):
if i < len(parameters):
type_parameters.add_lazy_item(name, self.constant_to_value,
parameters[i], subst,
self.vm.root_cfg_node)
else:
type_parameters[name] = self.unsolvable
return abstract_class(base_cls, type_parameters, self.vm)
elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node
return self.tuple_to_value([
self.constant_to_var(item, subst, self.vm.root_cfg_node)
for i, item in enumerate(pyval)
])
else:
raise NotImplementedError("Can't convert constant %s %r" %
(type(pyval), pyval))
def test_compatible_with__not_empty(self):
t = abstract.Tuple((self._var, ), self._vm)
self.assertTruthy(t)
def build_version_info(name, vm):
del name
tup = tuple(vm.convert.constant_to_var(i) for i in vm.python_version)
return abstract.Tuple(tup, vm)
def _build_namedtuple(self, name, field_names, field_types, late_annots,
node):
# Build an InterpreterClass representing the namedtuple.
if field_types:
# TODO(mdemello): Fix this to support late types.
field_types_union = abstract.Union(field_types, self.vm)
else:
field_types_union = self.vm.convert.none_type
members = {
n: t.instantiate(node)
for n, t in moves.zip(field_names, field_types)
}
# collections.namedtuple has: __dict__, __slots__ and _fields.
# typing.NamedTuple adds: _field_types, __annotations__ and _field_defaults.
# __slots__ and _fields are tuples containing the names of the fields.
slots = tuple(
self.vm.convert.build_string(node, f) for f in field_names)
members["__slots__"] = abstract.Tuple(slots, self.vm).to_variable(node)
members["_fields"] = abstract.Tuple(slots, self.vm).to_variable(node)
# __dict__ and _field_defaults are both collections.OrderedDicts that map
# field names (strings) to objects of the field types.
ordered_dict_cls = self.vm.convert.name_to_value(
"collections.OrderedDict", ast=self.collections_ast)
# In Python 2, keys can be `str` or `unicode`; support both.
# In Python 3, `str_type` and `unicode_type` are the same.
field_keys_union = abstract.Union(
[self.vm.convert.str_type, self.vm.convert.unicode_type], self.vm)
# Normally, we would use abstract_utils.K and abstract_utils.V, but
# collections.pyi doesn't conform to that standard.
field_dict_cls = abstract.ParameterizedClass(ordered_dict_cls, {
"K": field_keys_union,
"V": field_types_union
}, self.vm)
members["__dict__"] = field_dict_cls.instantiate(node)
members["_field_defaults"] = field_dict_cls.instantiate(node)
# _field_types and __annotations__ are both collections.OrderedDicts
# that map field names (strings) to the types of the fields.
field_types_cls = abstract.ParameterizedClass(
ordered_dict_cls, {
"K": field_keys_union,
"V": self.vm.convert.type_type
}, self.vm)
members["_field_types"] = field_types_cls.instantiate(node)
members["__annotations__"] = field_types_cls.instantiate(node)
# __new__
# We set the bound on this TypeParameter later. This gives __new__ the
# signature: def __new__(cls: Type[_Tname], ...) -> _Tname, i.e. the same
# signature that visitor.CreateTypeParametersForSignatures would create.
# This allows subclasses of the NamedTuple to get the correct type from
# their constructors.
cls_type_param = abstract.TypeParameter(
visitors.CreateTypeParametersForSignatures.PREFIX + name,
self.vm,
bound=None)
cls_type = abstract.ParameterizedClass(
self.vm.convert.type_type, {abstract_utils.T: cls_type_param},
self.vm)
# Use late annotations as field types if they exist.
params = [
Param(n, late_annots.get(n, t))
for n, t in moves.zip(field_names, field_types)
]
members["__new__"] = overlay_utils.make_method(
self.vm,
node,
name="__new__",
self_param=Param("cls", cls_type),
params=params,
return_type=cls_type_param,
)
# __init__
members["__init__"] = overlay_utils.make_method(self.vm,
node,
name="__init__",
varargs=Param("args"),
kwargs=Param("kwargs"))
# _make
# _make is a classmethod, so it needs to be wrapped by
# specialibuiltins.ClassMethodInstance.
# Like __new__, it uses the _Tname TypeVar.
sized_cls = self.vm.convert.name_to_value("typing.Sized")
iterable_type = abstract.ParameterizedClass(
self.vm.convert.name_to_value("typing.Iterable"),
{abstract_utils.T: field_types_union}, self.vm)
cls_type = abstract.ParameterizedClass(
self.vm.convert.type_type, {abstract_utils.T: cls_type_param},
self.vm)
len_type = abstract.CallableClass(
self.vm.convert.name_to_value("typing.Callable"), {
0: sized_cls,
abstract_utils.ARGS: sized_cls,
abstract_utils.RET: self.vm.convert.int_type
}, self.vm)
params = [
Param("iterable", iterable_type),
Param("new").unsolvable(self.vm, node),
Param("len", len_type).unsolvable(self.vm, node)
]
make = overlay_utils.make_method(self.vm,
node,
name="_make",
params=params,
self_param=Param("cls", cls_type),
return_type=cls_type_param)
make_args = function.Args(posargs=(make, ))
_, members["_make"] = self.vm.special_builtins["classmethod"].call(
node, None, make_args)
# _replace
# Like __new__, it uses the _Tname TypeVar. We have to annotate the `self`
# param to make sure the TypeVar is substituted correctly.
members["_replace"] = overlay_utils.make_method(
self.vm,
node,
name="_replace",
self_param=Param("self", cls_type_param),
return_type=cls_type_param,
kwargs=Param("kwds", field_types_union))
# __getnewargs__
getnewargs_tuple_params = dict(
tuple(enumerate(field_types)) +
((abstract_utils.T, field_types_union), ))
getnewargs_tuple = abstract.TupleClass(self.vm.convert.tuple_type,
getnewargs_tuple_params,
self.vm)
members["__getnewargs__"] = overlay_utils.make_method(
self.vm, node, name="__getnewargs__", return_type=getnewargs_tuple)
# __getstate__
members["__getstate__"] = overlay_utils.make_method(
self.vm, node, name="__getstate__")
# _asdict
members["_asdict"] = overlay_utils.make_method(
self.vm, node, name="_asdict", return_type=field_dict_cls)
# Finally, make the class.
cls_dict = abstract.Dict(self.vm)
cls_dict.update(node, members)
if name.__class__ is compat.UnicodeType:
# Unicode values should be ASCII.
name = compat.native_str(name.encode("ascii"))
node, cls_var = self.vm.make_class(
node=node,
name_var=self.vm.convert.build_string(node, name),
bases=[self.vm.convert.tuple_type.to_variable(node)],
class_dict_var=cls_dict.to_variable(node),
cls_var=None)
cls = cls_var.data[0]
# Now that the class has been made, we can complete the TypeParameter used
# by __new__, _make and _replace.
cls_type_param.bound = cls
# Add late annotations to the new class
if late_annots:
cls.late_annotations = late_annots
self.vm.classes_with_late_annotations.append(cls)
return node, cls_var
def _build_namedtuple(self, name, field_names, field_types, node):
# Build an InterpreterClass representing the namedtuple.
if field_types:
field_types_union = abstract.Union(field_types, self.vm)
else:
field_types_union = self.vm.convert.none_type
members = {
n: t.instantiate(node)
for n, t in moves.zip(field_names, field_types)
}
# collections.namedtuple has: __dict__, __slots__ and _fields.
# typing.NamedTuple adds: _field_types, __annotations__ and _field_defaults.
# __slots__ and _fields are tuples containing the names of the fields.
slots = tuple(
self.vm.convert.build_string(node, f) for f in field_names)
members["__slots__"] = abstract.Tuple(slots, self.vm).to_variable(node)
members["_fields"] = abstract.Tuple(slots, self.vm).to_variable(node)
# __dict__ and _field_defaults are both collections.OrderedDicts that map
# field names (strings) to objects of the field types.
ordered_dict_cls = self.vm.convert.name_to_value(
"collections.OrderedDict", ast=self.collections_ast)
# In Python 2, keys can be `str` or `unicode`; support both.
# In Python 3, `str_type` and `unicode_type` are the same.
field_keys_union = abstract.Union(
[self.vm.convert.str_type, self.vm.convert.unicode_type], self.vm)
# Normally, we would use abstract_utils.K and abstract_utils.V, but
# collections.pyi doesn't conform to that standard.
field_dict_cls = abstract.ParameterizedClass(ordered_dict_cls, {
"K": field_keys_union,
"V": field_types_union
}, self.vm)
members["__dict__"] = field_dict_cls.instantiate(node)
members["_field_defaults"] = field_dict_cls.instantiate(node)
# _field_types and __annotations__ are both collections.OrderedDicts
# that map field names (strings) to the types of the fields.
field_types_cls = abstract.ParameterizedClass(
ordered_dict_cls, {
"K": field_keys_union,
"V": self.vm.convert.type_type
}, self.vm)
members["_field_types"] = field_types_cls.instantiate(node)
members["__annotations__"] = field_types_cls.instantiate(node)
# __new__
new_annots = {}
new_lates = {}
for (n, t) in moves.zip(field_names, field_types):
# We don't support late annotations yet, but once we do, they'll show up
# as LateAnnotation objects to be stored in new_lates.
new_annots[n] = t
# We set the bound on this TypeParameter later. This gives __new__ the
# signature: def __new__(cls: Type[_Tname], ...) -> _Tname, i.e. the same
# signature that visitor.CreateTypeParametersForSignatures would create.
# This allows subclasses of the NamedTuple to get the correct type from
# their constructors.
cls_type_param = abstract.TypeParameter(
visitors.CreateTypeParametersForSignatures.PREFIX + name,
self.vm,
bound=None)
new_annots["cls"] = abstract.ParameterizedClass(
self.vm.convert.type_type, {abstract_utils.T: cls_type_param},
self.vm)
new_annots["return"] = cls_type_param
members["__new__"] = abstract.SimpleFunction(
name="__new__",
param_names=("cls", ) + tuple(field_names),
varargs_name=None,
kwonly_params=(),
kwargs_name=None,
defaults={},
annotations=new_annots,
late_annotations=new_lates,
vm=self.vm).to_variable(node)
# __init__
members["__init__"] = abstract.SimpleFunction(
name="__init__",
param_names=("self", ),
varargs_name="args",
kwonly_params=(),
kwargs_name="kwargs",
defaults={},
annotations={},
late_annotations={},
vm=self.vm).to_variable(node)
# _make
# _make is a classmethod, so it needs to be wrapped by
# specialibuiltins.ClassMethodInstance.
# Like __new__, it uses the _Tname TypeVar.
sized_cls = self.vm.convert.name_to_value("typing.Sized")
iterable_type = abstract.ParameterizedClass(
self.vm.convert.name_to_value("typing.Iterable"),
{abstract_utils.T: field_types_union}, self.vm)
make = abstract.SimpleFunction(
name="_make",
param_names=("cls", "iterable", "new", "len"),
varargs_name=None,
kwonly_params=(),
kwargs_name=None,
defaults={
"new": self.vm.convert.unsolvable.to_variable(node),
"len": self.vm.convert.unsolvable.to_variable(node)
},
annotations={
"cls":
abstract.ParameterizedClass(self.vm.convert.type_type,
{abstract_utils.T: cls_type_param},
self.vm),
"iterable":
iterable_type,
"new":
self.vm.convert.unsolvable,
"len":
abstract.Callable(
self.vm.convert.name_to_value("typing.Callable"), {
0: sized_cls,
abstract_utils.ARGS: sized_cls,
abstract_utils.RET: self.vm.convert.int_type
}, self.vm),
"return":
cls_type_param
},
late_annotations={},
vm=self.vm).to_variable(node)
make_args = function.Args(posargs=(make, ))
_, members["_make"] = self.vm.special_builtins["classmethod"].call(
node, None, make_args)
# _replace
# Like __new__, it uses the _Tname TypeVar. We have to annotate the `self`
# param to make sure the TypeVar is substituted correctly.
members["_replace"] = abstract.SimpleFunction(
name="_replace",
param_names=("self", ),
varargs_name=None,
kwonly_params=(),
kwargs_name="kwds",
defaults={},
annotations={
"self": cls_type_param,
"kwds": field_types_union,
"return": cls_type_param
},
late_annotations={},
vm=self.vm).to_variable(node)
# __getnewargs__
getnewargs_tuple_params = dict(
tuple(enumerate(field_types)) +
((abstract_utils.T, field_types_union), ))
getnewargs_tuple = abstract.TupleClass(self.vm.convert.tuple_type,
getnewargs_tuple_params,
self.vm)
members["__getnewargs__"] = abstract.SimpleFunction(
name="__getnewargs__",
param_names=("self", ),
varargs_name=None,
kwonly_params=(),
kwargs_name=None,
defaults={},
annotations={
"return": getnewargs_tuple
},
late_annotations={},
vm=self.vm).to_variable(node)
# __getstate__
members["__getstate__"] = abstract.SimpleFunction(
name="__getstate__",
param_names=("self", ),
varargs_name=None,
kwonly_params=(),
kwargs_name=None,
defaults={},
annotations={},
late_annotations={},
vm=self.vm).to_variable(node)
# _asdict
members["_asdict"] = abstract.SimpleFunction(
name="_asdict",
param_names=("self", ),
varargs_name=None,
kwonly_params=(),
kwargs_name=None,
defaults={},
annotations={
"return": field_dict_cls
},
late_annotations={},
vm=self.vm).to_variable(node)
# Finally, make the class.
abs_membs = abstract.Dict(self.vm)
abs_membs.update(node, members)
cls_var = self.vm.make_class(
node=node,
name_var=self.vm.convert.build_string(node, name),
bases=[self.vm.convert.tuple_type.to_variable(node)],
class_dict_var=abs_membs.to_variable(node),
cls_var=None)
# Now that the class has been made, we can complete the TypeParameter used
# by __new__, _make and _replace.
cls_type_param.bound = cls_var.data[0]
return cls_var
def test_compatible_with__not_empty(self):
t = abstract.Tuple((self._var, ), self._vm, self._node)
self.assertIs(True, t.compatible_with(True))
self.assertIs(False, t.compatible_with(False))
def test_compatible_with__not_empty(self):
t = abstract.Tuple((self._var, ), self._vm)
self.assertIs(True, compare.compatible_with(t, True))
self.assertIs(False, compare.compatible_with(t, False))
def test_compatible_with__empty(self):
t = abstract.Tuple((), self._vm)
self.assertIs(False, t.compatible_with(True))
self.assertIs(True, t.compatible_with(False))
def test_getitem__concrete_index(self):
t = abstract.Tuple((self._var, ), self._vm, self._node)
index = self._vm.convert.constant_to_var("index", 0)
node, var = t.getitem_slot(self._node, index)
self.assertIs(node, self._node)
self.assertIs(var, self._var)
