def testInstantiateTypeParameterType(self):
params = {abstract.T: abstract.TypeParameter(abstract.T, self._vm)}
cls = abstract.ParameterizedClass(self._vm.convert.type_type, params,
self._vm)
self.assertListEqual(
cls.instantiate(self._node).data, [self._vm.convert.unsolvable])
def create_new_kwargs_value(self, arg_type):
"""Create a kwargs argument given its element type."""
params = {abstract_utils.K: self.str_type, abstract_utils.V: arg_type}
return abstract.ParameterizedClass(self.dict_type, params, self.vm)
def testClassAgainstTypeUnion(self):
left = self._make_class("foo")
union = abstract.Union((left, ), self.vm)
right = abstract.ParameterizedClass(self.type_type,
{abstract.T: union}, self.vm)
self.assertMatch(left, right)
def create_new_varargs_value(self, arg_type):
"""Create a varargs argument given its element type."""
params = {abstract_utils.T: arg_type}
return abstract.ParameterizedClass(self.tuple_type, params, self.vm)
def _build_namedtuple(self, name, field_names, field_types, late_annots,
node):
# Build an InterpreterClass representing the namedtuple.
if field_types:
# TODO(mdemello): Fix this to support late types.
field_types_union = abstract.Union(field_types, self.vm)
else:
field_types_union = self.vm.convert.none_type
members = {
n: t.instantiate(node)
for n, t in moves.zip(field_names, field_types)
}
# collections.namedtuple has: __dict__, __slots__ and _fields.
# typing.NamedTuple adds: _field_types, __annotations__ and _field_defaults.
# __slots__ and _fields are tuples containing the names of the fields.
slots = tuple(
self.vm.convert.build_string(node, f) for f in field_names)
members["__slots__"] = abstract.Tuple(slots, self.vm).to_variable(node)
members["_fields"] = abstract.Tuple(slots, self.vm).to_variable(node)
# __dict__ and _field_defaults are both collections.OrderedDicts that map
# field names (strings) to objects of the field types.
ordered_dict_cls = self.vm.convert.name_to_value(
"collections.OrderedDict", ast=self.collections_ast)
# In Python 2, keys can be `str` or `unicode`; support both.
# In Python 3, `str_type` and `unicode_type` are the same.
field_keys_union = abstract.Union(
[self.vm.convert.str_type, self.vm.convert.unicode_type], self.vm)
# Normally, we would use abstract_utils.K and abstract_utils.V, but
# collections.pyi doesn't conform to that standard.
field_dict_cls = abstract.ParameterizedClass(ordered_dict_cls, {
"K": field_keys_union,
"V": field_types_union
}, self.vm)
members["__dict__"] = field_dict_cls.instantiate(node)
members["_field_defaults"] = field_dict_cls.instantiate(node)
# _field_types and __annotations__ are both collections.OrderedDicts
# that map field names (strings) to the types of the fields.
field_types_cls = abstract.ParameterizedClass(
ordered_dict_cls, {
"K": field_keys_union,
"V": self.vm.convert.type_type
}, self.vm)
members["_field_types"] = field_types_cls.instantiate(node)
members["__annotations__"] = field_types_cls.instantiate(node)
# __new__
# We set the bound on this TypeParameter later. This gives __new__ the
# signature: def __new__(cls: Type[_Tname], ...) -> _Tname, i.e. the same
# signature that visitor.CreateTypeParametersForSignatures would create.
# This allows subclasses of the NamedTuple to get the correct type from
# their constructors.
cls_type_param = abstract.TypeParameter(
visitors.CreateTypeParametersForSignatures.PREFIX + name,
self.vm,
bound=None)
cls_type = abstract.ParameterizedClass(
self.vm.convert.type_type, {abstract_utils.T: cls_type_param},
self.vm)
# Use late annotations as field types if they exist.
params = [
Param(n, late_annots.get(n, t))
for n, t in moves.zip(field_names, field_types)
]
members["__new__"] = overlay_utils.make_method(
self.vm,
node,
name="__new__",
self_param=Param("cls", cls_type),
params=params,
return_type=cls_type_param,
)
# __init__
members["__init__"] = overlay_utils.make_method(self.vm,
node,
name="__init__",
varargs=Param("args"),
kwargs=Param("kwargs"))
# _make
# _make is a classmethod, so it needs to be wrapped by
# specialibuiltins.ClassMethodInstance.
# Like __new__, it uses the _Tname TypeVar.
sized_cls = self.vm.convert.name_to_value("typing.Sized")
iterable_type = abstract.ParameterizedClass(
self.vm.convert.name_to_value("typing.Iterable"),
{abstract_utils.T: field_types_union}, self.vm)
cls_type = abstract.ParameterizedClass(
self.vm.convert.type_type, {abstract_utils.T: cls_type_param},
self.vm)
len_type = abstract.CallableClass(
self.vm.convert.name_to_value("typing.Callable"), {
0: sized_cls,
abstract_utils.ARGS: sized_cls,
abstract_utils.RET: self.vm.convert.int_type
}, self.vm)
params = [
Param("iterable", iterable_type),
Param("new").unsolvable(self.vm, node),
Param("len", len_type).unsolvable(self.vm, node)
]
make = overlay_utils.make_method(self.vm,
node,
name="_make",
params=params,
self_param=Param("cls", cls_type),
return_type=cls_type_param)
make_args = function.Args(posargs=(make, ))
_, members["_make"] = self.vm.special_builtins["classmethod"].call(
node, None, make_args)
# _replace
# Like __new__, it uses the _Tname TypeVar. We have to annotate the `self`
# param to make sure the TypeVar is substituted correctly.
members["_replace"] = overlay_utils.make_method(
self.vm,
node,
name="_replace",
self_param=Param("self", cls_type_param),
return_type=cls_type_param,
kwargs=Param("kwds", field_types_union))
# __getnewargs__
getnewargs_tuple_params = dict(
tuple(enumerate(field_types)) +
((abstract_utils.T, field_types_union), ))
getnewargs_tuple = abstract.TupleClass(self.vm.convert.tuple_type,
getnewargs_tuple_params,
self.vm)
members["__getnewargs__"] = overlay_utils.make_method(
self.vm, node, name="__getnewargs__", return_type=getnewargs_tuple)
# __getstate__
members["__getstate__"] = overlay_utils.make_method(
self.vm, node, name="__getstate__")
# _asdict
members["_asdict"] = overlay_utils.make_method(
self.vm, node, name="_asdict", return_type=field_dict_cls)
# Finally, make the class.
cls_dict = abstract.Dict(self.vm)
cls_dict.update(node, members)
if name.__class__ is compat.UnicodeType:
# Unicode values should be ASCII.
name = compat.native_str(name.encode("ascii"))
node, cls_var = self.vm.make_class(
node=node,
name_var=self.vm.convert.build_string(node, name),
bases=[self.vm.convert.tuple_type.to_variable(node)],
class_dict_var=cls_dict.to_variable(node),
cls_var=None)
cls = cls_var.data[0]
# Now that the class has been made, we can complete the TypeParameter used
# by __new__, _make and _replace.
cls_type_param.bound = cls
# Add late annotations to the new class
if late_annots:
cls.late_annotations = late_annots
self.vm.classes_with_late_annotations.append(cls)
return node, cls_var
def _build_namedtuple(self, name, field_names, field_types, node):
# Build an InterpreterClass representing the namedtuple.
if field_types:
field_types_union = abstract.Union(field_types, self.vm)
else:
field_types_union = self.vm.convert.none_type
members = {
n: t.instantiate(node)
for n, t in moves.zip(field_names, field_types)
}
# collections.namedtuple has: __dict__, __slots__ and _fields.
# typing.NamedTuple adds: _field_types, __annotations__ and _field_defaults.
# __slots__ and _fields are tuples containing the names of the fields.
slots = tuple(
self.vm.convert.build_string(node, f) for f in field_names)
members["__slots__"] = abstract.Tuple(slots, self.vm).to_variable(node)
members["_fields"] = abstract.Tuple(slots, self.vm).to_variable(node)
# __dict__ and _field_defaults are both collections.OrderedDicts that map
# field names (strings) to objects of the field types.
ordered_dict_cls = self.vm.convert.name_to_value(
"collections.OrderedDict", ast=self.collections_ast)
# In Python 2, keys can be `str` or `unicode`; support both.
# In Python 3, `str_type` and `unicode_type` are the same.
field_keys_union = abstract.Union(
[self.vm.convert.str_type, self.vm.convert.unicode_type], self.vm)
# Normally, we would use abstract_utils.K and abstract_utils.V, but
# collections.pyi doesn't conform to that standard.
field_dict_cls = abstract.ParameterizedClass(ordered_dict_cls, {
"K": field_keys_union,
"V": field_types_union
}, self.vm)
members["__dict__"] = field_dict_cls.instantiate(node)
members["_field_defaults"] = field_dict_cls.instantiate(node)
# _field_types and __annotations__ are both collections.OrderedDicts
# that map field names (strings) to the types of the fields.
field_types_cls = abstract.ParameterizedClass(
ordered_dict_cls, {
"K": field_keys_union,
"V": self.vm.convert.type_type
}, self.vm)
members["_field_types"] = field_types_cls.instantiate(node)
members["__annotations__"] = field_types_cls.instantiate(node)
# __new__
new_annots = {}
new_lates = {}
for (n, t) in moves.zip(field_names, field_types):
# We don't support late annotations yet, but once we do, they'll show up
# as LateAnnotation objects to be stored in new_lates.
new_annots[n] = t
# We set the bound on this TypeParameter later. This gives __new__ the
# signature: def __new__(cls: Type[_Tname], ...) -> _Tname, i.e. the same
# signature that visitor.CreateTypeParametersForSignatures would create.
# This allows subclasses of the NamedTuple to get the correct type from
# their constructors.
cls_type_param = abstract.TypeParameter(
visitors.CreateTypeParametersForSignatures.PREFIX + name,
self.vm,
bound=None)
new_annots["cls"] = abstract.ParameterizedClass(
self.vm.convert.type_type, {abstract_utils.T: cls_type_param},
self.vm)
new_annots["return"] = cls_type_param
members["__new__"] = abstract.SimpleFunction(
name="__new__",
param_names=("cls", ) + tuple(field_names),
varargs_name=None,
kwonly_params=(),
kwargs_name=None,
defaults={},
annotations=new_annots,
late_annotations=new_lates,
vm=self.vm).to_variable(node)
# __init__
members["__init__"] = abstract.SimpleFunction(
name="__init__",
param_names=("self", ),
varargs_name="args",
kwonly_params=(),
kwargs_name="kwargs",
defaults={},
annotations={},
late_annotations={},
vm=self.vm).to_variable(node)
# _make
# _make is a classmethod, so it needs to be wrapped by
# specialibuiltins.ClassMethodInstance.
# Like __new__, it uses the _Tname TypeVar.
sized_cls = self.vm.convert.name_to_value("typing.Sized")
iterable_type = abstract.ParameterizedClass(
self.vm.convert.name_to_value("typing.Iterable"),
{abstract_utils.T: field_types_union}, self.vm)
make = abstract.SimpleFunction(
name="_make",
param_names=("cls", "iterable", "new", "len"),
varargs_name=None,
kwonly_params=(),
kwargs_name=None,
defaults={
"new": self.vm.convert.unsolvable.to_variable(node),
"len": self.vm.convert.unsolvable.to_variable(node)
},
annotations={
"cls":
abstract.ParameterizedClass(self.vm.convert.type_type,
{abstract_utils.T: cls_type_param},
self.vm),
"iterable":
iterable_type,
"new":
self.vm.convert.unsolvable,
"len":
abstract.Callable(
self.vm.convert.name_to_value("typing.Callable"), {
0: sized_cls,
abstract_utils.ARGS: sized_cls,
abstract_utils.RET: self.vm.convert.int_type
}, self.vm),
"return":
cls_type_param
},
late_annotations={},
vm=self.vm).to_variable(node)
make_args = function.Args(posargs=(make, ))
_, members["_make"] = self.vm.special_builtins["classmethod"].call(
node, None, make_args)
# _replace
# Like __new__, it uses the _Tname TypeVar. We have to annotate the `self`
# param to make sure the TypeVar is substituted correctly.
members["_replace"] = abstract.SimpleFunction(
name="_replace",
param_names=("self", ),
varargs_name=None,
kwonly_params=(),
kwargs_name="kwds",
defaults={},
annotations={
"self": cls_type_param,
"kwds": field_types_union,
"return": cls_type_param
},
late_annotations={},
vm=self.vm).to_variable(node)
# __getnewargs__
getnewargs_tuple_params = dict(
tuple(enumerate(field_types)) +
((abstract_utils.T, field_types_union), ))
getnewargs_tuple = abstract.TupleClass(self.vm.convert.tuple_type,
getnewargs_tuple_params,
self.vm)
members["__getnewargs__"] = abstract.SimpleFunction(
name="__getnewargs__",
param_names=("self", ),
varargs_name=None,
kwonly_params=(),
kwargs_name=None,
defaults={},
annotations={
"return": getnewargs_tuple
},
late_annotations={},
vm=self.vm).to_variable(node)
# __getstate__
members["__getstate__"] = abstract.SimpleFunction(
name="__getstate__",
param_names=("self", ),
varargs_name=None,
kwonly_params=(),
kwargs_name=None,
defaults={},
annotations={},
late_annotations={},
vm=self.vm).to_variable(node)
# _asdict
members["_asdict"] = abstract.SimpleFunction(
name="_asdict",
param_names=("self", ),
varargs_name=None,
kwonly_params=(),
kwargs_name=None,
defaults={},
annotations={
"return": field_dict_cls
},
late_annotations={},
vm=self.vm).to_variable(node)
# Finally, make the class.
abs_membs = abstract.Dict(self.vm)
abs_membs.update(node, members)
cls_var = self.vm.make_class(
node=node,
name_var=self.vm.convert.build_string(node, name),
bases=[self.vm.convert.tuple_type.to_variable(node)],
class_dict_var=abs_membs.to_variable(node),
cls_var=None)
# Now that the class has been made, we can complete the TypeParameter used
# by __new__, _make and _replace.
cls_type_param.bound = cls_var.data[0]
return cls_var
def construct_constant_from_value(self, name, pyval, subst, node):
"""Create a AtomicAbstractValue that represents a python constant.
This supports both constant from code constant pools and PyTD constants such
as classes. This also supports builtin python objects such as int and float.
Args:
name: The name of this constant. Used for naming its attribute variables.
pyval: The python or PyTD value to convert.
subst: The current type parameters.
node: The current CFG node.
Returns:
A Value that represents the constant, or None if we couldn't convert.
Raises:
NotImplementedError: If we don't know how to convert a value.
"""
if pyval is type:
return abstract.SimpleAbstractValue(name, self.vm)
elif isinstance(pyval, str):
return abstract.AbstractOrConcreteValue(
pyval, self.str_type, self.vm, node)
elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH:
# For small integers, preserve the actual value (for things like the
# level in IMPORT_NAME).
return abstract.AbstractOrConcreteValue(
pyval, self.int_type, self.vm, node)
elif pyval.__class__ in self.primitive_classes:
return self.primitive_class_instances[pyval.__class__]
elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)):
return abstract.AbstractOrConcreteValue(
pyval, self.primitive_classes[types.CodeType], self.vm, node)
elif pyval.__class__ in [types.FunctionType,
types.ModuleType,
types.GeneratorType,
type]:
try:
pyclass = self.vm.vmbuiltins.Lookup("__builtin__." + pyval.__name__)
return self.convert_constant_to_value(name, pyclass, subst, node)
except (KeyError, AttributeError):
log.debug("Failed to find pytd", exc_info=True)
raise
elif isinstance(pyval, pytd.TypeDeclUnit):
data = pyval.constants + pyval.classes + pyval.functions + pyval.aliases
members = {val.name.rsplit(".")[-1]: val
for val in data}
return abstract.Module(self.vm, node, pyval.name, members)
elif isinstance(pyval, pytd.Class):
if "." in pyval.name:
module, base_name = pyval.name.rsplit(".", 1)
cls = abstract.PyTDClass(base_name, pyval, self.vm)
cls.module = module
else:
cls = abstract.PyTDClass(name, pyval, self.vm)
return cls
elif isinstance(pyval, pytd.Function):
signatures = [abstract.PyTDSignature(pyval.name, sig, self.vm)
for sig in pyval.signatures]
f = abstract.PyTDFunction(
pyval.name, signatures, pyval.kind, self.vm, node)
return f
elif isinstance(pyval, pytd.ClassType):
assert pyval.cls
return self.convert_constant_to_value(pyval.name, pyval.cls, subst, node)
elif isinstance(pyval, pytd.NothingType):
return self.nothing
elif isinstance(pyval, pytd.AnythingType):
# TODO(kramm): This should be an Unsolveable. We don't need to solve this.
return self._create_new_unknown_value("AnythingType")
elif isinstance(pyval, pytd.FunctionType):
return self.construct_constant_from_value(
name, pyval.function, subst, node)
elif isinstance(pyval, pytd.UnionType):
return abstract.Union([
self.convert_constant_to_value(pytd.Print(t), t, subst, node)
for t in pyval.type_list], self.vm)
elif isinstance(pyval, pytd.TypeParameter):
return abstract.TypeParameter(pyval.name, self.vm)
elif isinstance(pyval, abstract.AsInstance):
cls = pyval.cls
if isinstance(cls, pytd.ClassType):
cls = cls.cls
if isinstance(cls, pytd.Class):
# This key is also used in __init__
key = (abstract.Instance, cls)
if key not in self._convert_cache:
if cls.name in ["__builtin__.type", "__builtin__.property"]:
# An instance of "type" or of an anonymous property can be anything.
instance = self._create_new_unknown_value("type")
else:
mycls = self.convert_constant(cls.name, cls, subst, node)
instance = abstract.Instance(mycls, self.vm, node)
log.info("New pytd instance for %s: %r", cls.name, instance)
self._convert_cache[key] = instance
return self._convert_cache[key]
elif isinstance(cls, pytd.GenericType):
assert isinstance(cls.base_type, pytd.ClassType)
base_cls = cls.base_type.cls
instance = abstract.Instance(
self.convert_constant(base_cls.name, base_cls, subst, node),
self.vm, node)
for formal, actual in zip(base_cls.template, cls.parameters):
p = self.convert_constant(
repr(formal), abstract.AsInstance(actual), subst, node)
instance.initialize_type_parameter(node, formal.name, p)
return instance
else:
return self.convert_constant_to_value(name, cls, subst, node)
elif isinstance(pyval, pytd.GenericType):
assert isinstance(pyval.base_type, pytd.ClassType)
type_parameters = utils.LazyDict()
for param, value in zip(pyval.base_type.cls.template, pyval.parameters):
type_parameters.add_lazy_item(
param.name, self.convert_constant_to_value,
param.name, value, subst, node)
cls = self.convert_constant_to_value(
pytd.Print(pyval.base_type), pyval.base_type.cls, subst, node)
return abstract.ParameterizedClass(cls, type_parameters, self.vm)
elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node
return self.tuple_to_value(self.vm.root_cfg_node,
[self.convert_constant("tuple[%d]" % i, item,
subst, node)
for i, item in enumerate(pyval)])
else:
raise NotImplementedError("Can't convert constant %s %r" %
(type(pyval), pyval))
def make_initvar(t):
return abstract.ParameterizedClass(
initvar, {abstract_utils.T: t}, self.vm)