def test_union_type_ne(self):
u1 = pytd.UnionType((self.int, self.float))
u2 = pytd.UnionType((self.float, self.int, self.none_type))
self.assertNotEqual(u1, u2)
self.assertNotEqual(u2, u1)
self.assertEqual(u1.type_list, (self.int, self.float))
self.assertEqual(u2.type_list, (self.float, self.int, self.none_type))
def testUnionTypeEq(self):
u1 = pytd.UnionType((self.int, self.float))
u2 = pytd.UnionType((self.float, self.int))
self.assertEqual(u1, u2)
self.assertEqual(u2, u1)
self.assertEqual(u1.type_list, (self.int, self.float))
self.assertEqual(u2.type_list, (self.float, self.int))
def VisitGenericType(self, t):
module, name = self._GetModuleAndName(t)
if self._IsTyping(module):
if name == "Optional":
return pytd.UnionType(t.parameters + (pytd.NamedType("NoneType"),))
elif name == "Union":
return pytd.UnionType(t.parameters)
return t
def testOrder(self):
# pytd types' primary sort key is the class name, second sort key is
# the contents when interpreted as a (named)tuple.
nodes = [pytd.AnythingType(),
pytd.GenericType(self.list, (self.int,)),
pytd.NamedType("int"),
pytd.NothingType(),
pytd.UnionType((self.float,)),
pytd.UnionType((self.int,))]
for n1, n2 in zip(nodes[:-1], nodes[1:]):
self.assertLess(n1, n2)
self.assertLessEqual(n1, n2)
self.assertGreater(n2, n1)
self.assertGreaterEqual(n2, n1)
for p in itertools.permutations(nodes):
self.assertEquals(list(sorted(p)), nodes)
def JoinTypes(types):
"""Combine a list of types into a union type, if needed.
Leaves singular return values alone, or wraps a UnionType around them if there
are multiple ones, or if there are no elements in the list (or only
NothingType) return NothingType.
Arguments:
types: A list of types. This list might contain other UnionTypes. If
so, they are flattened.
Returns:
A type that represents the union of the types passed in. Order is preserved.
"""
queue = collections.deque(types)
seen = set()
new_types = []
while queue:
t = queue.popleft()
if isinstance(t, pytd.UnionType):
queue.extendleft(reversed(t.type_list))
elif isinstance(t, pytd.NothingType):
pass
elif t not in seen:
new_types.append(t)
seen.add(t)
if len(new_types) == 1:
return new_types.pop()
elif any(isinstance(t, pytd.AnythingType) for t in new_types):
return pytd.AnythingType()
elif new_types:
return pytd.UnionType(tuple(new_types)) # tuple() to make unions hashable
else:
return pytd.NothingType()
def _get_parameters(self, t1, t2):
if isinstance(t1, pytd.TupleType) and isinstance(t2, pytd.TupleType):
# No change needed; the parameters will be compared element-wise.
return t1.parameters, t2.parameters
elif isinstance(t2, pytd.TupleType):
# Since we call _get_parameters after confirming that t1 and t2 have
# compatible base types, t1 is a homogeneous tuple here.
return (t1.element_type,) * len(t2.parameters), t2.parameters
elif isinstance(t1, pytd.TupleType):
return (pytd.UnionType(type_list=t1.parameters),), t2.parameters
elif (isinstance(t1, pytd.CallableType) and
isinstance(t2, pytd.CallableType)):
# Flip the arguments, since argument types are contravariant.
return t2.args + (t1.ret,), t1.args + (t2.ret,)
elif (t1.base_type.cls.name == "__builtin__.type" and
t2.base_type.cls.name == "typing.Callable"):
# We'll only check the return type, since getting the argument types for
# initializing a class is tricky.
return t1.parameters, (t2.parameters[-1],)
elif (t1.base_type.cls.name == "typing.Callable" and
t2.base_type.cls.name == "__builtin__.type"):
return (t1.parameters[-1],), t2.parameters
elif isinstance(t1, pytd.CallableType):
# We're matching against GenericType(Callable, (Any, _RET)), so we don't
# need the argument types.
return (pytd.AnythingType(), t1.ret), t2.parameters
elif isinstance(t2, pytd.CallableType):
return t1.parameters, (pytd.AnythingType(), t2.ret)
else:
num_extra_params = len(t1.parameters) - len(t2.parameters)
# Matching, e.g., Dict[str, int] against Iterable[K] is legitimate.
assert num_extra_params >= 0, (t1.base_type.cls.name,
t2.base_type.cls.name)
t2_parameters = t2.parameters + (pytd.AnythingType(),) * num_extra_params
return t1.parameters, t2_parameters
def test_join_optional_anything_types(self):
"""Test that JoinTypes() simplifies unions containing 'Any' and 'None'."""
any_type = pytd.AnythingType()
none_type = pytd.NamedType("builtins.NoneType")
types = [pytd.NamedType("a"), any_type, none_type]
self.assertEqual(pytd_utils.JoinTypes(types),
pytd.UnionType((any_type, none_type)))
def match_Generic_against_Generic(self, t1, t2, subst): # pylint: disable=invalid-name
"""Match a pytd.GenericType against another pytd.GenericType."""
assert isinstance(t1.base_type, pytd.ClassType), type(t1.base_type)
assert isinstance(t2.base_type, pytd.ClassType), type(t2.base_type)
base1 = pytd.ClassType(t1.base_type.cls.name, t1.base_type.cls)
base2 = pytd.ClassType(t2.base_type.cls.name, t2.base_type.cls)
base_type_cmp = self.match_type_against_type(base1, base2, subst)
if base_type_cmp is booleq.FALSE:
return booleq.FALSE
if not isinstance(t1, pytd.TupleType) and isinstance(
t2, pytd.TupleType):
p1, = t1.parameters
param_cmp = [
self.match_type_against_type(p1, p2, subst)
for p2 in t2.parameters
]
else:
t1_parameters = t1.parameters
if isinstance(t1, pytd.TupleType):
if isinstance(t2, pytd.TupleType):
if len(t1_parameters) != len(t2.parameters):
return booleq.FALSE
else:
t1_parameters = (pytd.UnionType(type_list=t1_parameters), )
# Matching, e.g., Dict[str, int] against Iterable[K] is legitimate.
assert len(t1_parameters) >= len(
t2.parameters), t1.base_type.cls.name
# Type parameters are covariant:
# E.g. passing list[int] as argument for list[object] succeeds.
param_cmp = [
self.match_type_against_type(p1, p2, subst)
for p1, p2 in zip(t1_parameters, t2.parameters)
]
return booleq.And([base_type_cmp] + param_cmp)
def testComplexCombinedType(self):
"""Test parsing a type with both union and intersection."""
data1 = r"def foo(a: Foo or Bar and Zot) -> object: ..."
data2 = r"def foo(a: Foo or (Bar and Zot)) -> object: ..."
result1 = self.Parse(data1)
result2 = self.Parse(data2)
f = pytd.Function(
name="foo",
signatures=(pytd.Signature(params=(pytd.Parameter(
name="a",
type=pytd.UnionType(
type_list=(pytd.NamedType("Foo"),
pytd.IntersectionType(
type_list=(pytd.NamedType("Bar"),
pytd.NamedType("Zot"))))),
kwonly=False,
optional=False,
mutated_type=None), ),
starargs=None,
starstarargs=None,
return_type=pytd.NamedType("object"),
template=(),
exceptions=()), ),
kind=pytd.METHOD)
self.assertEqual(f, result1.Lookup("foo"))
self.assertEqual(f, result2.Lookup("foo"))
def value_to_pytd_type(self, node, v, seen, view):
"""Get a PyTD type representing this object, as seen at a node.
Args:
node: The node from which we want to observe this object.
v: The object.
seen: The set of values seen before while computing the type.
view: A Variable -> binding map.
Returns:
A PyTD type.
"""
if isinstance(v, (abstract.Empty, abstract.Nothing)):
return pytd.NothingType()
elif isinstance(v, abstract.TypeParameterInstance):
if (v.name in v.instance.type_parameters and
v.instance.type_parameters[v.name].bindings):
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, p, seen, view)
for p in v.instance.type_parameters[v.name].data)
else:
# The type parameter was never initialized
return pytd.AnythingType()
elif isinstance(v, (abstract.Function, abstract.IsInstance,
abstract.BoundFunction)):
return pytd.NamedType("typing.Callable")
elif isinstance(v, abstract.Class):
param = self.value_instance_to_pytd_type(node, v, None, seen, view)
return pytd.GenericType(base_type=pytd.NamedType("__builtin__.type"),
parameters=(param,))
elif isinstance(v, abstract.Module):
return pytd.NamedType("__builtin__.module")
elif isinstance(v, abstract.SimpleAbstractValue):
if v.cls:
classvalues = self._get_values(node, v.cls, view)
cls_types = []
for cls in classvalues:
cls_types.append(self.value_instance_to_pytd_type(
node, cls, v, seen=seen, view=view))
ret = pytd_utils.JoinTypes(cls_types)
visitors.InPlaceFillInClasses(ret, v.vm.loader.builtins)
return ret
else:
# We don't know this type's __class__, so return AnythingType to
# indicate that we don't know anything about what this is.
# This happens e.g. for locals / globals, which are returned from the
# code in class declarations.
log.info("Using ? for %s", v.name)
return pytd.AnythingType()
elif isinstance(v, abstract.Union):
return pytd.UnionType(tuple(self.value_to_pytd_type(node, o, seen, view)
for o in v.options))
elif isinstance(v, abstract.SuperInstance):
return pytd.NamedType("__builtin__.super")
elif isinstance(v, abstract.Unsolvable):
return pytd.AnythingType()
elif isinstance(v, abstract.Unknown):
return pytd.NamedType(v.class_name)
else:
raise NotImplementedError(v.__class__.__name__)
def value_instance_to_pytd_type(self, node, v, instance, seen, view):
"""Get the PyTD type an instance of this object would have.
Args:
node: The node.
v: The object.
instance: The instance.
seen: Already seen instances.
view: A Variable -> binding map.
Returns:
A PyTD type.
"""
if isinstance(v, abstract.Union):
return pytd.UnionType(tuple(
self.value_instance_to_pytd_type(node, t, instance, seen, view)
for t in v.options))
elif isinstance(v, abstract.AnnotationContainer):
return self.value_instance_to_pytd_type(
node, v.base_cls, instance, seen, view)
elif isinstance(v, mixin.Class):
if not self._detailed and v.official_name is None:
return pytd.AnythingType()
if seen is None:
# We make the set immutable to ensure that the seen instances for
# different parameter values don't interfere with one another.
seen = frozenset()
if instance in seen:
# We have a circular dependency in our types (e.g., lst[0] == lst). Stop
# descending into the type parameters.
type_params = ()
else:
type_params = tuple(t.name for t in v.template)
if instance is not None:
seen |= {instance}
type_arguments = self._value_to_parameter_types(
node, v, instance, type_params, seen, view)
base = pytd_utils.NamedTypeWithModule(v.official_name or v.name, v.module)
if self._is_tuple(v, instance):
if type_arguments:
homogeneous = False
else:
homogeneous = True
type_arguments = [pytd.NothingType()]
elif v.full_name == "typing.Callable":
homogeneous = not isinstance(v, abstract.CallableClass)
else:
homogeneous = len(type_arguments) == 1
return pytd_utils.MakeClassOrContainerType(
base, type_arguments, homogeneous)
elif isinstance(v, abstract.TypeParameter):
# We generate the full definition because, if this type parameter is
# imported, we will need the definition in order to declare it later.
return self._typeparam_to_def(node, v, v.name)
elif isinstance(v, typing_overlay.NoReturn):
return pytd.NothingType()
else:
log.info("Using ? for instance of %s", v.name)
return pytd.AnythingType()
def p_type_homogeneous(self, p):
"""type : named_or_external_type LBRACKET parameters RBRACKET"""
_, base_type, _, parameters, _ = p
if p[1] == pytd.NamedType('Union'):
p[0] = pytd.UnionType(parameters)
elif p[1] == pytd.NamedType('Optional'):
p[0] = pytd.UnionType(parameters[0], pytd.NamedType('None'))
elif len(parameters) == 2 and parameters[-1] is Ellipsis:
element_type, _ = parameters
if element_type is Ellipsis:
make_syntax_error(self, '[..., ...] not supported', p)
p[0] = pytd.HomogeneousContainerType(base_type=base_type,
parameters=(element_type, ))
else:
parameters = tuple(pytd.AnythingType() if p is Ellipsis else p
for p in parameters)
p[0] = pytd.GenericType(base_type=base_type, parameters=parameters)
def test_metaclass_union(self):
cls = abstract.InterpreterClass("X", [], {}, None, self._vm)
meta1 = abstract.InterpreterClass("M1", [], {}, None, self._vm)
meta2 = abstract.InterpreterClass("M2", [], {}, None, self._vm)
meta1.official_name = "M1"
meta2.official_name = "M2"
cls.cls = abstract.Union([meta1, meta2], self._vm)
pytd_cls = cls.to_pytd_def(self._vm.root_cfg_node, "X")
self.assertEqual(pytd_cls.metaclass, pytd.UnionType(
(pytd.NamedType("M1"), pytd.NamedType("M2"))))
def value_instance_to_pytd_type(self, node, v, instance, seen, view):
"""Get the PyTD type an instance of this object would have.
Args:
node: The node.
v: The object.
instance: The instance.
seen: Already seen instances.
view: A Variable -> binding map.
Returns:
A PyTD type.
"""
if isinstance(v, abstract.Union):
return pytd.UnionType(tuple(
self.value_instance_to_pytd_type(node, t, instance, seen, view)
for t in v.options))
elif isinstance(v, abstract.AnnotationContainer):
return self.value_instance_to_pytd_type(
node, v.base_cls, instance, seen, view)
elif isinstance(v, abstract.Class):
if v.official_name is None:
return pytd.AnythingType()
if seen is None:
seen = set()
if instance in seen:
# We have a circular dependency in our types (e.g., lst[0] == lst). Stop
# descending into the type parameters.
type_params = ()
else:
type_params = tuple(t.name for t in v.template)
if instance is not None:
seen.add(instance)
type_arguments = self._value_to_parameter_types(
node, v, instance, type_params, seen, view)
base = pytd_utils.NamedTypeWithModule(v.official_name, v.module)
if self._is_tuple(v, instance):
if type_arguments:
homogeneous = False
else:
homogeneous = True
type_arguments = [pytd.NothingType()]
else:
homogeneous = len(type_arguments) == 1
return pytd_utils.MakeClassOrContainerType(
base, type_arguments, homogeneous)
elif isinstance(v, abstract.TypeVariable):
return pytd.TypeParameter(v.name, None)
else:
log.info("Using ? for instance of %s", v.name)
return pytd.AnythingType()
def setUp(self):
self.options = config.Options.create(
python_version=self.PYTHON_VERSION, python_exe=self.PYTHON_EXE)
def t(name): # pylint: disable=invalid-name
return pytd.ClassType("__builtin__." + name)
self.bool = t("bool")
self.dict = t("dict")
self.float = t("float")
self.complex = t("complex")
self.int = t("int")
self.list = t("list")
self.none_type = t("NoneType")
self.object = t("object")
self.set = t("set")
self.frozenset = t("frozenset")
self.str = t("str")
self.bytearray = t("bytearray")
self.tuple = t("tuple")
self.unicode = t("unicode")
self.generator = t("generator")
self.function = pytd.ClassType("typing.Callable")
self.anything = pytd.AnythingType()
self.nothing = pytd.NothingType()
self.module = t("module")
self.file = t("file")
# The various union types use pytd_utils.CanonicalOrdering()'s ordering:
self.intorstr = pytd.UnionType((self.int, self.str))
self.strorunicode = pytd.UnionType((self.str, self.unicode))
self.intorfloat = pytd.UnionType((self.float, self.int))
self.intorfloatorstr = pytd.UnionType((self.float, self.int, self.str))
self.complexorstr = pytd.UnionType((self.complex, self.str))
self.intorfloatorcomplex = pytd.UnionType(
(self.int, self.float, self.complex))
self.int_tuple = pytd.GenericType(self.tuple, (self.int, ))
self.nothing_tuple = pytd.GenericType(self.tuple, (self.nothing, ))
self.intorfloat_tuple = pytd.GenericType(self.tuple,
(self.intorfloat, ))
self.int_set = pytd.GenericType(self.set, (self.int, ))
self.intorfloat_set = pytd.GenericType(self.set, (self.intorfloat, ))
self.unknown_frozenset = pytd.GenericType(self.frozenset,
(self.anything, ))
self.float_frozenset = pytd.GenericType(self.frozenset, (self.float, ))
self.empty_frozenset = pytd.GenericType(self.frozenset,
(self.nothing, ))
self.int_list = pytd.GenericType(self.list, (self.int, ))
self.str_list = pytd.GenericType(self.list, (self.str, ))
self.intorfloat_list = pytd.GenericType(self.list, (self.intorfloat, ))
self.intorstr_list = pytd.GenericType(self.list, (self.intorstr, ))
self.anything_list = pytd.GenericType(self.list, (self.anything, ))
self.nothing_list = pytd.GenericType(self.list, (self.nothing, ))
self.int_int_dict = pytd.GenericType(self.dict, (self.int, self.int))
self.int_str_dict = pytd.GenericType(self.dict, (self.int, self.str))
self.str_int_dict = pytd.GenericType(self.dict, (self.str, self.int))
self.nothing_nothing_dict = pytd.GenericType(
self.dict, (self.nothing, self.nothing))
def setUpClass(cls):
super().setUpClass()
# We use class-wide loader to avoid creating a new loader for every test
# method if not required.
cls._loader = None
def t(name): # pylint: disable=invalid-name
return pytd.ClassType("builtins." + name)
cls.bool = t("bool")
cls.dict = t("dict")
cls.float = t("float")
cls.complex = t("complex")
cls.int = t("int")
cls.list = t("list")
cls.none_type = t("NoneType")
cls.object = t("object")
cls.set = t("set")
cls.frozenset = t("frozenset")
cls.str = t("str")
cls.bytearray = t("bytearray")
cls.tuple = t("tuple")
cls.unicode = t("unicode")
cls.generator = t("generator")
cls.function = pytd.ClassType("typing.Callable")
cls.anything = pytd.AnythingType()
cls.nothing = pytd.NothingType()
cls.module = t("module")
cls.file = t("file")
# The various union types use pytd_utils.CanonicalOrdering()'s ordering:
cls.intorstr = pytd.UnionType((cls.int, cls.str))
cls.strorunicode = pytd.UnionType((cls.str, cls.unicode))
cls.intorfloat = pytd.UnionType((cls.float, cls.int))
cls.intorfloatorstr = pytd.UnionType((cls.float, cls.int, cls.str))
cls.complexorstr = pytd.UnionType((cls.complex, cls.str))
cls.intorfloatorcomplex = pytd.UnionType(
(cls.int, cls.float, cls.complex))
cls.int_tuple = pytd.GenericType(cls.tuple, (cls.int, ))
cls.nothing_tuple = pytd.TupleType(cls.tuple, ())
cls.intorfloat_tuple = pytd.GenericType(cls.tuple, (cls.intorfloat, ))
cls.int_set = pytd.GenericType(cls.set, (cls.int, ))
cls.intorfloat_set = pytd.GenericType(cls.set, (cls.intorfloat, ))
cls.unknown_frozenset = pytd.GenericType(cls.frozenset,
(cls.anything, ))
cls.float_frozenset = pytd.GenericType(cls.frozenset, (cls.float, ))
cls.empty_frozenset = pytd.GenericType(cls.frozenset, (cls.nothing, ))
cls.int_list = pytd.GenericType(cls.list, (cls.int, ))
cls.str_list = pytd.GenericType(cls.list, (cls.str, ))
cls.intorfloat_list = pytd.GenericType(cls.list, (cls.intorfloat, ))
cls.intorstr_list = pytd.GenericType(cls.list, (cls.intorstr, ))
cls.anything_list = pytd.GenericType(cls.list, (cls.anything, ))
cls.nothing_list = pytd.GenericType(cls.list, (cls.nothing, ))
cls.int_int_dict = pytd.GenericType(cls.dict, (cls.int, cls.int))
cls.int_str_dict = pytd.GenericType(cls.dict, (cls.int, cls.str))
cls.str_int_dict = pytd.GenericType(cls.dict, (cls.str, cls.int))
cls.nothing_nothing_dict = pytd.GenericType(cls.dict,
(cls.nothing, cls.nothing))
def _normal_param(name, param_type, default, kwonly):
"""Return a pytd.Parameter object for a normal argument."""
if default is not None:
default_type = _type_for_default(default)
if param_type is None:
param_type = default_type
elif default_type == pytd.NamedType("NoneType"):
param_type = pytd.UnionType((param_type, default_type))
if param_type is None:
# TODO(kramm): We should use __builtin__.object. (And other places)
param_type = pytd.NamedType("object")
optional = default is not None
return pytd.Parameter(name, param_type, kwonly, optional, None)
def _normal_param(name, param_type, default, kwonly):
"""Return a pytd.Parameter object for a normal argument."""
if default is not None:
default_type = _type_for_default(default)
if default_type == pytd.NamedType("NoneType"):
if param_type is not None:
param_type = pytd.UnionType((param_type, default_type))
elif param_type is None:
param_type = default_type
if param_type is None:
param_type = pytd.AnythingType()
optional = default is not None
return pytd.Parameter(name, param_type, kwonly, optional, None)
def testListConcatMultiType(self):
ty = self.Infer("""
def f():
x = []
x.append(1)
x.append("str")
return x + [1.3] + x
f()
""",
deep=False,
show_library_calls=True)
self.assertHasOnlySignatures(
ty.Lookup("f"), ((),
pytd.GenericType(self.list, (pytd.UnionType(
(self.int, self.float, self.str)), ))))
def testJoinTypes(self):
"""Test that JoinTypes() does recursive flattening."""
n1, n2, n3, n4, n5, n6 = [pytd.NamedType("n%d" % i) for i in xrange(6)]
# n1 or (n2 or (n3))
nested1 = pytd.UnionType((n1, pytd.UnionType((n2, pytd.UnionType((n3,))))))
# ((n4) or n5) or n6
nested2 = pytd.UnionType((pytd.UnionType((pytd.UnionType((n4,)), n5)), n6))
joined = pytd_utils.JoinTypes([nested1, nested2])
self.assertEqual(joined.type_list,
(n1, n2, n3, n4, n5, n6))
def match_Unknown_against_Generic(self, t1, t2, subst): # pylint: disable=invalid-name
assert isinstance(t2.base_type, pytd.ClassType)
# No inheritance for base classes - you can only inherit from an
# instantiated template, but not from a template itself.
base_match = booleq.Eq(t1.name, t2.base_type.cls.name)
type_params = [self.type_parameter(t1, t2.base_type.cls, item)
for item in t2.base_type.cls.template]
for type_param in type_params:
self.solver.register_variable(type_param.name)
if isinstance(t2, pytd.TupleType):
t2_parameters = (pytd.UnionType(type_list=t2.parameters),)
else:
t2_parameters = t2.parameters
params = [self.match_type_against_type(p1, p2, subst)
for p1, p2 in zip(type_params, t2_parameters)]
return booleq.And([base_match] + params)
def match_Function_against_Class(self, f1, cls2, subst, cache):
cls2_methods = cache.get(id(cls2))
if cls2_methods is None:
cls2_methods = cache[id(cls2)] = {f.name: f for f in cls2.methods}
if f1.name not in cls2_methods:
# The class itself doesn't have this method, but base classes might.
# TODO(kramm): This should do MRO order, not depth-first.
for base in cls2.parents:
if isinstance(base, pytd.AnythingType):
# AnythingType can contain any method. However, that would mean that
# a class that inherits from AnythingType contains any method
# imaginable, and hence is a match for anything. To prevent the bad
# results caused by that, return FALSE here.
return booleq.FALSE
elif isinstance(base, (pytd.ClassType, pytd.GenericType)):
if isinstance(base, pytd.ClassType):
cls = base.cls
values = tuple(pytd.AnythingType()
for _ in cls.template)
elif isinstance(base, pytd.TupleType):
cls = base.base_type.cls
values = (pytd.UnionType(type_list=base.parameters), )
else:
cls = base.base_type.cls
values = base.parameters
if values:
subst = subst.copy()
for param, value in zip(cls.template, values):
subst[param.type_param] = value
implication = self.match_Function_against_Class(
f1, cls, subst, cache)
if implication is not booleq.FALSE:
return implication
else:
# Funky types like UnionType are hard to match against (and shouldn't
# appear as a base class) so we treat them as catch-all.
log.warning("Assuming that %s has method %s",
pytd_utils.Print(base), f1.name)
return booleq.TRUE
return booleq.FALSE
else:
f2 = cls2_methods[f1.name]
return self.match_Function_against_Function(f1,
f2,
subst,
skip_self=True)
def testListConcatMultiType(self):
with self.Infer("""
def f():
x = []
x.append(1)
x.append("str")
return x + [1.3] + x
f()
""",
deep=False,
solve_unknowns=False,
extract_locals=False) as ty:
self.assertHasOnlySignatures(
ty.Lookup("f"),
((),
pytd.HomogeneousContainerType(self.list, (pytd.UnionType(
(self.int, self.float, self.str)), ))))
def VisitUnionType(self, union):
"""Push unions down into containers.
This collects similar container types in unions and merges them into
single instances with the union type pushed down to the element_type level.
Arguments:
union: A pytd.Union instance. Might appear in a parameter, a return type,
a constant type, etc.
Returns:
A simplified pytd.Union.
"""
if not any(isinstance(t, pytd.GenericType) for t in union.type_list):
# Optimization: If we're not going to change anything, return original.
return union
union = utils.JoinTypes(union.type_list) # flatten
if not isinstance(union, pytd.UnionType):
union = pytd.UnionType((union, ))
collect = {}
has_redundant_base_types = False
for t in union.type_list:
if isinstance(t, pytd.GenericType):
if t.base_type in collect:
has_redundant_base_types = True
collect[t.base_type] = tuple(
utils.JoinTypes([p1, p2])
for p1, p2 in zip(collect[t.base_type], t.parameters))
else:
collect[t.base_type] = t.parameters
if not has_redundant_base_types:
return union
result = pytd.NothingType()
done = set()
for t in union.type_list:
if isinstance(t, pytd.GenericType):
if t.base_type in done:
continue # already added
parameters = collect[t.base_type]
add = t.Replace(parameters=tuple(
p.Visit(CombineContainers()) for p in parameters))
done.add(t.base_type)
else:
add = t
result = utils.JoinTypes([result, add])
return result
def p_type_or(self, p):
"""type : type OR type"""
# This rule depends on precedence specification
# UnionType flattens any contained UnionType's
p[0] = pytd.UnionType((p[1], p[3]))
def _constant_to_value(self, pyval, subst, get_node):
"""Create a AtomicAbstractValue that represents a python constant.
This supports both constant from code constant pools and PyTD constants such
as classes. This also supports builtin python objects such as int and float.
Args:
pyval: The python or PyTD value to convert.
subst: The current type parameters.
get_node: A getter function for the current node.
Returns:
A Value that represents the constant, or None if we couldn't convert.
Raises:
NotImplementedError: If we don't know how to convert a value.
TypeParameterError: If we can't find a substitution for a type parameter.
"""
if pyval.__class__ is str:
# We use a subclass of str, compat.BytesPy3, to mark Python 3
# bytestrings, which are converted to abstract bytes instances.
# compat.BytesType dispatches to this when appropriate.
return abstract.AbstractOrConcreteValue(pyval, self.str_type,
self.vm)
elif isinstance(pyval, compat.UnicodeType):
return abstract.AbstractOrConcreteValue(pyval, self.unicode_type,
self.vm)
elif isinstance(pyval, compat.BytesType):
return abstract.AbstractOrConcreteValue(pyval, self.bytes_type,
self.vm)
elif isinstance(pyval, bool):
return self.true if pyval is True else self.false
elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH:
# For small integers, preserve the actual value (for things like the
# level in IMPORT_NAME).
return abstract.AbstractOrConcreteValue(pyval, self.int_type,
self.vm)
elif isinstance(pyval, compat.LongType):
# long is aliased to int
return self.primitive_class_instances[int]
elif pyval.__class__ in self.primitive_classes:
return self.primitive_class_instances[pyval.__class__]
elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)):
return abstract.AbstractOrConcreteValue(
pyval, self.primitive_classes[types.CodeType], self.vm)
elif pyval is super:
return special_builtins.Super(self.vm)
elif pyval is object:
return special_builtins.Object(self.vm)
elif pyval.__class__ is type:
try:
return self.name_to_value(self._type_to_name(pyval), subst)
except (KeyError, AttributeError):
log.debug("Failed to find pytd", exc_info=True)
raise
elif isinstance(pyval, pytd.LateType):
actual = self._load_late_type(pyval)
return self._constant_to_value(actual, subst, get_node)
elif isinstance(pyval, pytd.TypeDeclUnit):
return self._create_module(pyval)
elif isinstance(pyval, pytd.Module):
mod = self.vm.loader.import_name(pyval.module_name)
return self._create_module(mod)
elif isinstance(pyval, pytd.Class):
if pyval.name == "__builtin__.super":
return self.vm.special_builtins["super"]
elif pyval.name == "__builtin__.object":
return self.object_type
elif pyval.name == "types.ModuleType":
return self.module_type
elif pyval.name == "_importlib_modulespec.ModuleType":
# Python 3's typeshed uses a stub file indirection to define ModuleType
# even though it is exported via types.pyi.
return self.module_type
else:
module, dot, base_name = pyval.name.rpartition(".")
try:
cls = abstract.PyTDClass(base_name, pyval, self.vm)
except mro.MROError as e:
self.vm.errorlog.mro_error(self.vm.frames, base_name,
e.mro_seqs)
cls = self.unsolvable
else:
if dot:
cls.module = module
return cls
elif isinstance(pyval, pytd.Function):
signatures = [
abstract.PyTDSignature(pyval.name, sig, self.vm)
for sig in pyval.signatures
]
type_new = self.vm.lookup_builtin("__builtin__.type").Lookup(
"__new__")
if pyval is type_new:
f_cls = special_builtins.TypeNew
else:
f_cls = abstract.PyTDFunction
f = f_cls(pyval.name, signatures, pyval.kind, self.vm)
f.is_abstract = pyval.is_abstract
return f
elif isinstance(pyval, pytd.ClassType):
assert pyval.cls
return self.constant_to_value(pyval.cls, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.NothingType):
return self.empty
elif isinstance(pyval, pytd.AnythingType):
return self.unsolvable
elif (isinstance(pyval, pytd.Constant)
and isinstance(pyval.type, pytd.AnythingType)):
# We allow "X = ... # type: Any" to declare X as a type.
return self.unsolvable
elif isinstance(pyval, pytd.FunctionType):
return self.constant_to_value(pyval.function, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.UnionType):
options = [
self.constant_to_value(t, subst, self.vm.root_cfg_node)
for t in pyval.type_list
]
if len(options) > 1:
return abstract.Union(options, self.vm)
else:
return options[0]
elif isinstance(pyval, pytd.TypeParameter):
constraints = tuple(
self.constant_to_value(c, {}, self.vm.root_cfg_node)
for c in pyval.constraints)
bound = (pyval.bound and self.constant_to_value(
pyval.bound, {}, self.vm.root_cfg_node))
return abstract.TypeParameter(pyval.name,
self.vm,
constraints=constraints,
bound=bound)
elif isinstance(pyval, abstract.AsInstance):
cls = pyval.cls
if isinstance(cls, pytd.LateType):
actual = self._load_late_type(cls)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
cls = actual.cls
if isinstance(cls, pytd.ClassType):
cls = cls.cls
if (isinstance(cls, pytd.GenericType)
and cls.base_type.name == "typing.ClassVar"):
param, = cls.parameters
return self.constant_to_value(abstract.AsInstance(param),
subst, self.vm.root_cfg_node)
elif isinstance(cls,
pytd.GenericType) or (isinstance(cls, pytd.Class)
and cls.template):
# If we're converting a generic Class, need to create a new instance of
# it. See test_classes.testGenericReinstantiated.
if isinstance(cls, pytd.Class):
params = tuple(t.type_param.upper_value
for t in cls.template)
cls = pytd.GenericType(base_type=pytd.ClassType(
cls.name, cls),
parameters=params)
if isinstance(cls.base_type, pytd.LateType):
actual = self._load_late_type(cls.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base_cls = actual.cls
else:
assert isinstance(cls.base_type, pytd.ClassType)
base_cls = cls.base_type.cls
assert isinstance(base_cls, pytd.Class), base_cls
if base_cls.name == "__builtin__.type":
c, = cls.parameters
if isinstance(c, pytd.TypeParameter):
if not subst or c.name not in subst:
raise self.TypeParameterError(c.name)
return self.merge_classes(get_node(),
subst[c.name].data)
else:
return self.constant_to_value(c, subst,
self.vm.root_cfg_node)
elif isinstance(cls, pytd.TupleType):
content = tuple(
self.constant_to_var(abstract.AsInstance(p), subst,
get_node())
for p in cls.parameters)
return abstract.Tuple(content, self.vm)
elif isinstance(cls, pytd.CallableType):
clsval = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
return abstract.Instance(clsval, self.vm)
else:
clsval = self.constant_to_value(base_cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(clsval, self.vm)
num_params = len(cls.parameters)
assert num_params <= len(base_cls.template)
for i, formal in enumerate(base_cls.template):
if i < num_params:
node = get_node()
p = self.constant_to_var(
abstract.AsInstance(cls.parameters[i]), subst,
node)
else:
# An omitted type parameter implies `Any`.
node = self.vm.root_cfg_node
p = self.unsolvable.to_variable(node)
instance.merge_type_parameter(node, formal.name, p)
return instance
elif isinstance(cls, pytd.Class):
assert not cls.template
# This key is also used in __init__
key = (abstract.Instance, cls)
if key not in self._convert_cache:
if cls.name in [
"__builtin__.type", "__builtin__.property"
]:
# An instance of "type" or of an anonymous property can be anything.
instance = self._create_new_unknown_value("type")
else:
mycls = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(mycls, self.vm)
log.info("New pytd instance for %s: %r", cls.name,
instance)
self._convert_cache[key] = instance
return self._convert_cache[key]
else:
return self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
elif (isinstance(pyval, pytd.GenericType)
and pyval.base_type.name == "typing.ClassVar"):
param, = pyval.parameters
return self.constant_to_value(param, subst, self.vm.root_cfg_node)
elif isinstance(pyval, pytd.GenericType):
if isinstance(pyval.base_type, pytd.LateType):
actual = self._load_late_type(pyval.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base = actual.cls
else:
assert isinstance(pyval.base_type, pytd.ClassType)
base = pyval.base_type.cls
assert isinstance(base, pytd.Class), base
base_cls = self.constant_to_value(base, subst,
self.vm.root_cfg_node)
if not isinstance(base_cls, abstract.Class):
# base_cls can be, e.g., an unsolvable due to an mro error.
return self.unsolvable
if isinstance(pyval, pytd.TupleType):
abstract_class = abstract.TupleClass
template = list(range(len(pyval.parameters))) + [abstract.T]
parameters = pyval.parameters + (pytd.UnionType(
pyval.parameters), )
elif isinstance(pyval, pytd.CallableType):
abstract_class = abstract.Callable
template = list(range(len(
pyval.args))) + [abstract.ARGS, abstract.RET]
parameters = pyval.args + (pytd_utils.JoinTypes(
pyval.args), pyval.ret)
else:
abstract_class = abstract.ParameterizedClass
template = tuple(t.name for t in base.template)
parameters = pyval.parameters
assert (pyval.base_type.name == "typing.Generic"
or len(parameters) <= len(template))
type_parameters = datatypes.LazyDict()
for i, name in enumerate(template):
if i < len(parameters):
type_parameters.add_lazy_item(name, self.constant_to_value,
parameters[i], subst,
self.vm.root_cfg_node)
else:
type_parameters[name] = self.unsolvable
return abstract_class(base_cls, type_parameters, self.vm)
elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node
return self.tuple_to_value([
self.constant_to_var(item, subst, self.vm.root_cfg_node)
for i, item in enumerate(pyval)
])
else:
raise NotImplementedError("Can't convert constant %s %r" %
(type(pyval), pyval))
def value_to_pytd_type(self, node, v, seen, view):
"""Get a PyTD type representing this object, as seen at a node.
Args:
node: The node from which we want to observe this object.
v: The object.
seen: The set of values seen before while computing the type.
view: A Variable -> binding map.
Returns:
A PyTD type.
"""
if isinstance(v, (abstract.Empty, typing_overlay.NoReturn)):
return pytd.NothingType()
elif isinstance(v, abstract.TypeParameterInstance):
if v.instance.get_instance_type_parameter(v.full_name).bindings:
# The type parameter was initialized. Set the view to None, since we
# don't include v.instance in the view.
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, p, seen, None)
for p in v.instance.get_instance_type_parameter(v.full_name).data)
elif v.param.constraints:
return pytd_utils.JoinTypes(
self.value_instance_to_pytd_type(node, p, None, seen, view)
for p in v.param.constraints)
elif v.param.bound:
return self.value_instance_to_pytd_type(
node, v.param.bound, None, seen, view)
else:
return pytd.AnythingType()
elif isinstance(v, typing_overlay.TypeVar):
return pytd.NamedType("__builtin__.type")
elif isinstance(v, abstract.FUNCTION_TYPES):
try:
signatures = abstract_utils.get_signatures(v)
except NotImplementedError:
return pytd.NamedType("typing.Callable")
if len(signatures) == 1:
val = self.signature_to_callable(signatures[0], self.vm)
if (not isinstance(v, abstract.PYTD_FUNCTION_TYPES) or
not self.vm.annotations_util.get_type_parameters(val)):
# This is a workaround to make sure we don't put unexpected type
# parameters in call traces.
return self.value_instance_to_pytd_type(node, val, None, seen, view)
return pytd.NamedType("typing.Callable")
elif isinstance(v, (abstract.ClassMethod, abstract.StaticMethod)):
return self.value_to_pytd_type(node, v.method, seen, view)
elif isinstance(v, (special_builtins.IsInstance,
special_builtins.ClassMethodCallable)):
return pytd.NamedType("typing.Callable")
elif isinstance(v, mixin.Class):
param = self.value_instance_to_pytd_type(node, v, None, seen, view)
return pytd.GenericType(base_type=pytd.NamedType("__builtin__.type"),
parameters=(param,))
elif isinstance(v, abstract.Module):
return pytd.NamedType("__builtin__.module")
elif isinstance(v, abstract.SimpleAbstractValue):
if v.cls:
ret = self.value_instance_to_pytd_type(
node, v.cls, v, seen=seen, view=view)
ret.Visit(visitors.FillInLocalPointers(
{"__builtin__": self.vm.loader.builtins}))
return ret
else:
# We don't know this type's __class__, so return AnythingType to
# indicate that we don't know anything about what this is.
# This happens e.g. for locals / globals, which are returned from the
# code in class declarations.
log.info("Using ? for %s", v.name)
return pytd.AnythingType()
elif isinstance(v, abstract.Union):
return pytd.UnionType(tuple(self.value_to_pytd_type(node, o, seen, view)
for o in v.options))
elif isinstance(v, special_builtins.SuperInstance):
return pytd.NamedType("__builtin__.super")
elif isinstance(v, (abstract.Unsolvable, abstract.TypeParameter)):
# Arguably, the type of a type parameter is NamedType("typing.TypeVar"),
# but pytype doesn't know how to handle that, so let's just go with Any.
return pytd.AnythingType()
elif isinstance(v, abstract.Unknown):
return pytd.NamedType(v.class_name)
elif isinstance(v, abstract.BuildClass):
return pytd.NamedType("typing.Callable")
else:
raise NotImplementedError(v.__class__.__name__)
def VisitUnionType(self, node):
return pytd.UnionType(tuple(sorted(node.type_list)))
def new_union_type(self, types):
"""Return a new UnionType composed of the specified types."""
# UnionType flattens any contained UnionType's.
return pytd.UnionType(tuple(types))
def testAnyReplacement(self):
union = pytd.UnionType((pytd.NamedType("a"), pytd.NamedType("b")))
self.assertEqual(
union.Visit(visitors.ReplaceUnionsWithAny()), pytd.AnythingType())
