def testSetsAdd(self):
ty = self.Infer("""
def f():
x = set([])
x.add(1)
x.add(10)
return x
f()
""",
deep=False,
show_library_calls=True)
self.assertHasOnlySignatures(
ty.Lookup("f"), ((), pytd.GenericType(self.set, (self.int, ))))
def test_list_concat_unlike(self):
ty = self.Infer("""
def f():
x = []
x.append(1)
x.append(2)
x.append(3)
return ["str"] + x
f()
""", deep=False, show_library_calls=True)
self.assertHasOnlySignatures(
ty.Lookup("f"),
((), pytd.GenericType(self.list, (self.intorstr,))))
def _parameterized_type(self, base_type, parameters):
"""Return a parameterized type."""
if self._is_any(base_type):
return pytd.AnythingType()
elif len(parameters) == 2 and parameters[-1] is self.ELLIPSIS and (
not self._is_callable_base_type(base_type)):
element_type = parameters[0]
if element_type is self.ELLIPSIS:
raise ParseError("[..., ...] not supported")
return pytd.GenericType(base_type=base_type,
parameters=(element_type,))
else:
parameters = tuple(pytd.AnythingType() if p is self.ELLIPSIS else p
for p in parameters)
if self._is_tuple_base_type(base_type):
return self._heterogeneous_tuple(base_type, parameters)
elif (self._is_callable_base_type(base_type) and
self._is_heterogeneous_tuple(parameters[0])):
if len(parameters) > 2:
raise ParseError(
"Expected 2 parameters to Callable, got %d" % len(parameters))
if len(parameters) == 1:
# We're usually happy to treat omitted parameters as "Any", but we
# need a return type for CallableType, or we wouldn't know whether the
# last parameter is an argument or return type.
parameters += (pytd.AnythingType(),)
if self._is_empty_tuple(parameters[0]):
parameters = parameters[1:]
else:
parameters = parameters[0].parameters + parameters[1:]
return pytd.CallableType(base_type=base_type, parameters=parameters)
else:
assert parameters
if (self._is_callable_base_type(base_type) and
not self._is_any(parameters[0])):
raise ParseError(
"First argument to Callable must be a list of argument types")
return pytd.GenericType(base_type=base_type, parameters=parameters)
def testListConcatMultiType(self):
ty = self.Infer("""
def f():
x = []
x.append(1)
x.append("str")
return x + [1.3] + x
f()
""",
deep=False,
show_library_calls=True)
self.assertHasOnlySignatures(
ty.Lookup("f"), ((),
pytd.GenericType(self.list, (pytd.UnionType(
(self.int, self.float, self.str)), ))))
def test_mutual_recursion(self):
ast = self._import(a="""
from typing import List
X = List[Y]
Y = List[X]
""")
actual_x = ast.Lookup("a.X")
expected_x = pytd.Alias(name="a.X",
type=pytd.GenericType(
base_type=pytd.ClassType("builtins.list"),
parameters=(pytd.LateType(
"a.Y", recursive=True), )))
self.assertEqual(actual_x, expected_x)
actual_y = ast.Lookup("a.Y")
expected_y = pytd.Alias(
name="a.Y",
type=pytd.GenericType(
base_type=pytd.ClassType("builtins.list"),
parameters=(pytd.GenericType(
base_type=pytd.ClassType("builtins.list"),
parameters=(pytd.LateType("a.Y", recursive=True), )), )))
self.assertEqual(actual_y, expected_y)
def testAliasPrinting(self):
a = pytd.Alias("MyList", pytd.GenericType(
pytd.NamedType("typing.List"), (pytd.AnythingType(),)))
ty = pytd.TypeDeclUnit(
name="test",
constants=(),
type_params=(),
classes=(),
functions=(),
aliases=(a,))
expected = textwrap.dedent("""
from typing import Any, List
MyList = List[Any]""")
self.assertMultiLineEqual(expected.strip(), pytd.Print(ty).strip())
def testOrder(self):
# pytd types' primary sort key is the class name, second sort key is
# the contents when interpreted as a (named)tuple.
nodes = [pytd.AnythingType(),
pytd.GenericType(self.list, (self.int,)),
pytd.NamedType("int"),
pytd.NothingType(),
pytd.UnionType((self.float,)),
pytd.UnionType((self.int,))]
for n1, n2 in zip(nodes[:-1], nodes[1:]):
self.assertLess(n1, n2)
self.assertLessEqual(n1, n2)
self.assertGreater(n2, n1)
self.assertGreaterEqual(n2, n1)
for p in itertools.permutations(nodes):
self.assertEquals(list(sorted(p)), nodes)
def test_sets(self):
ty = self.Infer("""
def f():
x = set([1,2,3])
if x:
x = x | set()
y = x
return x
else:
x.add(10)
return x
f()
""", deep=False, show_library_calls=True)
self.assertHasOnlySignatures(
ty.Lookup("f"),
((), pytd.GenericType(self.set, (self.int,))))
def p_type_homogeneous(self, p):
"""type : named_or_external_type LBRACKET parameters RBRACKET"""
_, base_type, _, parameters, _ = p
if p[1] == pytd.NamedType('Union'):
p[0] = pytd.UnionType(parameters)
elif p[1] == pytd.NamedType('Optional'):
p[0] = pytd.UnionType(parameters[0], pytd.NamedType('None'))
elif len(parameters) == 2 and parameters[-1] is Ellipsis:
element_type, _ = parameters
if element_type is Ellipsis:
make_syntax_error(self, '[..., ...] not supported', p)
p[0] = pytd.HomogeneousContainerType(base_type=base_type,
parameters=(element_type, ))
else:
parameters = tuple(pytd.AnythingType() if p is Ellipsis else p
for p in parameters)
p[0] = pytd.GenericType(base_type=base_type, parameters=parameters)
def _pytd_return_type(name: str, return_type: Optional[pytd_node.Node],
is_async: bool) -> pytd_node.Node:
"""Convert function return type to pytd."""
if name == "__init__":
if (return_type is None or isinstance(return_type, pytd.AnythingType)):
ret = pytd.NamedType("NoneType")
else:
ret = return_type
elif is_async:
base = pytd.NamedType("typing.Coroutine")
params = (pytd.AnythingType(), pytd.AnythingType(), return_type)
ret = pytd.GenericType(base, params)
elif return_type is None:
ret = pytd.AnythingType()
else:
ret = return_type
return ret
def _parameterized_type(self, base_type, parameters):
"""Return a parameterized type."""
if base_type == pytd.NamedType("typing.Callable"):
# TODO(kramm): Support Callable[[params], ret].
return base_type
elif len(parameters) == 2 and parameters[-1] is self.ELLIPSIS:
element_type = parameters[0]
if element_type is self.ELLIPSIS:
raise ParseError("[..., ...] not supported")
return pytd.HomogeneousContainerType(base_type=base_type,
parameters=(element_type, ))
else:
parameters = tuple(pytd.AnythingType() if p is self.ELLIPSIS else p
for p in parameters)
if self._is_tuple_base_type(base_type):
return self._heterogeneous_tuple(base_type, parameters)
else:
assert parameters
return pytd.GenericType(base_type=base_type,
parameters=parameters)
def test_signature_annotations(self):
# def f(self: Any, *args: Any)
self_param = pytd.Parameter("self", pytd.AnythingType(), False, False, None)
# Imitate the parser's conversion of '*args: Any' to '*args: Tuple[Any]'.
tup = pytd.ClassType("__builtin__.tuple")
tup.cls = self._vm.convert.tuple_type.pytd_cls
any_tuple = pytd.GenericType(tup, (pytd.AnythingType(),))
args_param = pytd.Parameter("args", any_tuple, False, True, None)
sig = function.Signature.from_pytd(
self._vm, "f", pytd.Signature(
(self_param,), args_param, None, pytd.AnythingType(), (), ()))
self.assertEqual(repr(sig),
"def f(self: Any, *args: Tuple[Any, ...]) -> Any")
self.assertIs(sig.annotations["self"], self._vm.convert.unsolvable)
args_type = sig.annotations["args"]
self.assertIsInstance(args_type, abstract.ParameterizedClass)
self.assertIs(args_type.base_cls, self._vm.convert.tuple_type)
self.assertListEqual(list(args_type.formal_type_parameters.items()),
[(abstract_utils.T, self._vm.convert.unsolvable)])
self.assertIs(sig.drop_first_parameter().annotations["args"], args_type)
def testVerifyHeterogeneousTuple(self):
# Error: does not inherit from Generic
base = pytd.ClassType("tuple")
base.cls = pytd.Class("tuple", None, (), (), (), (), None, ())
t1 = pytd.TupleType(base, (pytd.NamedType("str"), pytd.NamedType("float")))
self.assertRaises(visitors.ContainerError,
lambda: t1.Visit(visitors.VerifyContainers()))
# Error: Generic[str, float]
gen = pytd.ClassType("typing.Generic")
gen.cls = pytd.Class("typing.Generic", None, (), (), (), (), None, ())
t2 = pytd.TupleType(gen, (pytd.NamedType("str"), pytd.NamedType("float")))
self.assertRaises(visitors.ContainerError,
lambda: t2.Visit(visitors.VerifyContainers()))
# Okay
param = pytd.TypeParameter("T")
parent = pytd.GenericType(gen, (param,))
base.cls = pytd.Class(
"tuple", None, (parent,), (), (), (), None, (pytd.TemplateItem(param),))
t3 = pytd.TupleType(base, (pytd.NamedType("str"), pytd.NamedType("float")))
t3.Visit(visitors.VerifyContainers())
def new_typed_dict(self, name, items, total):
"""Returns a type for a TypedDict.
This method is currently called only for TypedDict objects defined via
the following function-based syntax:
Foo = TypedDict('Foo', {'a': int, 'b': str}, total=False)
rather than the recommended class-based syntax.
Args:
name: the name of the TypedDict instance, e.g., "'Foo'".
items: a {key: value_type} dict, e.g., {"'a'": "int", "'b'": "str"}.
total: A tuple of a single kwarg, e.g., ("total", NamedType("False")), or
None when no kwarg is passed.
"""
# TODO(b/157603915): Add real support for TypedDict.
del name, items, total # unused
return pytd.GenericType(
pytd.NamedType("typing.Dict"),
(pytd.NamedType("str"), pytd.NamedType("typing.Any")))
def _namedtuple_new(self, name, fields):
"""Build a __new__ method for a namedtuple with the given fields.
For a namedtuple defined as NamedTuple("_", [("foo", int), ("bar", str)]),
generates the method
def __new__(cls: Type[_T], foo: int, bar: str) -> _T: ...
where _T is a TypeVar bounded by the class type.
Args:
name: The class name.
fields: A list of (name, type) pairs representing the namedtuple fields.
Returns:
A _NameAndSig object for a __new__ method.
"""
type_param = pytd.TypeParameter("_T" + name, bound=pytd.NamedType(name))
self._type_params.append(type_param)
cls_arg = (
"cls", pytd.GenericType(pytd.NamedType("type"), (type_param,)), None)
args = [cls_arg] + [(n, t, None) for n, t in fields]
return self.new_function((), "__new__", args, type_param, ())
def testSelf(self):
"""Test handling of self."""
data = textwrap.dedent("""
U = TypeVar('U')
V = TypeVar('V')
class MyClass(typing.Generic[U, V], object):
def f1(self) -> ?
""")
result = self.Parse(data)
myclass = result.Lookup("MyClass")
self.assertEquals([t.name for t in myclass.template], ["U", "V"])
f = myclass.Lookup("f1").signatures[0]
self_param = f.params[0]
self.assertEquals(self_param.name, "self")
u, v = myclass.template
self.assertEquals(
self_param.type,
pytd.GenericType(pytd.NamedType("MyClass"),
(u.type_param, v.type_param)))
def convert_string_type(string_type, unknown, mapping, global_lookup, depth=0):
"""Convert a string representing a type back to a pytd type."""
try:
# Check whether this is a type declared in a pytd.
cls = global_lookup.Lookup(string_type)
base_type = pytd_utils.NamedOrClassType(cls.name, cls)
except KeyError:
# If we don't have a pytd for this type, it can't be a template.
cls = None
base_type = pytd_utils.NamedOrClassType(string_type, cls)
if cls and cls.template:
parameters = []
for t in cls.template:
type_param_name = unknown + "." + string_type + "." + t.name
if type_param_name in mapping and depth < MAX_DEPTH:
string_type_params = mapping[type_param_name]
parameters.append(convert_string_type_list(
string_type_params, unknown, mapping, global_lookup, depth + 1))
else:
parameters.append(pytd.AnythingType())
return pytd.GenericType(base_type, tuple(parameters))
else:
return base_type
def testNamedAgainstGeneric(self):
m = type_match.TypeMatch({})
eq = m.match_type_against_type(pytd.GenericType(pytd.NamedType("A"), ()),
pytd.NamedType("A"), {})
self.assertEqual(eq, booleq.TRUE)
def test_convert_union(self):
t = pytd.GenericType(pytd.NamedType("typing.Union"),
(pytd.NamedType("str"), pytd.NamedType("float")))
self.assertEquals(self.convert(t), "Union[str, float]")
def test_convert_optional(self):
t = pytd.GenericType(pytd.NamedType("typing.Optional"),
(pytd.NamedType("str"), ))
self.assertEquals(self.convert(t), "Union[str, None]")
def setUpClass(cls):
super().setUpClass()
# We use class-wide loader to avoid creating a new loader for every test
# method if not required.
cls._loader = None
def t(name): # pylint: disable=invalid-name
return pytd.ClassType("builtins." + name)
cls.bool = t("bool")
cls.dict = t("dict")
cls.float = t("float")
cls.complex = t("complex")
cls.int = t("int")
cls.list = t("list")
cls.none_type = t("NoneType")
cls.object = t("object")
cls.set = t("set")
cls.frozenset = t("frozenset")
cls.str = t("str")
cls.bytearray = t("bytearray")
cls.tuple = t("tuple")
cls.unicode = t("unicode")
cls.generator = t("generator")
cls.function = pytd.ClassType("typing.Callable")
cls.anything = pytd.AnythingType()
cls.nothing = pytd.NothingType()
cls.module = t("module")
cls.file = t("file")
# The various union types use pytd_utils.CanonicalOrdering()'s ordering:
cls.intorstr = pytd.UnionType((cls.int, cls.str))
cls.strorunicode = pytd.UnionType((cls.str, cls.unicode))
cls.intorfloat = pytd.UnionType((cls.float, cls.int))
cls.intorfloatorstr = pytd.UnionType((cls.float, cls.int, cls.str))
cls.complexorstr = pytd.UnionType((cls.complex, cls.str))
cls.intorfloatorcomplex = pytd.UnionType(
(cls.int, cls.float, cls.complex))
cls.int_tuple = pytd.GenericType(cls.tuple, (cls.int, ))
cls.nothing_tuple = pytd.TupleType(cls.tuple, ())
cls.intorfloat_tuple = pytd.GenericType(cls.tuple, (cls.intorfloat, ))
cls.int_set = pytd.GenericType(cls.set, (cls.int, ))
cls.intorfloat_set = pytd.GenericType(cls.set, (cls.intorfloat, ))
cls.unknown_frozenset = pytd.GenericType(cls.frozenset,
(cls.anything, ))
cls.float_frozenset = pytd.GenericType(cls.frozenset, (cls.float, ))
cls.empty_frozenset = pytd.GenericType(cls.frozenset, (cls.nothing, ))
cls.int_list = pytd.GenericType(cls.list, (cls.int, ))
cls.str_list = pytd.GenericType(cls.list, (cls.str, ))
cls.intorfloat_list = pytd.GenericType(cls.list, (cls.intorfloat, ))
cls.intorstr_list = pytd.GenericType(cls.list, (cls.intorstr, ))
cls.anything_list = pytd.GenericType(cls.list, (cls.anything, ))
cls.nothing_list = pytd.GenericType(cls.list, (cls.nothing, ))
cls.int_int_dict = pytd.GenericType(cls.dict, (cls.int, cls.int))
cls.int_str_dict = pytd.GenericType(cls.dict, (cls.int, cls.str))
cls.str_int_dict = pytd.GenericType(cls.dict, (cls.str, cls.int))
cls.nothing_nothing_dict = pytd.GenericType(cls.dict,
(cls.nothing, cls.nothing))
def pytd_list(typ: str) -> pytd_node.Node:
if typ:
return pytd.GenericType(
pytd.NamedType("typing.List"), (pytd.NamedType(typ),))
else:
return pytd.NamedType("typing.List")
def p_type_tuple(self, p):
# Used for function types, e.g. # Callable[[args...], return]
"""type : LBRACKET type_list RBRACKET"""
p[0] = pytd.GenericType(pytd.NamedType('tuple'), tuple(p[2]))
def p_type_generic(self, p):
"""type : named_or_external_type LBRACKET parameters COMMA RBRACKET"""
_, base_type, _, parameters, _, _ = p
p[0] = pytd.GenericType(base_type=base_type, parameters=parameters)
def pytd_type(value: pytd_node.Node) -> pytd_node.Node:
return pytd.GenericType(pytd.NamedType("type"), (value,))
def value_to_pytd_type(self, node, v, seen, view):
"""Get a PyTD type representing this object, as seen at a node.
Args:
node: The node from which we want to observe this object.
v: The object.
seen: The set of values seen before while computing the type.
view: A Variable -> binding map.
Returns:
A PyTD type.
"""
if isinstance(v, (abstract.Empty, typing_overlay.NoReturn)):
return pytd.NothingType()
elif isinstance(v, abstract.TypeParameterInstance):
if v.module in self._scopes:
return self._typeparam_to_def(node, v.param, v.param.name)
elif v.instance.get_instance_type_parameter(v.full_name).bindings:
# The type parameter was initialized. Set the view to None, since we
# don't include v.instance in the view.
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, p, seen, None) for p in
v.instance.get_instance_type_parameter(v.full_name).data)
elif v.param.constraints:
return pytd_utils.JoinTypes(
self.value_instance_to_pytd_type(node, p, None, seen, view)
for p in v.param.constraints)
elif v.param.bound:
return self.value_instance_to_pytd_type(
node, v.param.bound, None, seen, view)
else:
return pytd.AnythingType()
elif isinstance(v, typing_overlay.TypeVar):
return pytd.NamedType("builtins.type")
elif isinstance(v, dataclass_overlay.FieldInstance):
if not v.default:
return pytd.AnythingType()
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, d, seen, view)
for d in v.default.data)
elif isinstance(v, abstract.FUNCTION_TYPES):
try:
signatures = abstract_utils.get_signatures(v)
except NotImplementedError:
return pytd.NamedType("typing.Callable")
if len(signatures) == 1:
val = self.signature_to_callable(signatures[0])
if not isinstance(
v, abstract.PYTD_FUNCTION_TYPES) or not val.formal:
# This is a workaround to make sure we don't put unexpected type
# parameters in call traces.
return self.value_instance_to_pytd_type(
node, val, None, seen, view)
return pytd.NamedType("typing.Callable")
elif isinstance(v, (abstract.ClassMethod, abstract.StaticMethod)):
return self.value_to_pytd_type(node, v.method, seen, view)
elif isinstance(v, (special_builtins.IsInstance,
special_builtins.ClassMethodCallable)):
return pytd.NamedType("typing.Callable")
elif isinstance(v, class_mixin.Class):
param = self.value_instance_to_pytd_type(node, v, None, seen, view)
return pytd.GenericType(base_type=pytd.NamedType("builtins.type"),
parameters=(param, ))
elif isinstance(v, abstract.Module):
return pytd.NamedType("builtins.module")
elif (self._output_mode >= Converter.OutputMode.LITERAL
and isinstance(v, abstract.ConcreteValue)
and isinstance(v.pyval, (int, str, bytes))):
# LITERAL mode is used only for pretty-printing, so we just stringify the
# inner value rather than properly converting it.
return pytd.Literal(repr(v.pyval))
elif isinstance(v, abstract.SimpleValue):
if v.cls:
ret = self.value_instance_to_pytd_type(node,
v.cls,
v,
seen=seen,
view=view)
ret.Visit(
visitors.FillInLocalPointers(
{"builtins": self.vm.loader.builtins}))
return ret
else:
# We don't know this type's __class__, so return AnythingType to
# indicate that we don't know anything about what this is.
# This happens e.g. for locals / globals, which are returned from the
# code in class declarations.
log.info("Using Any for %s", v.name)
return pytd.AnythingType()
elif isinstance(v, abstract.Union):
opts = []
for o in v.options:
# NOTE: Guarding printing of type parameters behind _detailed until
# round-tripping is working properly.
if self._detailed and isinstance(o, abstract.TypeParameter):
opt = self._typeparam_to_def(node, o, o.name)
else:
opt = self.value_to_pytd_type(node, o, seen, view)
opts.append(opt)
return pytd.UnionType(tuple(opts))
elif isinstance(v, special_builtins.SuperInstance):
return pytd.NamedType("builtins.super")
elif isinstance(v, abstract.TypeParameter):
# Arguably, the type of a type parameter is NamedType("typing.TypeVar"),
# but pytype doesn't know how to handle that, so let's just go with Any
# unless self._detailed is set.
if self._detailed:
return pytd.NamedType("typing.TypeVar")
else:
return pytd.AnythingType()
elif isinstance(v, abstract.Unsolvable):
return pytd.AnythingType()
elif isinstance(v, abstract.Unknown):
return pytd.NamedType(v.class_name)
elif isinstance(v, abstract.BuildClass):
return pytd.NamedType("typing.Callable")
else:
raise NotImplementedError(v.__class__.__name__)
def _constant_to_value(self, pyval, subst, get_node):
"""Create a AtomicAbstractValue that represents a python constant.
This supports both constant from code constant pools and PyTD constants such
as classes. This also supports builtin python objects such as int and float.
Args:
pyval: The python or PyTD value to convert.
subst: The current type parameters.
get_node: A getter function for the current node.
Returns:
A Value that represents the constant, or None if we couldn't convert.
Raises:
NotImplementedError: If we don't know how to convert a value.
TypeParameterError: If we can't find a substitution for a type parameter.
"""
if pyval.__class__ is str:
# We use a subclass of str, compat.BytesPy3, to mark Python 3
# bytestrings, which are converted to abstract bytes instances.
# compat.BytesType dispatches to this when appropriate.
return abstract.AbstractOrConcreteValue(pyval, self.str_type,
self.vm)
elif isinstance(pyval, compat.UnicodeType):
return abstract.AbstractOrConcreteValue(pyval, self.unicode_type,
self.vm)
elif isinstance(pyval, compat.BytesType):
return abstract.AbstractOrConcreteValue(pyval, self.bytes_type,
self.vm)
elif isinstance(pyval, bool):
return self.true if pyval is True else self.false
elif isinstance(pyval, int) and -1 <= pyval <= MAX_IMPORT_DEPTH:
# For small integers, preserve the actual value (for things like the
# level in IMPORT_NAME).
return abstract.AbstractOrConcreteValue(pyval, self.int_type,
self.vm)
elif isinstance(pyval, compat.LongType):
# long is aliased to int
return self.primitive_class_instances[int]
elif pyval.__class__ in self.primitive_classes:
return self.primitive_class_instances[pyval.__class__]
elif isinstance(pyval, (loadmarshal.CodeType, blocks.OrderedCode)):
return abstract.AbstractOrConcreteValue(
pyval, self.primitive_classes[types.CodeType], self.vm)
elif pyval is super:
return special_builtins.Super(self.vm)
elif pyval is object:
return special_builtins.Object(self.vm)
elif pyval.__class__ is type:
try:
return self.name_to_value(self._type_to_name(pyval), subst)
except (KeyError, AttributeError):
log.debug("Failed to find pytd", exc_info=True)
raise
elif isinstance(pyval, pytd.LateType):
actual = self._load_late_type(pyval)
return self._constant_to_value(actual, subst, get_node)
elif isinstance(pyval, pytd.TypeDeclUnit):
return self._create_module(pyval)
elif isinstance(pyval, pytd.Module):
mod = self.vm.loader.import_name(pyval.module_name)
return self._create_module(mod)
elif isinstance(pyval, pytd.Class):
if pyval.name == "__builtin__.super":
return self.vm.special_builtins["super"]
elif pyval.name == "__builtin__.object":
return self.object_type
elif pyval.name == "types.ModuleType":
return self.module_type
elif pyval.name == "_importlib_modulespec.ModuleType":
# Python 3's typeshed uses a stub file indirection to define ModuleType
# even though it is exported via types.pyi.
return self.module_type
else:
module, dot, base_name = pyval.name.rpartition(".")
try:
cls = abstract.PyTDClass(base_name, pyval, self.vm)
except mro.MROError as e:
self.vm.errorlog.mro_error(self.vm.frames, base_name,
e.mro_seqs)
cls = self.unsolvable
else:
if dot:
cls.module = module
return cls
elif isinstance(pyval, pytd.Function):
signatures = [
abstract.PyTDSignature(pyval.name, sig, self.vm)
for sig in pyval.signatures
]
type_new = self.vm.lookup_builtin("__builtin__.type").Lookup(
"__new__")
if pyval is type_new:
f_cls = special_builtins.TypeNew
else:
f_cls = abstract.PyTDFunction
f = f_cls(pyval.name, signatures, pyval.kind, self.vm)
f.is_abstract = pyval.is_abstract
return f
elif isinstance(pyval, pytd.ClassType):
assert pyval.cls
return self.constant_to_value(pyval.cls, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.NothingType):
return self.empty
elif isinstance(pyval, pytd.AnythingType):
return self.unsolvable
elif (isinstance(pyval, pytd.Constant)
and isinstance(pyval.type, pytd.AnythingType)):
# We allow "X = ... # type: Any" to declare X as a type.
return self.unsolvable
elif isinstance(pyval, pytd.FunctionType):
return self.constant_to_value(pyval.function, subst,
self.vm.root_cfg_node)
elif isinstance(pyval, pytd.UnionType):
options = [
self.constant_to_value(t, subst, self.vm.root_cfg_node)
for t in pyval.type_list
]
if len(options) > 1:
return abstract.Union(options, self.vm)
else:
return options[0]
elif isinstance(pyval, pytd.TypeParameter):
constraints = tuple(
self.constant_to_value(c, {}, self.vm.root_cfg_node)
for c in pyval.constraints)
bound = (pyval.bound and self.constant_to_value(
pyval.bound, {}, self.vm.root_cfg_node))
return abstract.TypeParameter(pyval.name,
self.vm,
constraints=constraints,
bound=bound)
elif isinstance(pyval, abstract.AsInstance):
cls = pyval.cls
if isinstance(cls, pytd.LateType):
actual = self._load_late_type(cls)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
cls = actual.cls
if isinstance(cls, pytd.ClassType):
cls = cls.cls
if (isinstance(cls, pytd.GenericType)
and cls.base_type.name == "typing.ClassVar"):
param, = cls.parameters
return self.constant_to_value(abstract.AsInstance(param),
subst, self.vm.root_cfg_node)
elif isinstance(cls,
pytd.GenericType) or (isinstance(cls, pytd.Class)
and cls.template):
# If we're converting a generic Class, need to create a new instance of
# it. See test_classes.testGenericReinstantiated.
if isinstance(cls, pytd.Class):
params = tuple(t.type_param.upper_value
for t in cls.template)
cls = pytd.GenericType(base_type=pytd.ClassType(
cls.name, cls),
parameters=params)
if isinstance(cls.base_type, pytd.LateType):
actual = self._load_late_type(cls.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base_cls = actual.cls
else:
assert isinstance(cls.base_type, pytd.ClassType)
base_cls = cls.base_type.cls
assert isinstance(base_cls, pytd.Class), base_cls
if base_cls.name == "__builtin__.type":
c, = cls.parameters
if isinstance(c, pytd.TypeParameter):
if not subst or c.name not in subst:
raise self.TypeParameterError(c.name)
return self.merge_classes(get_node(),
subst[c.name].data)
else:
return self.constant_to_value(c, subst,
self.vm.root_cfg_node)
elif isinstance(cls, pytd.TupleType):
content = tuple(
self.constant_to_var(abstract.AsInstance(p), subst,
get_node())
for p in cls.parameters)
return abstract.Tuple(content, self.vm)
elif isinstance(cls, pytd.CallableType):
clsval = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
return abstract.Instance(clsval, self.vm)
else:
clsval = self.constant_to_value(base_cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(clsval, self.vm)
num_params = len(cls.parameters)
assert num_params <= len(base_cls.template)
for i, formal in enumerate(base_cls.template):
if i < num_params:
node = get_node()
p = self.constant_to_var(
abstract.AsInstance(cls.parameters[i]), subst,
node)
else:
# An omitted type parameter implies `Any`.
node = self.vm.root_cfg_node
p = self.unsolvable.to_variable(node)
instance.merge_type_parameter(node, formal.name, p)
return instance
elif isinstance(cls, pytd.Class):
assert not cls.template
# This key is also used in __init__
key = (abstract.Instance, cls)
if key not in self._convert_cache:
if cls.name in [
"__builtin__.type", "__builtin__.property"
]:
# An instance of "type" or of an anonymous property can be anything.
instance = self._create_new_unknown_value("type")
else:
mycls = self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
instance = abstract.Instance(mycls, self.vm)
log.info("New pytd instance for %s: %r", cls.name,
instance)
self._convert_cache[key] = instance
return self._convert_cache[key]
else:
return self.constant_to_value(cls, subst,
self.vm.root_cfg_node)
elif (isinstance(pyval, pytd.GenericType)
and pyval.base_type.name == "typing.ClassVar"):
param, = pyval.parameters
return self.constant_to_value(param, subst, self.vm.root_cfg_node)
elif isinstance(pyval, pytd.GenericType):
if isinstance(pyval.base_type, pytd.LateType):
actual = self._load_late_type(pyval.base_type)
if not isinstance(actual, pytd.ClassType):
return self.unsolvable
base = actual.cls
else:
assert isinstance(pyval.base_type, pytd.ClassType)
base = pyval.base_type.cls
assert isinstance(base, pytd.Class), base
base_cls = self.constant_to_value(base, subst,
self.vm.root_cfg_node)
if not isinstance(base_cls, abstract.Class):
# base_cls can be, e.g., an unsolvable due to an mro error.
return self.unsolvable
if isinstance(pyval, pytd.TupleType):
abstract_class = abstract.TupleClass
template = list(range(len(pyval.parameters))) + [abstract.T]
parameters = pyval.parameters + (pytd.UnionType(
pyval.parameters), )
elif isinstance(pyval, pytd.CallableType):
abstract_class = abstract.Callable
template = list(range(len(
pyval.args))) + [abstract.ARGS, abstract.RET]
parameters = pyval.args + (pytd_utils.JoinTypes(
pyval.args), pyval.ret)
else:
abstract_class = abstract.ParameterizedClass
template = tuple(t.name for t in base.template)
parameters = pyval.parameters
assert (pyval.base_type.name == "typing.Generic"
or len(parameters) <= len(template))
type_parameters = datatypes.LazyDict()
for i, name in enumerate(template):
if i < len(parameters):
type_parameters.add_lazy_item(name, self.constant_to_value,
parameters[i], subst,
self.vm.root_cfg_node)
else:
type_parameters[name] = self.unsolvable
return abstract_class(base_cls, type_parameters, self.vm)
elif pyval.__class__ is tuple: # only match raw tuple, not namedtuple/Node
return self.tuple_to_value([
self.constant_to_var(item, subst, self.vm.root_cfg_node)
for i, item in enumerate(pyval)
])
else:
raise NotImplementedError("Can't convert constant %s %r" %
(type(pyval), pyval))
def VisitUnionType(self, union):
"""Push unions down into containers.
This collects similar container types in unions and merges them into
single instances with the union type pushed down to the element_type level.
Arguments:
union: A pytd.Union instance. Might appear in a parameter, a return type,
a constant type, etc.
Returns:
A simplified pytd.Union.
"""
if not any(isinstance(t, pytd.GenericType) for t in union.type_list):
# Optimization: If we're not going to change anything, return original.
return union
union = pytd_utils.JoinTypes(union.type_list) # flatten
if not isinstance(union, pytd.UnionType):
union = pytd.UnionType((union, ))
merge_tuples = self._should_merge(pytd.TupleType, union)
merge_callables = self._should_merge(pytd.CallableType, union)
if merge_tuples or merge_callables:
type_list = []
for t in union.type_list:
if merge_tuples and isinstance(t, pytd.TupleType):
t = pytd.GenericType(base_type=t.base_type,
parameters=(pytd.UnionType(
t.parameters), ))
elif merge_callables and isinstance(t, pytd.CallableType):
t = pytd.GenericType(base_type=t.base_type,
parameters=(pytd.AnythingType(),
t.ret))
type_list.append(t)
union = union.Replace(type_list=tuple(type_list))
collect = {}
has_redundant_base_types = False
for t in union.type_list:
if isinstance(t, pytd.GenericType):
key = self._key(t)
if key in collect:
has_redundant_base_types = True
collect[key] = tuple(
pytd_utils.JoinTypes([p1, p2])
for p1, p2 in zip(collect[key], t.parameters))
else:
collect[key] = t.parameters
if not has_redundant_base_types:
return union
result = pytd.NothingType()
done = set()
for t in union.type_list:
if isinstance(t, pytd.GenericType):
key = self._key(t)
if key in done:
continue # already added
parameters = collect[key]
add = t.Replace(parameters=tuple(
p.Visit(CombineContainers()) for p in parameters))
done.add(key)
else:
add = t
result = pytd_utils.JoinTypes([result, add])
return result
def value_to_pytd_type(self, node, v, seen, view):
"""Get a PyTD type representing this object, as seen at a node.
Args:
node: The node from which we want to observe this object.
v: The object.
seen: The set of values seen before while computing the type.
view: A Variable -> binding map.
Returns:
A PyTD type.
"""
if isinstance(v, (abstract.Empty, typing_overlay.NoReturn)):
return pytd.NothingType()
elif isinstance(v, abstract.TypeParameterInstance):
if v.instance.get_instance_type_parameter(v.full_name).bindings:
# The type parameter was initialized. Set the view to None, since we
# don't include v.instance in the view.
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, p, seen, None)
for p in v.instance.get_instance_type_parameter(v.full_name).data)
elif v.param.constraints:
return pytd_utils.JoinTypes(
self.value_instance_to_pytd_type(node, p, None, seen, view)
for p in v.param.constraints)
elif v.param.bound:
return self.value_instance_to_pytd_type(
node, v.param.bound, None, seen, view)
else:
return pytd.AnythingType()
elif isinstance(v, typing_overlay.TypeVar):
return pytd.NamedType("__builtin__.type")
elif isinstance(v, abstract.FUNCTION_TYPES):
try:
signatures = abstract_utils.get_signatures(v)
except NotImplementedError:
return pytd.NamedType("typing.Callable")
if len(signatures) == 1:
val = self.signature_to_callable(signatures[0], self.vm)
if (not isinstance(v, abstract.PYTD_FUNCTION_TYPES) or
not self.vm.annotations_util.get_type_parameters(val)):
# This is a workaround to make sure we don't put unexpected type
# parameters in call traces.
return self.value_instance_to_pytd_type(node, val, None, seen, view)
return pytd.NamedType("typing.Callable")
elif isinstance(v, (abstract.ClassMethod, abstract.StaticMethod)):
return self.value_to_pytd_type(node, v.method, seen, view)
elif isinstance(v, (special_builtins.IsInstance,
special_builtins.ClassMethodCallable)):
return pytd.NamedType("typing.Callable")
elif isinstance(v, mixin.Class):
param = self.value_instance_to_pytd_type(node, v, None, seen, view)
return pytd.GenericType(base_type=pytd.NamedType("__builtin__.type"),
parameters=(param,))
elif isinstance(v, abstract.Module):
return pytd.NamedType("__builtin__.module")
elif isinstance(v, abstract.SimpleAbstractValue):
if v.cls:
ret = self.value_instance_to_pytd_type(
node, v.cls, v, seen=seen, view=view)
ret.Visit(visitors.FillInLocalPointers(
{"__builtin__": self.vm.loader.builtins}))
return ret
else:
# We don't know this type's __class__, so return AnythingType to
# indicate that we don't know anything about what this is.
# This happens e.g. for locals / globals, which are returned from the
# code in class declarations.
log.info("Using ? for %s", v.name)
return pytd.AnythingType()
elif isinstance(v, abstract.Union):
return pytd.UnionType(tuple(self.value_to_pytd_type(node, o, seen, view)
for o in v.options))
elif isinstance(v, special_builtins.SuperInstance):
return pytd.NamedType("__builtin__.super")
elif isinstance(v, (abstract.Unsolvable, abstract.TypeParameter)):
# Arguably, the type of a type parameter is NamedType("typing.TypeVar"),
# but pytype doesn't know how to handle that, so let's just go with Any.
return pytd.AnythingType()
elif isinstance(v, abstract.Unknown):
return pytd.NamedType(v.class_name)
elif isinstance(v, abstract.BuildClass):
return pytd.NamedType("typing.Callable")
else:
raise NotImplementedError(v.__class__.__name__)
def setUp(self):
self.options = config.Options.create(python_version=self.PYTHON_VERSION,
python_exe=self.PYTHON_EXE)
def t(name): # pylint: disable=invalid-name
return pytd.ClassType("__builtin__." + name)
self.bool = t("bool")
self.dict = t("dict")
self.float = t("float")
self.complex = t("complex")
self.int = t("int")
if self.PYTHON_VERSION[0] == 2:
self.long = t("long")
self.list = t("list")
self.none_type = t("NoneType")
self.object = t("object")
self.set = t("set")
self.frozenset = t("frozenset")
self.str = t("str")
self.bytearray = t("bytearray")
self.tuple = t("tuple")
self.unicode = t("unicode")
self.generator = t("generator")
self.function = t("function")
self.anything = pytd.AnythingType()
self.nothing = pytd.NothingType()
self.module = t("module")
self.file = t("file")
# The various union types use pytd_utils.CanonicalOrdering()'s ordering:
self.intorstr = pytd.UnionType((self.int, self.str))
self.strorunicode = pytd.UnionType((self.str, self.unicode))
self.intorfloat = pytd.UnionType((self.float, self.int))
self.intorfloatorstr = pytd.UnionType((self.float, self.int, self.str))
self.complexorstr = pytd.UnionType((self.complex, self.str))
if self.PYTHON_VERSION[0] == 3:
self.intorfloatorlong = self.intorfloat
self.intorfloatorlongorcomplex = pytd.UnionType(
(self.int, self.float, self.complex))
else:
self.intorfloatorlong = pytd.UnionType((self.int, self.float, self.long))
self.intorfloatorlongorcomplex = pytd.UnionType(
(self.int, self.float, self.long, self.complex))
self.int_tuple = pytd.HomogeneousContainerType(self.tuple, (self.int,))
self.nothing_tuple = pytd.HomogeneousContainerType(self.tuple,
(self.nothing,))
self.intorfloat_tuple = pytd.HomogeneousContainerType(self.tuple,
(self.intorfloat,))
self.int_set = pytd.HomogeneousContainerType(self.set, (self.int,))
self.intorfloat_set = pytd.HomogeneousContainerType(self.set,
(self.intorfloat,))
# TODO(pludemann): simplify this (test_and2)
self.unknown_frozenset = pytd.HomogeneousContainerType(
self.frozenset, (self.anything,))
self.float_frozenset = pytd.HomogeneousContainerType(self.frozenset,
(self.float,))
self.empty_frozenset = pytd.HomogeneousContainerType(self.frozenset,
(self.nothing,))
self.int_list = pytd.HomogeneousContainerType(self.list, (self.int,))
self.str_list = pytd.HomogeneousContainerType(self.list, (self.str,))
self.intorfloat_list = pytd.HomogeneousContainerType(self.list,
(self.intorfloat,))
self.intorstr_list = pytd.HomogeneousContainerType(self.list,
(self.intorstr,))
self.anything_list = pytd.HomogeneousContainerType(self.list,
(self.anything,))
self.nothing_list = pytd.HomogeneousContainerType(self.list,
(self.nothing,))
self.int_int_dict = pytd.GenericType(self.dict, (self.int, self.int))
self.int_str_dict = pytd.GenericType(self.dict, (self.int, self.str))
self.str_int_dict = pytd.GenericType(self.dict, (self.str, self.int))
self.nothing_nothing_dict = pytd.GenericType(self.dict,
(self.nothing, self.nothing))
