def from_const(cls, node: ast3.NameConstant):
if node.value is None:
return pytd.NamedType("None")
return cls(type(node.value).__name__, node.value)
def NamedOrClassType(name, cls):
"""Create Classtype / NamedType."""
if cls is None:
return pytd.NamedType(name)
else:
return pytd.ClassType(name, cls)
def VisitClassType(self, node):
return pytd.NamedType(node.name)
def testPrintHeterogeneousTuple(self):
t = pytd.TupleType(pytd.NamedType("tuple"),
(pytd.NamedType("str"), pytd.NamedType("float")))
self.assertEqual("Tuple[str, float]", pytd.Print(t))
def register_class_name(self, class_name):
"""Register a class name so that it can shadow aliases."""
if not self._current_condition.active:
return
self._type_map[class_name] = pytd.NamedType(class_name)
def test_anything(self):
m = type_match.TypeMatch({})
self.assertMatch(m, pytd.AnythingType(), pytd.AnythingType())
self.assertMatch(m, pytd.AnythingType(), pytd.NamedType("x"))
self.assertMatch(m, pytd.NamedType("x"), pytd.AnythingType())
def test_nothing_right(self):
m = type_match.TypeMatch({})
eq = m.match_type_against_type(pytd.NamedType("A"), pytd.NothingType(),
{})
self.assertEqual(eq, booleq.FALSE)
def testJoinAnythingTypes(self):
"""Test that JoinTypes() simplifies unions containing '?'."""
types = [pytd.AnythingType(), pytd.NamedType("a")]
self.assertIsInstance(pytd_utils.JoinTypes(types), pytd.AnythingType)
def value_to_pytd_type(self, node, v, seen, view):
"""Get a PyTD type representing this object, as seen at a node.
Args:
node: The node from which we want to observe this object.
v: The object.
seen: The set of values seen before while computing the type.
view: A Variable -> binding map.
Returns:
A PyTD type.
"""
if isinstance(v, (abstract.Empty, typing_overlay.NoReturn)):
return pytd.NothingType()
elif isinstance(v, abstract.TypeParameterInstance):
if v.module in self._scopes:
return self._typeparam_to_def(node, v.param, v.param.name)
elif v.instance.get_instance_type_parameter(v.full_name).bindings:
# The type parameter was initialized. Set the view to None, since we
# don't include v.instance in the view.
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, p, seen, None) for p in
v.instance.get_instance_type_parameter(v.full_name).data)
elif v.param.constraints:
return pytd_utils.JoinTypes(
self.value_instance_to_pytd_type(node, p, None, seen, view)
for p in v.param.constraints)
elif v.param.bound:
return self.value_instance_to_pytd_type(
node, v.param.bound, None, seen, view)
else:
return pytd.AnythingType()
elif isinstance(v, typing_overlay.TypeVar):
return pytd.NamedType("builtins.type")
elif isinstance(v, dataclass_overlay.FieldInstance):
if not v.default:
return pytd.AnythingType()
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, d, seen, view)
for d in v.default.data)
elif isinstance(v, abstract.FUNCTION_TYPES):
try:
signatures = abstract_utils.get_signatures(v)
except NotImplementedError:
return pytd.NamedType("typing.Callable")
if len(signatures) == 1:
val = self.signature_to_callable(signatures[0])
if not isinstance(
v, abstract.PYTD_FUNCTION_TYPES) or not val.formal:
# This is a workaround to make sure we don't put unexpected type
# parameters in call traces.
return self.value_instance_to_pytd_type(
node, val, None, seen, view)
return pytd.NamedType("typing.Callable")
elif isinstance(v, (abstract.ClassMethod, abstract.StaticMethod)):
return self.value_to_pytd_type(node, v.method, seen, view)
elif isinstance(v, (special_builtins.IsInstance,
special_builtins.ClassMethodCallable)):
return pytd.NamedType("typing.Callable")
elif isinstance(v, class_mixin.Class):
param = self.value_instance_to_pytd_type(node, v, None, seen, view)
return pytd.GenericType(base_type=pytd.NamedType("builtins.type"),
parameters=(param, ))
elif isinstance(v, abstract.Module):
return pytd.NamedType("builtins.module")
elif (self._output_mode >= Converter.OutputMode.LITERAL
and isinstance(v, abstract.ConcreteValue)
and isinstance(v.pyval, (int, str, bytes))):
# LITERAL mode is used only for pretty-printing, so we just stringify the
# inner value rather than properly converting it.
return pytd.Literal(repr(v.pyval))
elif isinstance(v, abstract.SimpleValue):
if v.cls:
ret = self.value_instance_to_pytd_type(node,
v.cls,
v,
seen=seen,
view=view)
ret.Visit(
visitors.FillInLocalPointers(
{"builtins": self.vm.loader.builtins}))
return ret
else:
# We don't know this type's __class__, so return AnythingType to
# indicate that we don't know anything about what this is.
# This happens e.g. for locals / globals, which are returned from the
# code in class declarations.
log.info("Using Any for %s", v.name)
return pytd.AnythingType()
elif isinstance(v, abstract.Union):
opts = []
for o in v.options:
# NOTE: Guarding printing of type parameters behind _detailed until
# round-tripping is working properly.
if self._detailed and isinstance(o, abstract.TypeParameter):
opt = self._typeparam_to_def(node, o, o.name)
else:
opt = self.value_to_pytd_type(node, o, seen, view)
opts.append(opt)
return pytd.UnionType(tuple(opts))
elif isinstance(v, special_builtins.SuperInstance):
return pytd.NamedType("builtins.super")
elif isinstance(v, abstract.TypeParameter):
# Arguably, the type of a type parameter is NamedType("typing.TypeVar"),
# but pytype doesn't know how to handle that, so let's just go with Any
# unless self._detailed is set.
if self._detailed:
return pytd.NamedType("typing.TypeVar")
else:
return pytd.AnythingType()
elif isinstance(v, abstract.Unsolvable):
return pytd.AnythingType()
elif isinstance(v, abstract.Unknown):
return pytd.NamedType(v.class_name)
elif isinstance(v, abstract.BuildClass):
return pytd.NamedType("typing.Callable")
else:
raise NotImplementedError(v.__class__.__name__)
def build_class(self, class_name, bases, keywords, decorators,
defs) -> pytd.Class:
"""Build a pytd.Class from definitions collected from an ast node."""
parents, namedtuple_index = classdef.get_parents(bases)
metaclass = classdef.get_metaclass(keywords, parents)
constants, methods, aliases, slots, classes = _split_definitions(defs)
# Make sure we don't have duplicate definitions.
classdef.check_for_duplicate_defs(methods, constants, aliases)
# Generate a NamedTuple proxy base class if needed
if namedtuple_index is not None:
namedtuple_parent = self.new_named_tuple(class_name,
[(c.name, c.type)
for c in constants])
parents[namedtuple_index] = namedtuple_parent
constants = []
if aliases:
vals_dict = {
val.name: val
for val in constants + aliases + methods + classes
}
for val in aliases:
name = val.name
seen_names = set()
while isinstance(val, pytd.Alias):
if isinstance(val.type, pytd.NamedType):
_, _, base_name = val.type.name.rpartition(".")
if base_name in seen_names:
# This happens in cases like:
# class X:
# Y = something.Y
# Since we try to resolve aliases immediately, we don't know what
# type to fill in when the alias value comes from outside the
# class. The best we can do is Any.
val = pytd.Constant(name, pytd.AnythingType())
continue
seen_names.add(base_name)
if base_name in vals_dict:
val = vals_dict[base_name]
continue
# The alias value comes from outside the class. The best we can do is
# to fill in Any.
val = pytd.Constant(name, pytd.AnythingType())
if isinstance(val, function.NameAndSig):
val = dataclasses.replace(val, name=name)
methods.append(val)
else:
if isinstance(val, pytd.Class):
t = pytdgen.pytd_type(
pytd.NamedType(class_name + "." + val.name))
else:
t = val.type
constants.append(pytd.Constant(name, t))
parents = [p for p in parents if not isinstance(p, pytd.NothingType)]
methods = function.merge_method_signatures(methods)
if not parents and class_name not in ["classobj", "object"]:
# A parent-less class inherits from classobj in Python 2 and from object
# in Python 3. typeshed assumes the Python 3 behavior for all stubs, so we
# do the same here.
parents = (pytd.NamedType("object"), )
return pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(parents),
methods=tuple(methods),
constants=tuple(constants),
classes=tuple(classes),
decorators=tuple(decorators),
slots=slots,
template=())
def testJoinSingleType(self):
"""Test that JoinTypes() returns single types as-is."""
a = pytd.NamedType("a")
self.assertEqual(pytd_utils.JoinTypes([a]), a)
self.assertEqual(pytd_utils.JoinTypes([a, a]), a)
def pytd_type(value: pytd_node.Node) -> pytd_node.Node:
return pytd.GenericType(pytd.NamedType("type"), (value,))
def pytd_list(typ: str) -> pytd_node.Node:
if typ:
return pytd.GenericType(
pytd.NamedType("typing.List"), (pytd.NamedType(typ),))
else:
return pytd.NamedType("typing.List")
def to_pytd(self): return pytd.NamedType(self.type)
def test_named_late(self):
m = type_match.TypeMatch({})
eq = m.match_type_against_type(pytd.LateType("X"), pytd.NamedType("X"),
{})
self.assertEqual(eq, booleq.FALSE)
def value_to_pytd_type(self, node, v, seen, view):
"""Get a PyTD type representing this object, as seen at a node.
Args:
node: The node from which we want to observe this object.
v: The object.
seen: The set of values seen before while computing the type.
view: A Variable -> binding map.
Returns:
A PyTD type.
"""
if isinstance(v, (abstract.Empty, typing_overlay.NoReturn)):
return pytd.NothingType()
elif isinstance(v, abstract.TypeParameterInstance):
if v.instance.get_instance_type_parameter(v.full_name).bindings:
# The type parameter was initialized. Set the view to None, since we
# don't include v.instance in the view.
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, p, seen, None) for p in
v.instance.get_instance_type_parameter(v.full_name).data)
elif v.param.constraints:
return pytd_utils.JoinTypes(
self.value_instance_to_pytd_type(node, p, None, seen, view)
for p in v.param.constraints)
elif v.param.bound:
return self.value_instance_to_pytd_type(
node, v.param.bound, None, seen, view)
else:
return pytd.AnythingType()
elif isinstance(v, typing_overlay.TypeVar):
return pytd.NamedType("__builtin__.type")
elif isinstance(v, abstract.FUNCTION_TYPES):
try:
signatures = abstract_utils.get_signatures(v)
except NotImplementedError:
return pytd.NamedType("typing.Callable")
if len(signatures) == 1:
val = self.signature_to_callable(signatures[0], self.vm)
if (not isinstance(v, abstract.PYTD_FUNCTION_TYPES) or
not self.vm.annotations_util.get_type_parameters(val)):
# This is a workaround to make sure we don't put unexpected type
# parameters in call traces.
return self.value_instance_to_pytd_type(
node, val, None, seen, view)
return pytd.NamedType("typing.Callable")
elif isinstance(v, (abstract.ClassMethod, abstract.StaticMethod)):
return self.value_to_pytd_type(node, v.method, seen, view)
elif isinstance(v, (special_builtins.IsInstance,
special_builtins.ClassMethodCallable)):
return pytd.NamedType("typing.Callable")
elif isinstance(v, mixin.Class):
param = self.value_instance_to_pytd_type(node, v, None, seen, view)
return pytd.GenericType(
base_type=pytd.NamedType("__builtin__.type"),
parameters=(param, ))
elif isinstance(v, abstract.Module):
return pytd.NamedType("__builtin__.module")
elif isinstance(v, abstract.SimpleAbstractValue):
if v.cls:
ret = self.value_instance_to_pytd_type(node,
v.cls,
v,
seen=seen,
view=view)
ret.Visit(
visitors.FillInLocalPointers(
{"__builtin__": self.vm.loader.builtins}))
return ret
else:
# We don't know this type's __class__, so return AnythingType to
# indicate that we don't know anything about what this is.
# This happens e.g. for locals / globals, which are returned from the
# code in class declarations.
log.info("Using ? for %s", v.name)
return pytd.AnythingType()
elif isinstance(v, abstract.Union):
return pytd.UnionType(
tuple(
self.value_to_pytd_type(node, o, seen, view)
for o in v.options))
elif isinstance(v, special_builtins.SuperInstance):
return pytd.NamedType("__builtin__.super")
elif isinstance(v, abstract.TypeParameter):
# Arguably, the type of a type parameter is NamedType("typing.TypeVar"),
# but pytype doesn't know how to handle that, so let's just go with Any
# unless self._detailed is set.
if self._detailed:
return pytd.NamedType("typing.TypeVar")
else:
return pytd.AnythingType()
elif isinstance(v, abstract.Unsolvable):
return pytd.AnythingType()
elif isinstance(v, abstract.Unknown):
return pytd.NamedType(v.class_name)
elif isinstance(v, abstract.BuildClass):
return pytd.NamedType("typing.Callable")
else:
raise NotImplementedError(v.__class__.__name__)
def test_named_against_generic(self):
m = type_match.TypeMatch({})
eq = m.match_type_against_type(
pytd.GenericType(pytd.NamedType("A"), ()), pytd.NamedType("A"), {})
self.assertEqual(eq, booleq.TRUE)
def test_convert_optional(self):
t = pytd.GenericType(pytd.NamedType("typing.Optional"),
(pytd.NamedType("str"),))
self.assertEqual(self.convert(t), "Optional[str]")
def test_anything_as_top(self):
m = type_match.TypeMatch({}, any_also_is_bottom=False)
self.assertMatch(m, pytd.AnythingType(), pytd.AnythingType())
self.assertNoMatch(m, pytd.AnythingType(), pytd.NamedType("x"))
self.assertMatch(m, pytd.NamedType("x"), pytd.AnythingType())
def test_convert_union(self):
t = pytd.GenericType(pytd.NamedType("typing.Union"),
(pytd.NamedType("str"), pytd.NamedType("float")))
self.assertEqual(self.convert(t), "Union[str, float]")
def testAnyReplacement(self):
union = pytd.UnionType((pytd.NamedType("a"), pytd.NamedType("b")))
self.assertEqual(
union.Visit(visitors.ReplaceUnionsWithAny()), pytd.AnythingType())
def test_convert_list(self):
t = pytd.NamedType("typing.List")
self.assertEqual(self.convert(t), "list")
def testLookupTypingClass(self):
node = visitors.LookupClasses(pytd.NamedType("typing.Sequence"),
self.loader.concat_all())
assert node.cls
def test_convert_tuple(self):
t = pytd.NamedType("typing.Tuple")
self.assertEqual(self.convert(t), "tuple")
def add_class(self, class_name, parent_args, defs):
"""Add a class to the module.
Args:
class_name: The name of the class (a string).
parent_args: A list of parent types and (keyword, value) tuples.
Parent types must be instances of pytd.Type. Keyword tuples must
appear at the end of the list. Currently the only supported keyword
is 'metaclass'.
defs: A list of constant (pytd.Constant) and function (_NameAndSig)
definitions.
Raises:
ParseError: if defs contains duplicate names (excluding multiple
definitions of a function, which is allowed).
"""
# Process parent_args, extracting parents and possibly a metaclass.
parents = []
metaclass = None
for i, p in enumerate(parent_args):
if isinstance(p, pytd.Type):
parents.append(p)
else:
keyword, value = p
if i != len(parent_args) - 1:
raise ParseError("metaclass must be last argument")
if keyword != "metaclass":
raise ParseError(
"Only 'metaclass' allowed as classdef kwarg")
metaclass = value
constants, methods = _split_definitions(defs)
all_names = (list(set(f.name
for f in methods)) + [c.name for c in constants])
duplicates = [
name for name, count in collections.Counter(all_names).items()
if count >= 2
]
if duplicates:
# TODO(kramm): raise a syntax error right when the identifier is defined.
raise ParseError("Duplicate identifier(s): " +
", ".join(duplicates))
# This check is performed after the above error checking so that errors
# will be spotted even in non-active conditional code.
if not self._current_condition.active:
return
# TODO(dbaum): Is NothingType even legal here? The grammar accepts it but
# perhaps it should be a ParseError.
parents = [p for p in parents if not isinstance(p, pytd.NothingType)]
methods, properties = _merge_signatures(methods)
# Ensure that old style classes inherit from classobj.
if not parents and class_name not in ["classobj", "object"]:
parents = (pytd.NamedType("classobj"), )
cls = pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(parents),
methods=tuple(methods),
constants=tuple(constants + properties),
template=())
self._classes.append(cls)
def test_convert_any(self):
t = pytd.NamedType("typing.Any")
self.assertEqual(self.convert(t), "Any")
def NamedTypeWithModule(name, module=None):
"""Create NamedType, dotted if we have a module."""
if module is None:
return pytd.NamedType(name)
else:
return pytd.NamedType(module + "." + name)
def resolve_name(self, name):
"""Resolve an alias or create a NamedType."""
return self.defs.type_map.get(name) or pytd.NamedType(name)
def value_to_pytd_type(self, node, v, seen, view):
"""Get a PyTD type representing this object, as seen at a node.
Args:
node: The node from which we want to observe this object.
v: The object.
seen: The set of values seen before while computing the type.
view: A Variable -> binding map.
Returns:
A PyTD type.
"""
if isinstance(v, (abstract.Empty, abstract.Nothing)):
return pytd.NothingType()
elif isinstance(v, abstract.TypeParameterInstance):
if v.instance.type_parameters[v.name].bindings:
# The type parameter was initialized.
return pytd_utils.JoinTypes(
self.value_to_pytd_type(node, p, seen, view)
for p in v.instance.type_parameters[v.name].data)
elif v.param.constraints:
return pytd_utils.JoinTypes(
self.value_instance_to_pytd_type(node, p, None, seen, view)
for p in v.param.constraints)
else:
return pytd.AnythingType()
elif isinstance(v, typing.TypeVar):
return pytd.NamedType("__builtin__.type")
elif isinstance(
v,
(abstract.InterpreterFunction, abstract.BoundInterpreterFunction)):
sig, = abstract.get_signatures(v)
return self.value_instance_to_pytd_type(
node, self.signature_to_callable(sig, v.vm), None, seen, view)
elif isinstance(v,
(abstract.PyTDFunction, abstract.BoundPyTDFunction)):
signatures = abstract.get_signatures(v)
if len(signatures) == 1:
val = self.signature_to_callable(signatures[0], v.vm)
if not v.vm.annotations_util.get_type_parameters(val):
# This is a workaround to make sure we don't put unexpected type
# parameters in call traces.
return self.value_instance_to_pytd_type(
node, val, None, seen, view)
return pytd.NamedType("typing.Callable")
elif isinstance(
v,
(special_builtins.IsInstance, abstract.ClassMethod,
abstract.StaticMethod, special_builtins.ClassMethodCallable)):
return pytd.NamedType("typing.Callable")
elif isinstance(v, abstract.Class):
param = self.value_instance_to_pytd_type(node, v, None, seen, view)
return pytd.GenericType(
base_type=pytd.NamedType("__builtin__.type"),
parameters=(param, ))
elif isinstance(v, abstract.Module):
return pytd.NamedType("__builtin__.module")
elif isinstance(v, abstract.SimpleAbstractValue):
if v.cls:
classvalues = self._get_values(node, v.cls, view)
cls_types = []
for cls in classvalues:
cls_types.append(
self.value_instance_to_pytd_type(node,
cls,
v,
seen=seen,
view=view))
ret = pytd_utils.JoinTypes(cls_types)
ret.Visit(
visitors.FillInLocalPointers(
{"__builtin__": v.vm.loader.builtins}))
return ret
else:
# We don't know this type's __class__, so return AnythingType to
# indicate that we don't know anything about what this is.
# This happens e.g. for locals / globals, which are returned from the
# code in class declarations.
log.info("Using ? for %s", v.name)
return pytd.AnythingType()
elif isinstance(v, abstract.Union):
return pytd.UnionType(
tuple(
self.value_to_pytd_type(node, o, seen, view)
for o in v.options))
elif isinstance(v, special_builtins.SuperInstance):
return pytd.NamedType("__builtin__.super")
elif isinstance(v, (abstract.Unsolvable, abstract.TypeParameter)):
# Arguably, the type of a type parameter is NamedType("typing.TypeVar"),
# but pytype doesn't know how to handle that, so let's just go with Any.
return pytd.AnythingType()
elif isinstance(v, abstract.Unknown):
return pytd.NamedType(v.class_name)
else:
raise NotImplementedError(v.__class__.__name__)
def _HasSubClassInSet(self, cls, known):
"""Queries whether a subclass of a type is present in a given set."""
return any(
pytd.NamedType(sub) in known for sub in self._subclasses[str(cls)])