def VisitClass(self, cls):
"""Modify the methods of a class.
This
(a) removes all reverse operators from the class
(b) adds the unreversed version of reverse operators from other classes.
Args:
cls: An instance of pytd.Class.
Returns:
A new class, with modified operators.
"""
methods_to_add = self.methods_to_add[cls]
new_methods = []
method_names = (set(method.name for method in cls.methods)
| set(methods_to_add.keys()))
for method_name in sorted(method_names):
if method_name in self._reverse_operator_names:
# This is a reverse operator (__radd__ etc.). We're going to add
# the counterpiece (__add__), so we can throw away the original.
continue
try:
method = cls.Lookup(method_name)
except KeyError:
method = pytd.Function(method_name, ())
# wrap the extra signatures into a method, for easier matching
extra_signatures = methods_to_add.get(method_name, [])
new_signatures = []
if method_name in self._reversible_operator_names:
# If this is a normal "unreversed" operator (__add__ etc.), see whether
# one of signatures we got from the reversed operators takes precedence.
for sig1 in method.signatures:
if not any(
self._MatchSignature(sig1, sig2)
for sig2 in extra_signatures):
new_signatures.append(sig1)
else:
new_signatures.extend(method.signatures)
new_methods.append(
method.Replace(signatures=(tuple(new_signatures +
extra_signatures))))
return cls.Replace(methods=tuple(new_methods))
def DummyMethod(name, *params):
"""Create a simple method using only "Any"s as types.
Arguments:
name: The name of the method
*params: The parameter names.
Returns:
A pytd.Function.
"""
def make_param(param):
return pytd.Parameter(param, type=pytd.AnythingType(), kwonly=False,
optional=False, mutated_type=None)
sig = pytd.Signature(tuple(make_param(param) for param in params),
starargs=None, starstarargs=None,
return_type=pytd.AnythingType(),
exceptions=(), template=())
return pytd.Function(name=name,
signatures=(sig,),
kind=pytd.METHOD,
flags=0)
def value_to_pytd_def(self, node, v, name):
"""Get a PyTD definition for this object.
Args:
node: The node.
v: The object.
name: The object name.
Returns:
A PyTD definition.
"""
if (isinstance(v, abstract.PyTDFunction)
and not isinstance(v, typing.TypeVar)):
return pytd.Function(
name=name,
signatures=tuple(sig.pytd_sig for sig in v.signatures),
kind=v.kind,
flags=pytd.Function.abstract_flag(v.is_abstract))
elif isinstance(v, abstract.InterpreterFunction):
return self._function_to_def(node, v, name)
elif isinstance(v, abstract.ParameterizedClass):
return pytd.Alias(name, v.get_instance_type(node))
elif isinstance(v, abstract.PyTDClass) and v.module:
# This happens if a module does e.g. "from x import y as z", i.e., copies
# something from another module to the local namespace. We *could*
# reproduce the entire class, but we choose a more dense representation.
return v.to_type(node)
elif isinstance(v, abstract.PyTDClass): # a namedtuple instance
assert name != v.name
return pytd.Alias(name, pytd.NamedType(v.name))
elif isinstance(v, abstract.InterpreterClass):
if v.official_name is None or name == v.official_name:
return self._class_to_def(node, v, name)
else:
return pytd.Alias(name, pytd.NamedType(v.official_name))
elif isinstance(v, abstract.TypeParameter):
return self._typeparam_to_def(node, v, name)
elif isinstance(v, abstract.Unsolvable):
return pytd.Constant(name, v.to_type(node))
else:
raise NotImplementedError(v.__class__.__name__)
def merge_method_signatures(
name_and_sigs: List[NameAndSig],
check_unhandled_decorator: bool = True) -> List[pytd.Function]:
"""Group the signatures by name, turning each group into a function."""
functions = collections.OrderedDict()
for fn in name_and_sigs:
if fn.name not in functions:
functions[fn.name] = _DecoratedFunction.make(fn)
else:
functions[fn.name].add_overload(fn)
methods = []
for name, fn in functions.items():
if name == "__new__" or fn.decorator == "staticmethod":
kind = pytd.MethodTypes.STATICMETHOD
elif name == "__init_subclass__" or fn.decorator == "classmethod":
kind = pytd.MethodTypes.CLASSMETHOD
elif fn.properties:
kind = pytd.MethodTypes.PROPERTY
# If we have only setters and/or deleters, replace them with a single
# method foo(...) -> Any, so that we infer a constant `foo: Any` even if
# the original method signatures are all `foo(...) -> None`. (If we have a
# getter we use its return type, but in the absence of a getter we want to
# fall back on Any since we cannot say anything about what the setter sets
# the type of foo to.)
if fn.properties.getter:
fn.sigs = [fn.properties.getter]
else:
sig = fn.properties.setter or fn.properties.deleter
fn.sigs = [sig.Replace(return_type=pytd.AnythingType())]
elif fn.decorator and check_unhandled_decorator:
raise ValueError("Unhandled decorator: %s" % fn.decorator)
else:
# Other decorators do not affect the kind
kind = pytd.MethodTypes.METHOD
flags = 0
if fn.is_abstract:
flags |= pytd.MethodFlags.ABSTRACT
if fn.is_coroutine:
flags |= pytd.MethodFlags.COROUTINE
methods.append(pytd.Function(name, tuple(fn.sigs), kind, flags))
return methods
def _call_traces_to_function(call_traces, prefix=""):
funcs = collections.defaultdict(pytd_utils.OrderedSet)
for funcvar, args, kws, retvar in call_traces:
if isinstance(funcvar.data, abstract.BoundFunction):
func = funcvar.data.underlying.signatures[0]
else:
func = funcvar.data.signatures[0]
arg_names = func.get_parameter_names()
arg_types = (a.data.to_type()
for a in func.get_bound_arguments() + list(args))
ret = pytd_utils.JoinTypes(t.to_type() for t in retvar.data)
funcs[funcvar.data.name].add(pytd.Signature(
tuple(pytd.Parameter(n, t)
for n, t in zip(arg_names, arg_types)) +
tuple(pytd.Parameter(name, a.data.to_type())
for name, a in kws),
ret, has_optional=False, exceptions=(), template=()))
functions = []
for name, signatures in funcs.items():
functions.append(pytd.Function(prefix + name, tuple(signatures)))
return functions
def _merge_method_signatures(signatures):
"""Group the signatures by name, turning each group into a function."""
name_to_signatures = collections.OrderedDict()
name_to_decorator = {}
name_to_is_abstract = collections.defaultdict(bool)
# map from function name to a bool indicating whether the function has an
# external definition
name_to_external_code = {}
for name, signature, decorator, external_code, is_abstract in signatures:
if name not in name_to_signatures:
name_to_signatures[name] = []
name_to_decorator[name] = decorator
if name_to_decorator[name] != decorator:
raise ParseError(
"Overloaded signatures for %s disagree on decorators" % name)
if name in name_to_external_code:
if external_code and name_to_external_code[name]:
raise ParseError("Multiple PYTHONCODEs for %s" % name)
elif external_code != name_to_external_code[name]:
raise ParseError("Mixed pytd and PYTHONCODEs for %s" % name)
else:
name_to_external_code[name] = external_code
name_to_signatures[name].append(signature)
name_to_is_abstract[name] |= is_abstract
methods = []
for name, signatures in name_to_signatures.items():
decorator = name_to_decorator[name]
is_abstract = name_to_is_abstract[name]
if name == "__new__" or decorator == "staticmethod":
kind = pytd.STATICMETHOD
elif decorator == "classmethod":
kind = pytd.CLASSMETHOD
else:
kind = pytd.METHOD
if name_to_external_code[name]:
methods.append(pytd.ExternalFunction(name, (), kind, is_abstract))
else:
methods.append(pytd.Function(name, tuple(signatures), kind, is_abstract))
return methods
def testComplexCombinedType(self):
"""Test parsing a type with both union and intersection."""
data1 = r"def foo(a: Foo or Bar and Zot) -> object"
data2 = r"def foo(a: Foo or (Bar and Zot)) -> object"
result1 = self.Parse(data1)
result2 = self.Parse(data2)
f = pytd.Function(
name="foo",
signatures=(pytd.Signature(params=(pytd.Parameter(
name="a",
type=pytd.UnionType(
type_list=(pytd.NamedType("Foo"),
pytd.IntersectionType(
type_list=(pytd.NamedType("Bar"),
pytd.NamedType("Zot")))))), ),
return_type=pytd.NamedType("object"),
template=(),
has_optional=False,
exceptions=()), ))
self.assertEqual(f, result1.Lookup("foo"))
self.assertEqual(f, result2.Lookup("foo"))
def _simple_func_to_def(self, node, v, name):
"""Convert a SimpleFunction to a PyTD definition."""
sig = v.signature
params = [
pytd.Parameter(p, sig.annotations[p].get_instance_type(node),
False, p in sig.defaults, None)
for p in sig.param_names
]
kwonly = [
pytd.Parameter(p, sig.annotations[p].get_instance_type(node), True,
p in sig.defaults, None) for p in sig.kwonly_params
]
if sig.varargs_name:
star = pytd.Parameter(
sig.varargs_name,
sig.annotations[sig.varargs_name].get_instance_type(node),
False, False, None)
else:
star = None
if sig.kwargs_name:
starstar = pytd.Parameter(
sig.kwargs_name,
sig.annotations[sig.kwargs_name].get_instance_type(node),
False, False, None)
else:
starstar = None
if sig.has_return_annotation:
ret_type = sig.annotations["return"].get_instance_type(node)
else:
ret_type = pytd.NamedType("__builtin__.NoneType")
pytd_sig = pytd.Signature(params=tuple(params + kwonly),
starargs=star,
starstarargs=starstar,
return_type=ret_type,
exceptions=(),
template=())
return pytd.Function(name, (pytd_sig, ), pytd.METHOD)
def _call_traces_to_function(call_traces, name_transform=lambda x: x):
funcs = collections.defaultdict(pytd_utils.OrderedSet)
for node, func, sigs, args, kws, retvar in call_traces:
# The lengths may be different in the presence of optional and kw args.
arg_names = max((sig.get_positional_names() for sig in sigs), key=len)
for i in range(len(arg_names)):
if not isinstance(func.data, abstract.BoundFunction) or i > 0:
arg_names[i] = function.argname(i)
arg_types = (a.data.to_type(node) for a in args)
ret = pytd_utils.JoinTypes(t.to_type(node) for t in retvar.data)
starargs = None
starstarargs = None
funcs[func.data.name].add(pytd.Signature(
tuple(pytd.Parameter(n, t, False, False, None)
for n, t in zip(arg_names, arg_types)) +
tuple(pytd.Parameter(name, a.data.to_type(node), False, False, None)
for name, a in kws),
starargs, starstarargs,
ret, exceptions=(), template=()))
functions = []
for name, signatures in funcs.items():
functions.append(pytd.Function(name_transform(name), tuple(signatures),
pytd.MethodTypes.METHOD))
return functions
def WrapTypeDeclUnit(name, items):
"""Given a list (classes, functions, etc.), wrap a pytd around them.
Args:
name: The name attribute of the resulting TypeDeclUnit.
items: A list of items. Can contain pytd.Class, pytd.Function and
pytd.Constant.
Returns:
A pytd.TypeDeclUnit.
Raises:
ValueError: In case of an invalid item in the list.
NameError: For name conflicts.
"""
functions = collections.OrderedDict()
classes = collections.OrderedDict()
constants = collections.defaultdict(TypeBuilder)
aliases = collections.OrderedDict()
typevars = collections.OrderedDict()
for item in items:
if isinstance(item, pytd.Function):
if item.name in functions:
if item.kind != functions[item.name].kind:
raise ValueError("Can't combine %s and %s" %
(item.kind, functions[item.name].kind))
functions[item.name] = pytd.Function(
item.name,
functions[item.name].signatures + item.signatures,
item.kind)
else:
functions[item.name] = item
elif isinstance(item, pytd.Class):
if item.name in classes:
raise NameError("Duplicate top level class: %r" % item.name)
classes[item.name] = item
elif isinstance(item, pytd.Constant):
constants[item.name].add_type(item.type)
elif isinstance(item, pytd.Alias):
if item.name in aliases:
raise NameError("Duplicate top level alias or import: %r" %
item.name)
aliases[item.name] = item
elif isinstance(item, pytd.TypeParameter):
if item.name in typevars:
raise NameError("Duplicate top level type parameter: %r" %
item.name)
typevars[item.name] = item
else:
raise ValueError("Invalid top level pytd item: %r" % type(item))
categories = {
"function": functions,
"class": classes,
"constant": constants,
"alias": aliases,
"typevar": typevars
}
for c1, c2 in itertools.combinations(categories, 2):
_check_intersection(categories[c1], categories[c2], c1, c2)
return pytd.TypeDeclUnit(name=name,
constants=tuple(
pytd.Constant(name, t.build())
for name, t in sorted(constants.items())),
type_params=tuple(typevars.values()),
classes=tuple(classes.values()),
functions=tuple(functions.values()),
aliases=tuple(aliases.values()))
def _merge_signatures(signatures):
"""Given a list of pytd function signature declarations, group them by name.
Converts a list of NameAndSig items to a list of Functions and a list of
Constants (grouping signatures by name). Constants are derived from
functions with @property decorators.
Arguments:
signatures: List[NameAndSig].
Returns:
Tuple[List[pytd.Function], List[pytd.Constant]].
Raises:
ParseError: if an error is encountered while trying to merge signatures.
"""
name_to_property_type = collections.OrderedDict()
method_signatures = []
for signature in signatures:
is_property, property_type = _try_parse_signature_as_property(
signature)
if is_property:
# Any methods with a decorator that looks like one of {@property,
# @foo.setter, @foo.deleter) will get merged into a Constant.
name = signature.name
if property_type or name not in name_to_property_type:
name_to_property_type[name] = property_type
# TODO(acaceres): warn if incompatible types? Or only take type
# from @property, not @foo.setter? Take last non-None for now.
else:
method_signatures.append(signature)
name_to_signatures = collections.OrderedDict()
# map name to (# external_code is {False,True}):
name_external = collections.defaultdict(lambda: {False: 0, True: 0})
name_to_decorators = {}
for name, signature, decorators, external_code in method_signatures:
if name in name_to_property_type:
raise ParseError("Incompatible signatures for %s" % name)
if name not in name_to_signatures:
name_to_signatures[name] = []
name_to_decorators[name] = decorators
if name_to_decorators[name] != decorators:
raise ParseError(
"Overloaded signatures for %s disagree on decorators" % name)
name_to_signatures[name].append(signature)
name_external[name][external_code] += 1
_verify_python_code(name_external)
methods = []
for name, signatures in name_to_signatures.items():
kind = pytd.METHOD
decorators = name_to_decorators[name]
if "classmethod" in decorators:
kind = pytd.CLASSMETHOD
if name == "__new__" or "staticmethod" in decorators:
kind = pytd.STATICMETHOD
if name_external[name][True]:
methods.append(pytd.ExternalFunction(name, (), kind))
else:
methods.append(pytd.Function(name, tuple(signatures), kind))
constants = []
for name, property_type in name_to_property_type.items():
if not property_type:
property_type = pytd.AnythingType()
constants.append(pytd.Constant(name, property_type))
return methods, constants
def _class_to_def(self, node, v, class_name):
"""Convert an InterpreterClass to a PyTD definition."""
self._scopes.append(class_name)
methods = {}
constants = collections.defaultdict(pytd_utils.TypeBuilder)
annots = abstract_utils.get_annotations_dict(v.members)
annotated_names = set()
def add_constants(iterator):
for name, t in iterator:
if t is None:
# Remove the entry from constants
annotated_names.add(name)
elif name not in annotated_names:
constants[name].add_type(t)
annotated_names.add(name)
add_constants(
self._ordered_attrs_to_instance_types(node, v.metadata, annots))
add_constants(self.annotations_to_instance_types(node, annots))
def get_decorated_method(name, value, func_slot):
fvar = getattr(value, func_slot)
func = abstract_utils.get_atomic_value(fvar, abstract.Function)
defn = self.value_to_pytd_def(node, func, name)
defn = defn.Visit(visitors.DropMutableParameters())
return defn
def add_decorated_method(name, value, kind):
try:
defn = get_decorated_method(name, value, "func")
except (AttributeError, abstract_utils.ConversionError):
constants[name].add_type(pytd.AnythingType())
return
defn = defn.Replace(kind=kind)
methods[name] = defn
# If decorators are output as aliases to NamedTypes, they will be converted
# to Functions and fail a verification step if those functions have type
# parameters. Since we just want the function name, and since we have a
# fully resolved name at this stage, we just output a minimal pytd.Function
sig = pytd.Signature((), None, None, pytd.AnythingType(), (), ())
decorators = [
pytd.Alias(x, pytd.Function(x, (sig, ), pytd.MethodTypes.METHOD,
0)) for x in v.decorators
]
# class-level attributes
for name, member in v.members.items():
if name in CLASS_LEVEL_IGNORE or name in annotated_names:
continue
for value in member.FilteredData(self.vm.exitpoint, strict=False):
if isinstance(value, special_builtins.PropertyInstance):
# For simplicity, output properties as constants, since our parser
# turns them into constants anyway.
if value.fget:
for typ in self._function_to_return_types(
node, value.fget):
constants[name].add_type(
pytd.Annotated(typ, ("'property'", )))
else:
constants[name].add_type(
pytd.Annotated(pytd.AnythingType(),
("'property'", )))
elif isinstance(value, special_builtins.StaticMethodInstance):
add_decorated_method(name, value,
pytd.MethodTypes.STATICMETHOD)
elif isinstance(value, special_builtins.ClassMethodInstance):
add_decorated_method(name, value,
pytd.MethodTypes.CLASSMETHOD)
elif isinstance(value, abstract.Function):
# value_to_pytd_def returns different pytd node types depending on the
# input type, which pytype struggles to reason about.
method = cast(pytd.Function,
self.value_to_pytd_def(node, value, name))
keep = lambda name: not name or name.startswith(v.name)
signatures = tuple(
s for s in method.signatures
if not s.params or keep(s.params[0].type.name))
if signatures and signatures != method.signatures:
# Filter out calls made from subclasses unless they are the only
# ones recorded; when inferring types for ParentClass.__init__, we
# do not want `self: Union[ParentClass, Subclass]`.
method = method.Replace(signatures=signatures)
# TODO(rechen): Removing mutations altogether won't work for generic
# classes. To support those, we'll need to change the mutated type's
# base to the current class, rename aliased type parameters, and
# replace any parameter not in the class or function template with
# its upper value.
methods[name] = method.Visit(
visitors.DropMutableParameters())
else:
cls = self.vm.convert.merge_classes([value])
node, attr = self.vm.attribute_handler.get_attribute(
node, cls, "__get__")
if attr:
# This attribute is a descriptor. Its type is the return value of
# its __get__ method.
for typ in self._function_to_return_types(node, attr):
constants[name].add_type(typ)
else:
constants[name].add_type(value.to_type(node))
# Instance-level attributes: all attributes from 'canonical' instances (that
# is, ones created by analyze.py:analyze_class()) are added. Attributes from
# non-canonical instances are added if their canonical values do not contain
# type parameters.
ignore = set(annotated_names)
canonical_attributes = set()
def add_attributes_from(instance):
for name, member in instance.members.items():
if name in CLASS_LEVEL_IGNORE or name in ignore:
continue
for value in member.FilteredData(self.vm.exitpoint,
strict=False):
typ = value.to_type(node)
if pytd_utils.GetTypeParameters(typ):
# This attribute's type comes from an annotation that contains a
# type parameter; we do not want to merge in substituted values of
# the type parameter.
canonical_attributes.add(name)
constants[name].add_type(typ)
for instance in v.canonical_instances:
add_attributes_from(instance)
ignore |= canonical_attributes
for instance in v.instances - v.canonical_instances:
add_attributes_from(instance)
for name in list(methods):
if name in constants:
# If something is both a constant and a method, it means that the class
# is, at some point, overwriting its own methods with an attribute.
del methods[name]
constants[name].add_type(pytd.AnythingType())
constants = [
pytd.Constant(name, builder.build())
for name, builder in constants.items() if builder
]
metaclass = v.metaclass(node)
if metaclass is not None:
metaclass = metaclass.get_instance_type(node)
# Some of the class's bases may not be in global scope, so they won't show
# up in the output. In that case, fold the base class's type information
# into this class's pytd.
bases = []
missing_bases = []
for basevar in v.bases():
if basevar.data == [self.vm.convert.oldstyleclass_type]:
continue
elif len(basevar.bindings) == 1:
b, = basevar.data
if b.official_name is None and isinstance(
b, abstract.InterpreterClass):
missing_bases.append(b)
else:
bases.append(b.get_instance_type(node))
else:
bases.append(
pytd_utils.JoinTypes(
b.get_instance_type(node) for b in basevar.data))
# Collect nested classes
# TODO(mdemello): We cannot put these in the output yet; they fail in
# load_dependencies because of the dotted class name (google/pytype#150)
classes = [
self._class_to_def(node, x, x.name) for x in v.get_inner_classes()
]
classes = [x.Replace(name=class_name + "." + x.name) for x in classes]
cls = pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(bases),
methods=tuple(methods.values()),
constants=tuple(constants),
classes=(),
decorators=tuple(decorators),
slots=v.slots,
template=())
for base in missing_bases:
base_cls = self.value_to_pytd_def(node, base, base.name)
cls = pytd_utils.MergeBaseClass(cls, base_cls)
self._scopes.pop()
return cls
def generate_ast(self):
return pytd.Function(
name=self.name,
signatures=tuple(s.pytd_sig for s in self.signatures),
kind=self.kind,
flags=pytd.MethodFlag.abstract_flag(self.is_abstract))
