def _function_call_combination_to_signature(self, func, call_combination,
num_combinations):
node_after, combination, return_value = call_combination
params = []
for i, (name, kwonly, optional) in enumerate(func.get_parameters()):
if i < func.nonstararg_count and name in func.signature.annotations:
t = func.signature.annotations[name].get_instance_type(
node_after)
else:
t = combination[name].data.to_type(node_after)
# Python uses ".0" etc. for the names of parameters that are tuples,
# like e.g. in: "def f((x, y), z)".
params.append(
pytd.Parameter(name.replace(".", "_"), t, kwonly, optional,
None))
ret = self._function_call_to_return_type(node_after, func,
return_value,
num_combinations)
if func.has_varargs():
if func.signature.varargs_name in func.signature.annotations:
annot = func.signature.annotations[func.signature.varargs_name]
typ = annot.get_instance_type(node_after)
else:
typ = pytd.NamedType("builtins.tuple")
starargs = pytd.Parameter(func.signature.varargs_name, typ, False,
True, None)
else:
starargs = None
if func.has_kwargs():
if func.signature.kwargs_name in func.signature.annotations:
annot = func.signature.annotations[func.signature.kwargs_name]
typ = annot.get_instance_type(node_after)
else:
typ = pytd.NamedType("builtins.dict")
starstarargs = pytd.Parameter(func.signature.kwargs_name, typ,
False, True, None)
else:
starstarargs = None
return pytd.Signature(
params=tuple(params),
starargs=starargs,
starstarargs=starstarargs,
return_type=ret,
exceptions=(), # TODO(b/159052087): record exceptions
template=())
def set_defaults(self, defaults):
"""Set signature's default arguments. Requires rebuilding PyTD signature.
Args:
defaults: An iterable of function argument defaults.
Returns:
Self with an updated signature.
"""
defaults = list(defaults)
params = []
for param in reversed(self.pytd_sig.params):
if defaults:
defaults.pop(
) # Discard the default. Unless we want to update type?
params.append(
pytd.Parameter(name=param.name,
type=param.type,
kwonly=param.kwonly,
optional=True,
mutated_type=param.mutated_type))
else:
params.append(
pytd.Parameter(
name=param.name,
type=param.type,
kwonly=param.kwonly,
optional=False, # Reset any previously-set defaults
mutated_type=param.mutated_type))
new_sig = pytd.Signature(params=tuple(reversed(params)),
starargs=self.pytd_sig.starargs,
starstarargs=self.pytd_sig.starstarargs,
return_type=self.pytd_sig.return_type,
exceptions=self.pytd_sig.exceptions,
template=self.pytd_sig.template)
# Now update self
self.pytd_sig = new_sig
self.param_types = [
self.vm.convert.constant_to_value(p.type,
subst=datatypes.AliasingDict(),
node=self.vm.root_node)
for p in self.pytd_sig.params
]
self.signature = Signature.from_pytd(self.vm, self.name, self.pytd_sig)
return self
def test_signature_annotations(self):
# def f(self: Any, *args: Any)
self_param = pytd.Parameter("self", pytd.AnythingType(), False, False,
None)
args_param = pytd.Parameter("args", pytd.AnythingType(), False, True,
None)
sig = function.Signature.from_pytd(
self._vm, "f",
pytd.Signature((self_param, ), args_param, None,
pytd.AnythingType(), (), ()))
self.assertIs(sig.annotations["self"], self._vm.convert.unsolvable)
args_type = sig.annotations["args"] # Should be Tuple[Any]
self.assertIsInstance(args_type, abstract.ParameterizedClass)
self.assertIs(args_type.base_cls,
abstract.get_atomic_value(self._vm.convert.tuple_type))
self.assertDictEqual(args_type.type_parameters,
{abstract.T: self._vm.convert.unsolvable})
self.assertIs(sig.drop_first_parameter().annotations["args"],
args_type)
def _make_init(self) -> function.NameAndSig:
"""Build an __init__ method for a namedtuple.
Builds a dummy __init__ that accepts any arguments. Needed because our
model of builtins.tuple uses __init__.
Returns:
A function.NameAndSig object for an __init__ method.
"""
self_arg = function.Param("self", pytd.AnythingType()).to_pytd()
ret = pytd.NamedType("NoneType")
sig = pytd.Signature(
params=(self_arg, ),
return_type=ret,
starargs=function.pytd_default_star_param(),
starstarargs=function.pytd_default_starstar_param(),
exceptions=(),
template=())
return function.NameAndSig("__init__", sig)
def test_signature_from_pytd(self):
# def f(self: Any, *args: Any)
self_param = pytd.Parameter("self", pytd.AnythingType(), False, False,
None)
args_param = pytd.Parameter("args", pytd.AnythingType(), False, True,
None)
sig = function.Signature.from_pytd(
self._vm, "f",
pytd.Signature((self_param, ), args_param, None,
pytd.AnythingType(), (), ()))
self.assertEquals(sig.name, "f")
self.assertSequenceEqual(sig.param_names, ("self", ))
self.assertEquals(sig.varargs_name, "args")
self.assertFalse(sig.kwonly_params)
self.assertIs(sig.kwargs_name, None)
self.assertSetEqual(set(sig.annotations), {"self", "args"})
self.assertFalse(sig.late_annotations)
self.assertFalse(sig.has_return_annotation)
self.assertTrue(sig.has_param_annotations)
def test_signature_annotations(self):
# def f(self: Any, *args: Any)
self_param = pytd.Parameter("self", pytd.AnythingType(), False, False, None)
# Imitate the parser's conversion of '*args: Any' to '*args: Tuple[Any]'.
tup = pytd.ClassType("__builtin__.tuple")
tup.cls = self._vm.convert.tuple_type.pytd_cls
any_tuple = pytd.GenericType(tup, (pytd.AnythingType(),))
args_param = pytd.Parameter("args", any_tuple, False, True, None)
sig = function.Signature.from_pytd(
self._vm, "f", pytd.Signature(
(self_param,), args_param, None, pytd.AnythingType(), (), ()))
self.assertEqual(repr(sig),
"def f(self: Any, *args: Tuple[Any, ...]) -> Any")
self.assertIs(sig.annotations["self"], self._vm.convert.unsolvable)
args_type = sig.annotations["args"]
self.assertIsInstance(args_type, abstract.ParameterizedClass)
self.assertIs(args_type.base_cls, self._vm.convert.tuple_type)
self.assertListEqual(list(args_type.formal_type_parameters.items()),
[(abstract_utils.T, self._vm.convert.unsolvable)])
self.assertIs(sig.drop_first_parameter().annotations["args"], args_type)
def DummyMethod(name, *params):
"""Create a simple method using only "Any"s as types.
Arguments:
name: The name of the method
*params: The parameter names.
Returns:
A pytd.Function.
"""
def make_param(param):
return pytd.Parameter(param, type=pytd.AnythingType(), kwonly=False,
optional=False, mutated_type=None)
sig = pytd.Signature(tuple(make_param(param) for param in params),
starargs=None, starstarargs=None,
return_type=pytd.AnythingType(),
exceptions=(), template=())
return pytd.Function(name=name,
signatures=(sig,),
kind=pytd.METHOD,
flags=0)
def _call_traces_to_function(call_traces, prefix=""):
funcs = collections.defaultdict(pytd_utils.OrderedSet)
for funcvar, args, kws, retvar in call_traces:
if isinstance(funcvar.data, abstract.BoundFunction):
func = funcvar.data.underlying.signatures[0]
else:
func = funcvar.data.signatures[0]
arg_names = func.get_parameter_names()
arg_types = (a.data.to_type()
for a in func.get_bound_arguments() + list(args))
ret = pytd_utils.JoinTypes(t.to_type() for t in retvar.data)
funcs[funcvar.data.name].add(pytd.Signature(
tuple(pytd.Parameter(n, t)
for n, t in zip(arg_names, arg_types)) +
tuple(pytd.Parameter(name, a.data.to_type())
for name, a in kws),
ret, has_optional=False, exceptions=(), template=()))
functions = []
for name, signatures in funcs.items():
functions.append(pytd.Function(prefix + name, tuple(signatures)))
return functions
def testComplexCombinedType(self):
"""Test parsing a type with both union and intersection."""
data1 = r"def foo(a: Foo or Bar and Zot) -> object"
data2 = r"def foo(a: Foo or (Bar and Zot)) -> object"
result1 = self.Parse(data1)
result2 = self.Parse(data2)
f = pytd.Function(
name="foo",
signatures=(pytd.Signature(params=(pytd.Parameter(
name="a",
type=pytd.UnionType(
type_list=(pytd.NamedType("Foo"),
pytd.IntersectionType(
type_list=(pytd.NamedType("Bar"),
pytd.NamedType("Zot")))))), ),
return_type=pytd.NamedType("object"),
template=(),
has_optional=False,
exceptions=()), ))
self.assertEqual(f, result1.Lookup("foo"))
self.assertEqual(f, result2.Lookup("foo"))
def _pytd_signature(
function: ast3.AST,
is_async: bool,
exceptions: Optional[List[pytd_node.Node]] = None) -> pytd.Signature:
"""Construct a pytd signature from an ast.FunctionDef node."""
name = function.name
args = function.args
pos_params = [Param.from_arg(a, False) for a in args.args]
kwonly_params = [Param.from_arg(a, True) for a in args.kwonlyargs]
_apply_defaults(pos_params, args.defaults)
_apply_defaults(kwonly_params, args.kw_defaults)
all_params = pos_params + kwonly_params
params = tuple([x.to_pytd() for x in all_params])
starargs = _pytd_star_param(args.vararg)
starstarargs = _pytd_starstar_param(args.kwarg)
ret = _pytd_return_type(name, function.returns, is_async)
exceptions = exceptions or []
return pytd.Signature(params=params,
return_type=ret,
starargs=starargs,
starstarargs=starstarargs,
exceptions=tuple(exceptions),
template=())
def _simple_func_to_def(self, node, v, name):
"""Convert a SimpleFunction to a PyTD definition."""
sig = v.signature
params = [
pytd.Parameter(p, sig.annotations[p].get_instance_type(node),
False, p in sig.defaults, None)
for p in sig.param_names
]
kwonly = [
pytd.Parameter(p, sig.annotations[p].get_instance_type(node), True,
p in sig.defaults, None) for p in sig.kwonly_params
]
if sig.varargs_name:
star = pytd.Parameter(
sig.varargs_name,
sig.annotations[sig.varargs_name].get_instance_type(node),
False, False, None)
else:
star = None
if sig.kwargs_name:
starstar = pytd.Parameter(
sig.kwargs_name,
sig.annotations[sig.kwargs_name].get_instance_type(node),
False, False, None)
else:
starstar = None
if sig.has_return_annotation:
ret_type = sig.annotations["return"].get_instance_type(node)
else:
ret_type = pytd.NamedType("__builtin__.NoneType")
pytd_sig = pytd.Signature(params=tuple(params + kwonly),
starargs=star,
starstarargs=starstar,
return_type=ret_type,
exceptions=(),
template=())
return pytd.Function(name, (pytd_sig, ), pytd.METHOD)
def _call_traces_to_function(call_traces, name_transform=lambda x: x):
funcs = collections.defaultdict(pytd_utils.OrderedSet)
for node, func, sigs, args, kws, retvar in call_traces:
# The lengths may be different in the presence of optional and kw args.
arg_names = max((sig.get_positional_names() for sig in sigs), key=len)
for i in range(len(arg_names)):
if not isinstance(func.data, abstract.BoundFunction) or i > 0:
arg_names[i] = function.argname(i)
arg_types = (a.data.to_type(node) for a in args)
ret = pytd_utils.JoinTypes(t.to_type(node) for t in retvar.data)
starargs = None
starstarargs = None
funcs[func.data.name].add(pytd.Signature(
tuple(pytd.Parameter(n, t, False, False, None)
for n, t in zip(arg_names, arg_types)) +
tuple(pytd.Parameter(name, a.data.to_type(node), False, False, None)
for name, a in kws),
starargs, starstarargs,
ret, exceptions=(), template=()))
functions = []
for name, signatures in funcs.items():
functions.append(pytd.Function(name_transform(name), tuple(signatures),
pytd.MethodTypes.METHOD))
return functions
def p_funcdef(self, p):
"""funcdef : DEF NAME LPAREN params RPAREN return raises signature maybe_body"""
_, _, name, _, params, _, return_type, raises, _, body = p
# TODO(kramm): Output a warning if we already encountered a signature
# with these types (but potentially different argument names)
if name == '__init__' and isinstance(return_type, pytd.AnythingType):
ret = pytd.NamedType('NoneType')
else:
ret = return_type
signature = pytd.Signature(params=tuple(params.required),
return_type=ret,
exceptions=tuple(raises),
template=(),
has_optional=params.has_optional)
typeparams = {
name: pytd.TypeParameter(name)
for name in self.context.typevars
}
used_typeparams = set()
signature = signature.Visit(
visitors.ReplaceTypes(typeparams, used_typeparams))
if used_typeparams:
signature = signature.Replace(template=tuple(
pytd.TemplateItem(typeparams[name])
for name in used_typeparams))
for mutator in body:
try:
signature = signature.Visit(mutator)
except NotImplementedError as e:
make_syntax_error(self, e.message, p)
if not mutator.successful:
make_syntax_error(self, 'No parameter named %s' % mutator.name,
p)
p[0] = NameAndSig(name=name, signature=signature, external_code=False)
def new_function(self, decorators, name, param_list, return_type, body):
"""Return a _NameAndSig object for the function.
Args:
decorators: List of decorator names.
name: Name of function.
param_list: List of parameters, where a paremeter is either a tuple
(name, type, default) or the ELLIPSIS special object. See
_validate_params for a more detailed description of allowed parameters.
return_type: A pytd type object.
body: ?
Returns:
A _NameAndSig object.
Raises:
ParseError: if any validity checks fail.
"""
if name == "__init__" and isinstance(return_type, pytd.AnythingType):
ret = pytd.NamedType("NoneType")
else:
ret = return_type
params = _validate_params(param_list)
exceptions = []
mutators = []
for stmt in body:
if isinstance(stmt, pytd.Type):
exceptions.append(stmt) # raise stmt
continue
assert isinstance(stmt, tuple) and len(stmt) == 2, stmt
mutators.append(_Mutator(stmt[0], stmt[1]))
signature = pytd.Signature(params=tuple(params.required), return_type=ret,
starargs=params.starargs,
starstarargs=params.starstarargs,
exceptions=tuple(exceptions), template=())
for mutator in mutators:
try:
signature = signature.Visit(mutator)
except NotImplementedError as e:
raise ParseError(utils.message(e))
if not mutator.successful:
raise ParseError("No parameter named %s" % mutator.name)
# Remove ignored decorators, raise ParseError for invalid decorators.
decorators = {d for d in decorators if _keep_decorator(d)}
# Extract out abstractmethod and coroutine decorators, there should be at
# most one remaining decorator.
def _check_decorator(decorators, decorator_set):
exists = any([x in decorators for x in decorator_set])
if exists:
decorators -= decorator_set
return exists
is_abstract = _check_decorator(
decorators, {"abstractmethod", "abc.abstractmethod"})
is_coroutine = _check_decorator(
decorators, {"coroutine", "async.coroutine", "coroutines.coroutine"})
# TODO(acaceres): if not inside a class, any decorator should be an error
if len(decorators) > 1:
raise ParseError("Too many decorators for %s" % name)
decorator, = decorators if decorators else (None,)
return _NameAndSig(name=name, signature=signature,
decorator=decorator,
is_abstract=is_abstract,
is_coroutine=is_coroutine)
def _class_to_def(self, node, v, class_name):
"""Convert an InterpreterClass to a PyTD definition."""
self._scopes.append(class_name)
methods = {}
constants = collections.defaultdict(pytd_utils.TypeBuilder)
annots = abstract_utils.get_annotations_dict(v.members)
annotated_names = set()
def add_constants(iterator):
for name, t in iterator:
if t is None:
# Remove the entry from constants
annotated_names.add(name)
elif name not in annotated_names:
constants[name].add_type(t)
annotated_names.add(name)
add_constants(
self._ordered_attrs_to_instance_types(node, v.metadata, annots))
add_constants(self.annotations_to_instance_types(node, annots))
def get_decorated_method(name, value, func_slot):
fvar = getattr(value, func_slot)
func = abstract_utils.get_atomic_value(fvar, abstract.Function)
defn = self.value_to_pytd_def(node, func, name)
defn = defn.Visit(visitors.DropMutableParameters())
return defn
def add_decorated_method(name, value, kind):
try:
defn = get_decorated_method(name, value, "func")
except (AttributeError, abstract_utils.ConversionError):
constants[name].add_type(pytd.AnythingType())
return
defn = defn.Replace(kind=kind)
methods[name] = defn
# If decorators are output as aliases to NamedTypes, they will be converted
# to Functions and fail a verification step if those functions have type
# parameters. Since we just want the function name, and since we have a
# fully resolved name at this stage, we just output a minimal pytd.Function
sig = pytd.Signature((), None, None, pytd.AnythingType(), (), ())
decorators = [
pytd.Alias(x, pytd.Function(x, (sig, ), pytd.MethodTypes.METHOD,
0)) for x in v.decorators
]
# class-level attributes
for name, member in v.members.items():
if name in CLASS_LEVEL_IGNORE or name in annotated_names:
continue
for value in member.FilteredData(self.vm.exitpoint, strict=False):
if isinstance(value, special_builtins.PropertyInstance):
# For simplicity, output properties as constants, since our parser
# turns them into constants anyway.
if value.fget:
for typ in self._function_to_return_types(
node, value.fget):
constants[name].add_type(
pytd.Annotated(typ, ("'property'", )))
else:
constants[name].add_type(
pytd.Annotated(pytd.AnythingType(),
("'property'", )))
elif isinstance(value, special_builtins.StaticMethodInstance):
add_decorated_method(name, value,
pytd.MethodTypes.STATICMETHOD)
elif isinstance(value, special_builtins.ClassMethodInstance):
add_decorated_method(name, value,
pytd.MethodTypes.CLASSMETHOD)
elif isinstance(value, abstract.Function):
# value_to_pytd_def returns different pytd node types depending on the
# input type, which pytype struggles to reason about.
method = cast(pytd.Function,
self.value_to_pytd_def(node, value, name))
keep = lambda name: not name or name.startswith(v.name)
signatures = tuple(
s for s in method.signatures
if not s.params or keep(s.params[0].type.name))
if signatures and signatures != method.signatures:
# Filter out calls made from subclasses unless they are the only
# ones recorded; when inferring types for ParentClass.__init__, we
# do not want `self: Union[ParentClass, Subclass]`.
method = method.Replace(signatures=signatures)
# TODO(rechen): Removing mutations altogether won't work for generic
# classes. To support those, we'll need to change the mutated type's
# base to the current class, rename aliased type parameters, and
# replace any parameter not in the class or function template with
# its upper value.
methods[name] = method.Visit(
visitors.DropMutableParameters())
else:
cls = self.vm.convert.merge_classes([value])
node, attr = self.vm.attribute_handler.get_attribute(
node, cls, "__get__")
if attr:
# This attribute is a descriptor. Its type is the return value of
# its __get__ method.
for typ in self._function_to_return_types(node, attr):
constants[name].add_type(typ)
else:
constants[name].add_type(value.to_type(node))
# Instance-level attributes: all attributes from 'canonical' instances (that
# is, ones created by analyze.py:analyze_class()) are added. Attributes from
# non-canonical instances are added if their canonical values do not contain
# type parameters.
ignore = set(annotated_names)
canonical_attributes = set()
def add_attributes_from(instance):
for name, member in instance.members.items():
if name in CLASS_LEVEL_IGNORE or name in ignore:
continue
for value in member.FilteredData(self.vm.exitpoint,
strict=False):
typ = value.to_type(node)
if pytd_utils.GetTypeParameters(typ):
# This attribute's type comes from an annotation that contains a
# type parameter; we do not want to merge in substituted values of
# the type parameter.
canonical_attributes.add(name)
constants[name].add_type(typ)
for instance in v.canonical_instances:
add_attributes_from(instance)
ignore |= canonical_attributes
for instance in v.instances - v.canonical_instances:
add_attributes_from(instance)
for name in list(methods):
if name in constants:
# If something is both a constant and a method, it means that the class
# is, at some point, overwriting its own methods with an attribute.
del methods[name]
constants[name].add_type(pytd.AnythingType())
constants = [
pytd.Constant(name, builder.build())
for name, builder in constants.items() if builder
]
metaclass = v.metaclass(node)
if metaclass is not None:
metaclass = metaclass.get_instance_type(node)
# Some of the class's bases may not be in global scope, so they won't show
# up in the output. In that case, fold the base class's type information
# into this class's pytd.
bases = []
missing_bases = []
for basevar in v.bases():
if basevar.data == [self.vm.convert.oldstyleclass_type]:
continue
elif len(basevar.bindings) == 1:
b, = basevar.data
if b.official_name is None and isinstance(
b, abstract.InterpreterClass):
missing_bases.append(b)
else:
bases.append(b.get_instance_type(node))
else:
bases.append(
pytd_utils.JoinTypes(
b.get_instance_type(node) for b in basevar.data))
# Collect nested classes
# TODO(mdemello): We cannot put these in the output yet; they fail in
# load_dependencies because of the dotted class name (google/pytype#150)
classes = [
self._class_to_def(node, x, x.name) for x in v.get_inner_classes()
]
classes = [x.Replace(name=class_name + "." + x.name) for x in classes]
cls = pytd.Class(name=class_name,
metaclass=metaclass,
parents=tuple(bases),
methods=tuple(methods.values()),
constants=tuple(constants),
classes=(),
decorators=tuple(decorators),
slots=v.slots,
template=())
for base in missing_bases:
base_cls = self.value_to_pytd_def(node, base, base.name)
cls = pytd_utils.MergeBaseClass(cls, base_cls)
self._scopes.pop()
return cls
def test_compatible_with(self):
pytd_sig = pytd.Signature((), None, None, pytd.AnythingType(), (), ())
sig = function.PyTDSignature("f", pytd_sig, self._vm)
f = abstract.PyTDFunction("f", (sig, ), pytd.METHOD, self._vm)
self.assertTruthy(f)
def assertNotHasSignature(self, func, parameter_types, return_type):
target = pytd.Signature(tuple(parameter_types), return_type, (), (), False)
if self.HasSignature(func, target):
# TODO(pludemann): don't assume function is 'f'
self.fail("Found signature: f{} -> {} in {}".
format(pytd.Print(target), pytd.Print(func)))
