def _function_call_combination_to_signature(self, func, call_combination, num_combinations): node_after, combination, return_value = call_combination params = [] for i, (name, kwonly, optional) in enumerate(func.get_parameters()): if i < func.nonstararg_count and name in func.signature.annotations: t = func.signature.annotations[name].get_instance_type( node_after) else: t = combination[name].data.to_type(node_after) # Python uses ".0" etc. for the names of parameters that are tuples, # like e.g. in: "def f((x, y), z)". params.append( pytd.Parameter(name.replace(".", "_"), t, kwonly, optional, None)) ret = self._function_call_to_return_type(node_after, func, return_value, num_combinations) if func.has_varargs(): if func.signature.varargs_name in func.signature.annotations: annot = func.signature.annotations[func.signature.varargs_name] typ = annot.get_instance_type(node_after) else: typ = pytd.NamedType("builtins.tuple") starargs = pytd.Parameter(func.signature.varargs_name, typ, False, True, None) else: starargs = None if func.has_kwargs(): if func.signature.kwargs_name in func.signature.annotations: annot = func.signature.annotations[func.signature.kwargs_name] typ = annot.get_instance_type(node_after) else: typ = pytd.NamedType("builtins.dict") starstarargs = pytd.Parameter(func.signature.kwargs_name, typ, False, True, None) else: starstarargs = None return pytd.Signature( params=tuple(params), starargs=starargs, starstarargs=starstarargs, return_type=ret, exceptions=(), # TODO(b/159052087): record exceptions template=())
def set_defaults(self, defaults): """Set signature's default arguments. Requires rebuilding PyTD signature. Args: defaults: An iterable of function argument defaults. Returns: Self with an updated signature. """ defaults = list(defaults) params = [] for param in reversed(self.pytd_sig.params): if defaults: defaults.pop( ) # Discard the default. Unless we want to update type? params.append( pytd.Parameter(name=param.name, type=param.type, kwonly=param.kwonly, optional=True, mutated_type=param.mutated_type)) else: params.append( pytd.Parameter( name=param.name, type=param.type, kwonly=param.kwonly, optional=False, # Reset any previously-set defaults mutated_type=param.mutated_type)) new_sig = pytd.Signature(params=tuple(reversed(params)), starargs=self.pytd_sig.starargs, starstarargs=self.pytd_sig.starstarargs, return_type=self.pytd_sig.return_type, exceptions=self.pytd_sig.exceptions, template=self.pytd_sig.template) # Now update self self.pytd_sig = new_sig self.param_types = [ self.vm.convert.constant_to_value(p.type, subst=datatypes.AliasingDict(), node=self.vm.root_node) for p in self.pytd_sig.params ] self.signature = Signature.from_pytd(self.vm, self.name, self.pytd_sig) return self
def test_signature_annotations(self): # def f(self: Any, *args: Any) self_param = pytd.Parameter("self", pytd.AnythingType(), False, False, None) args_param = pytd.Parameter("args", pytd.AnythingType(), False, True, None) sig = function.Signature.from_pytd( self._vm, "f", pytd.Signature((self_param, ), args_param, None, pytd.AnythingType(), (), ())) self.assertIs(sig.annotations["self"], self._vm.convert.unsolvable) args_type = sig.annotations["args"] # Should be Tuple[Any] self.assertIsInstance(args_type, abstract.ParameterizedClass) self.assertIs(args_type.base_cls, abstract.get_atomic_value(self._vm.convert.tuple_type)) self.assertDictEqual(args_type.type_parameters, {abstract.T: self._vm.convert.unsolvable}) self.assertIs(sig.drop_first_parameter().annotations["args"], args_type)
def _make_init(self) -> function.NameAndSig: """Build an __init__ method for a namedtuple. Builds a dummy __init__ that accepts any arguments. Needed because our model of builtins.tuple uses __init__. Returns: A function.NameAndSig object for an __init__ method. """ self_arg = function.Param("self", pytd.AnythingType()).to_pytd() ret = pytd.NamedType("NoneType") sig = pytd.Signature( params=(self_arg, ), return_type=ret, starargs=function.pytd_default_star_param(), starstarargs=function.pytd_default_starstar_param(), exceptions=(), template=()) return function.NameAndSig("__init__", sig)
def test_signature_from_pytd(self): # def f(self: Any, *args: Any) self_param = pytd.Parameter("self", pytd.AnythingType(), False, False, None) args_param = pytd.Parameter("args", pytd.AnythingType(), False, True, None) sig = function.Signature.from_pytd( self._vm, "f", pytd.Signature((self_param, ), args_param, None, pytd.AnythingType(), (), ())) self.assertEquals(sig.name, "f") self.assertSequenceEqual(sig.param_names, ("self", )) self.assertEquals(sig.varargs_name, "args") self.assertFalse(sig.kwonly_params) self.assertIs(sig.kwargs_name, None) self.assertSetEqual(set(sig.annotations), {"self", "args"}) self.assertFalse(sig.late_annotations) self.assertFalse(sig.has_return_annotation) self.assertTrue(sig.has_param_annotations)
def test_signature_annotations(self): # def f(self: Any, *args: Any) self_param = pytd.Parameter("self", pytd.AnythingType(), False, False, None) # Imitate the parser's conversion of '*args: Any' to '*args: Tuple[Any]'. tup = pytd.ClassType("__builtin__.tuple") tup.cls = self._vm.convert.tuple_type.pytd_cls any_tuple = pytd.GenericType(tup, (pytd.AnythingType(),)) args_param = pytd.Parameter("args", any_tuple, False, True, None) sig = function.Signature.from_pytd( self._vm, "f", pytd.Signature( (self_param,), args_param, None, pytd.AnythingType(), (), ())) self.assertEqual(repr(sig), "def f(self: Any, *args: Tuple[Any, ...]) -> Any") self.assertIs(sig.annotations["self"], self._vm.convert.unsolvable) args_type = sig.annotations["args"] self.assertIsInstance(args_type, abstract.ParameterizedClass) self.assertIs(args_type.base_cls, self._vm.convert.tuple_type) self.assertListEqual(list(args_type.formal_type_parameters.items()), [(abstract_utils.T, self._vm.convert.unsolvable)]) self.assertIs(sig.drop_first_parameter().annotations["args"], args_type)
def DummyMethod(name, *params): """Create a simple method using only "Any"s as types. Arguments: name: The name of the method *params: The parameter names. Returns: A pytd.Function. """ def make_param(param): return pytd.Parameter(param, type=pytd.AnythingType(), kwonly=False, optional=False, mutated_type=None) sig = pytd.Signature(tuple(make_param(param) for param in params), starargs=None, starstarargs=None, return_type=pytd.AnythingType(), exceptions=(), template=()) return pytd.Function(name=name, signatures=(sig,), kind=pytd.METHOD, flags=0)
def _call_traces_to_function(call_traces, prefix=""): funcs = collections.defaultdict(pytd_utils.OrderedSet) for funcvar, args, kws, retvar in call_traces: if isinstance(funcvar.data, abstract.BoundFunction): func = funcvar.data.underlying.signatures[0] else: func = funcvar.data.signatures[0] arg_names = func.get_parameter_names() arg_types = (a.data.to_type() for a in func.get_bound_arguments() + list(args)) ret = pytd_utils.JoinTypes(t.to_type() for t in retvar.data) funcs[funcvar.data.name].add(pytd.Signature( tuple(pytd.Parameter(n, t) for n, t in zip(arg_names, arg_types)) + tuple(pytd.Parameter(name, a.data.to_type()) for name, a in kws), ret, has_optional=False, exceptions=(), template=())) functions = [] for name, signatures in funcs.items(): functions.append(pytd.Function(prefix + name, tuple(signatures))) return functions
def testComplexCombinedType(self): """Test parsing a type with both union and intersection.""" data1 = r"def foo(a: Foo or Bar and Zot) -> object" data2 = r"def foo(a: Foo or (Bar and Zot)) -> object" result1 = self.Parse(data1) result2 = self.Parse(data2) f = pytd.Function( name="foo", signatures=(pytd.Signature(params=(pytd.Parameter( name="a", type=pytd.UnionType( type_list=(pytd.NamedType("Foo"), pytd.IntersectionType( type_list=(pytd.NamedType("Bar"), pytd.NamedType("Zot")))))), ), return_type=pytd.NamedType("object"), template=(), has_optional=False, exceptions=()), )) self.assertEqual(f, result1.Lookup("foo")) self.assertEqual(f, result2.Lookup("foo"))
def _pytd_signature( function: ast3.AST, is_async: bool, exceptions: Optional[List[pytd_node.Node]] = None) -> pytd.Signature: """Construct a pytd signature from an ast.FunctionDef node.""" name = function.name args = function.args pos_params = [Param.from_arg(a, False) for a in args.args] kwonly_params = [Param.from_arg(a, True) for a in args.kwonlyargs] _apply_defaults(pos_params, args.defaults) _apply_defaults(kwonly_params, args.kw_defaults) all_params = pos_params + kwonly_params params = tuple([x.to_pytd() for x in all_params]) starargs = _pytd_star_param(args.vararg) starstarargs = _pytd_starstar_param(args.kwarg) ret = _pytd_return_type(name, function.returns, is_async) exceptions = exceptions or [] return pytd.Signature(params=params, return_type=ret, starargs=starargs, starstarargs=starstarargs, exceptions=tuple(exceptions), template=())
def _simple_func_to_def(self, node, v, name): """Convert a SimpleFunction to a PyTD definition.""" sig = v.signature params = [ pytd.Parameter(p, sig.annotations[p].get_instance_type(node), False, p in sig.defaults, None) for p in sig.param_names ] kwonly = [ pytd.Parameter(p, sig.annotations[p].get_instance_type(node), True, p in sig.defaults, None) for p in sig.kwonly_params ] if sig.varargs_name: star = pytd.Parameter( sig.varargs_name, sig.annotations[sig.varargs_name].get_instance_type(node), False, False, None) else: star = None if sig.kwargs_name: starstar = pytd.Parameter( sig.kwargs_name, sig.annotations[sig.kwargs_name].get_instance_type(node), False, False, None) else: starstar = None if sig.has_return_annotation: ret_type = sig.annotations["return"].get_instance_type(node) else: ret_type = pytd.NamedType("__builtin__.NoneType") pytd_sig = pytd.Signature(params=tuple(params + kwonly), starargs=star, starstarargs=starstar, return_type=ret_type, exceptions=(), template=()) return pytd.Function(name, (pytd_sig, ), pytd.METHOD)
def _call_traces_to_function(call_traces, name_transform=lambda x: x): funcs = collections.defaultdict(pytd_utils.OrderedSet) for node, func, sigs, args, kws, retvar in call_traces: # The lengths may be different in the presence of optional and kw args. arg_names = max((sig.get_positional_names() for sig in sigs), key=len) for i in range(len(arg_names)): if not isinstance(func.data, abstract.BoundFunction) or i > 0: arg_names[i] = function.argname(i) arg_types = (a.data.to_type(node) for a in args) ret = pytd_utils.JoinTypes(t.to_type(node) for t in retvar.data) starargs = None starstarargs = None funcs[func.data.name].add(pytd.Signature( tuple(pytd.Parameter(n, t, False, False, None) for n, t in zip(arg_names, arg_types)) + tuple(pytd.Parameter(name, a.data.to_type(node), False, False, None) for name, a in kws), starargs, starstarargs, ret, exceptions=(), template=())) functions = [] for name, signatures in funcs.items(): functions.append(pytd.Function(name_transform(name), tuple(signatures), pytd.MethodTypes.METHOD)) return functions
def p_funcdef(self, p): """funcdef : DEF NAME LPAREN params RPAREN return raises signature maybe_body""" _, _, name, _, params, _, return_type, raises, _, body = p # TODO(kramm): Output a warning if we already encountered a signature # with these types (but potentially different argument names) if name == '__init__' and isinstance(return_type, pytd.AnythingType): ret = pytd.NamedType('NoneType') else: ret = return_type signature = pytd.Signature(params=tuple(params.required), return_type=ret, exceptions=tuple(raises), template=(), has_optional=params.has_optional) typeparams = { name: pytd.TypeParameter(name) for name in self.context.typevars } used_typeparams = set() signature = signature.Visit( visitors.ReplaceTypes(typeparams, used_typeparams)) if used_typeparams: signature = signature.Replace(template=tuple( pytd.TemplateItem(typeparams[name]) for name in used_typeparams)) for mutator in body: try: signature = signature.Visit(mutator) except NotImplementedError as e: make_syntax_error(self, e.message, p) if not mutator.successful: make_syntax_error(self, 'No parameter named %s' % mutator.name, p) p[0] = NameAndSig(name=name, signature=signature, external_code=False)
def new_function(self, decorators, name, param_list, return_type, body): """Return a _NameAndSig object for the function. Args: decorators: List of decorator names. name: Name of function. param_list: List of parameters, where a paremeter is either a tuple (name, type, default) or the ELLIPSIS special object. See _validate_params for a more detailed description of allowed parameters. return_type: A pytd type object. body: ? Returns: A _NameAndSig object. Raises: ParseError: if any validity checks fail. """ if name == "__init__" and isinstance(return_type, pytd.AnythingType): ret = pytd.NamedType("NoneType") else: ret = return_type params = _validate_params(param_list) exceptions = [] mutators = [] for stmt in body: if isinstance(stmt, pytd.Type): exceptions.append(stmt) # raise stmt continue assert isinstance(stmt, tuple) and len(stmt) == 2, stmt mutators.append(_Mutator(stmt[0], stmt[1])) signature = pytd.Signature(params=tuple(params.required), return_type=ret, starargs=params.starargs, starstarargs=params.starstarargs, exceptions=tuple(exceptions), template=()) for mutator in mutators: try: signature = signature.Visit(mutator) except NotImplementedError as e: raise ParseError(utils.message(e)) if not mutator.successful: raise ParseError("No parameter named %s" % mutator.name) # Remove ignored decorators, raise ParseError for invalid decorators. decorators = {d for d in decorators if _keep_decorator(d)} # Extract out abstractmethod and coroutine decorators, there should be at # most one remaining decorator. def _check_decorator(decorators, decorator_set): exists = any([x in decorators for x in decorator_set]) if exists: decorators -= decorator_set return exists is_abstract = _check_decorator( decorators, {"abstractmethod", "abc.abstractmethod"}) is_coroutine = _check_decorator( decorators, {"coroutine", "async.coroutine", "coroutines.coroutine"}) # TODO(acaceres): if not inside a class, any decorator should be an error if len(decorators) > 1: raise ParseError("Too many decorators for %s" % name) decorator, = decorators if decorators else (None,) return _NameAndSig(name=name, signature=signature, decorator=decorator, is_abstract=is_abstract, is_coroutine=is_coroutine)
def _class_to_def(self, node, v, class_name): """Convert an InterpreterClass to a PyTD definition.""" self._scopes.append(class_name) methods = {} constants = collections.defaultdict(pytd_utils.TypeBuilder) annots = abstract_utils.get_annotations_dict(v.members) annotated_names = set() def add_constants(iterator): for name, t in iterator: if t is None: # Remove the entry from constants annotated_names.add(name) elif name not in annotated_names: constants[name].add_type(t) annotated_names.add(name) add_constants( self._ordered_attrs_to_instance_types(node, v.metadata, annots)) add_constants(self.annotations_to_instance_types(node, annots)) def get_decorated_method(name, value, func_slot): fvar = getattr(value, func_slot) func = abstract_utils.get_atomic_value(fvar, abstract.Function) defn = self.value_to_pytd_def(node, func, name) defn = defn.Visit(visitors.DropMutableParameters()) return defn def add_decorated_method(name, value, kind): try: defn = get_decorated_method(name, value, "func") except (AttributeError, abstract_utils.ConversionError): constants[name].add_type(pytd.AnythingType()) return defn = defn.Replace(kind=kind) methods[name] = defn # If decorators are output as aliases to NamedTypes, they will be converted # to Functions and fail a verification step if those functions have type # parameters. Since we just want the function name, and since we have a # fully resolved name at this stage, we just output a minimal pytd.Function sig = pytd.Signature((), None, None, pytd.AnythingType(), (), ()) decorators = [ pytd.Alias(x, pytd.Function(x, (sig, ), pytd.MethodTypes.METHOD, 0)) for x in v.decorators ] # class-level attributes for name, member in v.members.items(): if name in CLASS_LEVEL_IGNORE or name in annotated_names: continue for value in member.FilteredData(self.vm.exitpoint, strict=False): if isinstance(value, special_builtins.PropertyInstance): # For simplicity, output properties as constants, since our parser # turns them into constants anyway. if value.fget: for typ in self._function_to_return_types( node, value.fget): constants[name].add_type( pytd.Annotated(typ, ("'property'", ))) else: constants[name].add_type( pytd.Annotated(pytd.AnythingType(), ("'property'", ))) elif isinstance(value, special_builtins.StaticMethodInstance): add_decorated_method(name, value, pytd.MethodTypes.STATICMETHOD) elif isinstance(value, special_builtins.ClassMethodInstance): add_decorated_method(name, value, pytd.MethodTypes.CLASSMETHOD) elif isinstance(value, abstract.Function): # value_to_pytd_def returns different pytd node types depending on the # input type, which pytype struggles to reason about. method = cast(pytd.Function, self.value_to_pytd_def(node, value, name)) keep = lambda name: not name or name.startswith(v.name) signatures = tuple( s for s in method.signatures if not s.params or keep(s.params[0].type.name)) if signatures and signatures != method.signatures: # Filter out calls made from subclasses unless they are the only # ones recorded; when inferring types for ParentClass.__init__, we # do not want `self: Union[ParentClass, Subclass]`. method = method.Replace(signatures=signatures) # TODO(rechen): Removing mutations altogether won't work for generic # classes. To support those, we'll need to change the mutated type's # base to the current class, rename aliased type parameters, and # replace any parameter not in the class or function template with # its upper value. methods[name] = method.Visit( visitors.DropMutableParameters()) else: cls = self.vm.convert.merge_classes([value]) node, attr = self.vm.attribute_handler.get_attribute( node, cls, "__get__") if attr: # This attribute is a descriptor. Its type is the return value of # its __get__ method. for typ in self._function_to_return_types(node, attr): constants[name].add_type(typ) else: constants[name].add_type(value.to_type(node)) # Instance-level attributes: all attributes from 'canonical' instances (that # is, ones created by analyze.py:analyze_class()) are added. Attributes from # non-canonical instances are added if their canonical values do not contain # type parameters. ignore = set(annotated_names) canonical_attributes = set() def add_attributes_from(instance): for name, member in instance.members.items(): if name in CLASS_LEVEL_IGNORE or name in ignore: continue for value in member.FilteredData(self.vm.exitpoint, strict=False): typ = value.to_type(node) if pytd_utils.GetTypeParameters(typ): # This attribute's type comes from an annotation that contains a # type parameter; we do not want to merge in substituted values of # the type parameter. canonical_attributes.add(name) constants[name].add_type(typ) for instance in v.canonical_instances: add_attributes_from(instance) ignore |= canonical_attributes for instance in v.instances - v.canonical_instances: add_attributes_from(instance) for name in list(methods): if name in constants: # If something is both a constant and a method, it means that the class # is, at some point, overwriting its own methods with an attribute. del methods[name] constants[name].add_type(pytd.AnythingType()) constants = [ pytd.Constant(name, builder.build()) for name, builder in constants.items() if builder ] metaclass = v.metaclass(node) if metaclass is not None: metaclass = metaclass.get_instance_type(node) # Some of the class's bases may not be in global scope, so they won't show # up in the output. In that case, fold the base class's type information # into this class's pytd. bases = [] missing_bases = [] for basevar in v.bases(): if basevar.data == [self.vm.convert.oldstyleclass_type]: continue elif len(basevar.bindings) == 1: b, = basevar.data if b.official_name is None and isinstance( b, abstract.InterpreterClass): missing_bases.append(b) else: bases.append(b.get_instance_type(node)) else: bases.append( pytd_utils.JoinTypes( b.get_instance_type(node) for b in basevar.data)) # Collect nested classes # TODO(mdemello): We cannot put these in the output yet; they fail in # load_dependencies because of the dotted class name (google/pytype#150) classes = [ self._class_to_def(node, x, x.name) for x in v.get_inner_classes() ] classes = [x.Replace(name=class_name + "." + x.name) for x in classes] cls = pytd.Class(name=class_name, metaclass=metaclass, parents=tuple(bases), methods=tuple(methods.values()), constants=tuple(constants), classes=(), decorators=tuple(decorators), slots=v.slots, template=()) for base in missing_bases: base_cls = self.value_to_pytd_def(node, base, base.name) cls = pytd_utils.MergeBaseClass(cls, base_cls) self._scopes.pop() return cls
def test_compatible_with(self): pytd_sig = pytd.Signature((), None, None, pytd.AnythingType(), (), ()) sig = function.PyTDSignature("f", pytd_sig, self._vm) f = abstract.PyTDFunction("f", (sig, ), pytd.METHOD, self._vm) self.assertTruthy(f)
def assertNotHasSignature(self, func, parameter_types, return_type): target = pytd.Signature(tuple(parameter_types), return_type, (), (), False) if self.HasSignature(func, target): # TODO(pludemann): don't assume function is 'f' self.fail("Found signature: f{} -> {} in {}". format(pytd.Print(target), pytd.Print(func)))