def testAdjustTypeParametersWithBuiltins(self): ast = self.ParseWithBuiltins(""" T = TypeVar("T") K = TypeVar("K") V = TypeVar("V") class Foo(List[int]): pass class Bar(Dict[T, int]): pass class Baz(Generic[K, V]): pass class Qux(Baz[str, int]): pass """) foo = ast.Lookup("Foo") bar = ast.Lookup("Bar") qux = ast.Lookup("Qux") foo_parent, = foo.parents bar_parent, = bar.parents qux_parent, = qux.parents # Expected: # Class(Foo, parent=GenericType(List, parameters=(int,)), template=()) # Class(Bar, parent=GenericType(Dict, parameters=(T, int)), template=(T)) # Class(Qux, parent=GenericType(Baz, parameters=(str, int)), template=()) self.assertEqual((pytd.ClassType("int"),), foo_parent.parameters) self.assertEqual((), foo.template) self.assertEqual( (pytd.TypeParameter("T", scope="Bar"), pytd.ClassType("int")), bar_parent.parameters) self.assertEqual( (pytd.TemplateItem(pytd.TypeParameter("T", scope="Bar")),), bar.template) self.assertEqual((pytd.ClassType("str"), pytd.ClassType("int")), qux_parent.parameters) self.assertEqual((), qux.template)
def testConcatTypeParameters(self): """Test for concatenating ASTs with type parameters.""" ast1 = self.Parse("""T = TypeVar("T")""", name="__builtin__") ast2 = self.Parse("""T = TypeVar("T")""") combined = pytd_utils.Concat(ast1, ast2) self.assertEqual(combined.Lookup("__builtin__.T"), pytd.TypeParameter("T", scope="__builtin__")) self.assertEqual(combined.Lookup("T"), pytd.TypeParameter("T", scope=None))
def testAdjustTypeParametersWithDuplicates(self): ast = self.ParseWithBuiltins(""" T = TypeVar("T") class A(Dict[T, T], Generic[T]): pass """) a = ast.Lookup("A") self.assertEqual( (pytd.TemplateItem(pytd.TypeParameter("T", (), None, "A")), pytd.TemplateItem(pytd.TypeParameter("T", (), None, "A"))), a.template)
def SplitParents(parser, p, parents): """Strip the special Generic[...] class out of the base classes.""" template = () other_parents = [] for parent in parents: if (isinstance(parent, pytd.GenericType) and parent.base_type == pytd.ExternalType('Generic', 'typing')): if not all( isinstance(param, pytd.NamedType) for param in parent.parameters): make_syntax_error( parser, 'Illegal template parameter %s' % pytd.Print(parent), p) if template: make_syntax_error(parser, 'Duplicate Template base class', p) template = tuple( pytd.TemplateItem(pytd.TypeParameter(param.name)) for param in parent.parameters) all_names = [t.name for t in template] duplicates = [ name for name, count in collections.Counter(all_names).items() if count >= 2 ] if duplicates: make_syntax_error( parser, 'Duplicate template parameters' + ', '.join(duplicates), p) else: if parent != pytd.NothingType(): other_parents.append(parent) return template, tuple(other_parents)
def value_to_pytd_def(self, node, v, name): """Get a PyTD definition for this object. Args: node: The node. v: The object. name: The object name. Returns: A PyTD definition. """ if isinstance(v, abstract.PyTDFunction): return pytd.Function( name, tuple(sig.pytd_sig for sig in v.signatures), pytd.METHOD) elif isinstance(v, abstract.InterpreterFunction): return self._function_to_def(node, v, name) elif isinstance(v, abstract.ParameterizedClass): return pytd.Alias(name, v.get_instance_type(node)) elif isinstance(v, abstract.PyTDClass): # This happens if a module does e.g. "from x import y as z", i.e., copies # something from another module to the local namespace. We *could* # reproduce the entire class, but we choose a more dense representation. return v.to_type(node) elif isinstance(v, abstract.InterpreterClass): return self._class_to_def(node, v, name) elif isinstance(v, abstract.TypeVariable): return pytd.TypeParameter(name, None) elif isinstance(v, abstract.Unsolvable): return pytd.Constant(name, v.to_type(node)) else: raise NotImplementedError(v.__class__.__name__)
def _parameterized_type(self, base_type: Any, parameters): """Return a parameterized type.""" if self._matches_named_type(base_type, _LITERAL_TYPES): return pytd_literal(parameters) elif self._matches_named_type(base_type, _ANNOTATED_TYPES): return pytd_annotated(parameters) elif self._matches_named_type(base_type, _FINAL_TYPES): typ, = parameters return pytd.GenericType(pytd.NamedType("typing.Final"), (typ, )) elif self._matches_named_type(base_type, _TYPEGUARD_TYPES): # We do not yet support PEP 647, User-Defined Type Guards. To avoid # blocking typeshed, convert type guards to plain bools. return pytd.NamedType("bool") elif any(isinstance(p, types.Pyval) for p in parameters): parameters = ", ".join( p.repr_str() if isinstance(p, types.Pyval) else "_" for p in parameters) raise ParseError("%s[%s] not supported" % (pytd_utils.Print(base_type), parameters)) elif pytdgen.is_any(base_type): return pytd.AnythingType() elif len(parameters) == 2 and parameters[-1] is self.ELLIPSIS and ( not self._matches_named_type(base_type, _CALLABLE_TYPES)): element_type = parameters[0] if element_type is self.ELLIPSIS: raise ParseError("[..., ...] not supported") return pytd.GenericType(base_type=base_type, parameters=(element_type, )) else: processed_parameters = [] # We do not yet support PEP 612, Parameter Specification Variables. # To avoid blocking typeshed from adopting this PEP, we convert new # features to approximations that only use supported features. for p in parameters: if p is self.ELLIPSIS: processed = pytd.AnythingType() elif (p in self.param_specs and self._matches_full_name( base_type, "typing.Generic")): # Replacing a ParamSpec with a TypeVar isn't correct, but it'll work # for simple cases in which the filled value is also a ParamSpec. if not any(t.name == p.name for t in self.type_params): self.type_params.append(pytd.TypeParameter(p.name)) processed = p elif (p in self.param_specs or (isinstance(p, pytd.GenericType) and self._matches_full_name(p, _CONCATENATE_TYPES))): processed = pytd.AnythingType() else: processed = p processed_parameters.append(processed) parameters = tuple(processed_parameters) if self._matches_named_type(base_type, _TUPLE_TYPES): return pytdgen.heterogeneous_tuple(base_type, parameters) elif self._matches_named_type(base_type, _CALLABLE_TYPES): return pytdgen.pytd_callable(base_type, parameters) else: assert parameters return pytd.GenericType(base_type=base_type, parameters=parameters)
def add_type_var(self, name, typevar): """Add a type variable, <name> = TypeVar(<name_arg>, <args>).""" if name != typevar.name: raise ParseError("TypeVar name needs to be %r (not %r)" % ( typevar.name, name)) bound = typevar.bound if isinstance(bound, str): bound = pytd.NamedType(bound) constraints = tuple(typevar.constraints) if typevar.constraints else () self.type_params.append(pytd.TypeParameter( name=name, constraints=constraints, bound=bound))
def testAdjustTypeParameters(self): ast = self.Parse(""" T = TypeVar("T") T2 = TypeVar("T2") def f(x: T) -> T class A(Generic[T]): def a(self, x: T2) -> None: self = A[T or T2] """) f = ast.Lookup("f") sig, = f.signatures p_x, = sig.params self.assertEqual(sig.template, (pytd.TemplateItem(pytd.TypeParameter("T", scope="f")),)) self.assertEqual(p_x.type, pytd.TypeParameter("T", scope="f")) cls = ast.Lookup("A") f_cls, = cls.methods sig_cls, = f_cls.signatures p_self, p_x_cls = sig_cls.params self.assertEqual(cls.template, (pytd.TemplateItem(pytd.TypeParameter("T", scope="A")),)) self.assertEqual(sig_cls.template, (pytd.TemplateItem( pytd.TypeParameter("T2", scope="A.a")),)) self.assertEqual(p_self.type.parameters, (pytd.TypeParameter("T", scope="A"),)) self.assertEqual(p_x_cls.type, pytd.TypeParameter("T2", scope="A.a"))
def new_type(self, name: Union[str, pytd_node.Node], parameters: Optional[List[pytd.Type]] = None) -> pytd.Type: """Return the AST for a type. Args: name: The name of the type. parameters: List of type parameters. Returns: A pytd type node. Raises: ParseError: if the wrong number of parameters is supplied for the base_type - e.g., 2 parameters to Optional or no parameters to Union. """ base_type = self.resolve_type(name) for p in self.param_specs: if base_type.name.startswith(f"{p.name}."): _, attr = base_type.name.split(".", 1) if attr not in ("args", "kwargs"): raise ParseError( f"Unrecognized ParamSpec attribute: {attr}") # We do not yet support typing.ParamSpec, so replace references to its # args and kwargs attributes with Any. return pytd.AnythingType() if not isinstance(base_type, pytd.NamedType): # We assume that all type parameters have been defined. Since pytype # orders type parameters to appear before classes and functions, this # assumption is generally safe. AnyStr is special-cased because imported # type parameters aren't recognized. type_params = self.type_params + [ pytd.TypeParameter("typing.AnyStr") ] base_type = base_type.Visit(_InsertTypeParameters(type_params)) try: resolved_type = visitors.MaybeSubstituteParameters( base_type, parameters) except ValueError as e: raise ParseError(str(e)) from e if resolved_type: return resolved_type if parameters is not None: if (len(parameters) > 1 and isinstance(base_type, pytd.NamedType) and base_type.name == "typing.Optional"): raise ParseError("Too many options to %s" % base_type.name) return self._parameterized_type(base_type, parameters) else: if (isinstance(base_type, pytd.NamedType) and base_type.name in _TYPING_SETS): raise ParseError("Missing options to %s" % base_type.name) return base_type
def testDiffPickleAst(self): ast1 = pytd.TypeDeclUnit("foo", (), (), (), (), ()) ast2 = ast1.Replace(type_params=(pytd.TypeParameter("T", (), None, None),)) with file_utils.Tempdir() as d: data = [] for ast in (ast1, ast2): filename = os.path.join(d.path, "foo.pickled") serialize_ast.StoreAst(ast, filename) with open(filename, "rb") as fi: data.append(fi.read()) named_pickles1 = [("foo", data[0])] named_pickles2 = [("foo", data[1])] self.assertTrue(pytd_utils.DiffNamedPickles(named_pickles1, named_pickles2))
def value_instance_to_pytd_type(self, node, v, instance, seen, view): """Get the PyTD type an instance of this object would have. Args: node: The node. v: The object. instance: The instance. seen: Already seen instances. view: A Variable -> binding map. Returns: A PyTD type. """ if isinstance(v, abstract.Union): return pytd.UnionType(tuple( self.value_instance_to_pytd_type(node, t, instance, seen, view) for t in v.options)) elif isinstance(v, abstract.AnnotationContainer): return self.value_instance_to_pytd_type( node, v.base_cls, instance, seen, view) elif isinstance(v, abstract.Class): if v.official_name is None: return pytd.AnythingType() if seen is None: seen = set() if instance in seen: # We have a circular dependency in our types (e.g., lst[0] == lst). Stop # descending into the type parameters. type_params = () else: type_params = tuple(t.name for t in v.template) if instance is not None: seen.add(instance) type_arguments = self._value_to_parameter_types( node, v, instance, type_params, seen, view) base = pytd_utils.NamedTypeWithModule(v.official_name, v.module) if self._is_tuple(v, instance): if type_arguments: homogeneous = False else: homogeneous = True type_arguments = [pytd.NothingType()] else: homogeneous = len(type_arguments) == 1 return pytd_utils.MakeClassOrContainerType( base, type_arguments, homogeneous) elif isinstance(v, abstract.TypeVariable): return pytd.TypeParameter(v.name, None) else: log.info("Using ? for instance of %s", v.name) return pytd.AnythingType()
def testTypeParameters(self): """Test parsing of type parameters.""" src = textwrap.dedent(""" T = TypeVar("T") T2 = TypeVar("T2") def f(x: T) -> T class A(Generic[T]): def a(self, x: T2) -> None: self := A[T or T2] """) tree = parser.TypeDeclParser().Parse(src) param1 = tree.Lookup("T") param2 = tree.Lookup("T2") self.assertEquals(param1, pytd.TypeParameter("T", None)) self.assertEquals(param2, pytd.TypeParameter("T2", None)) self.assertEquals(tree.type_params, (param1, param2)) f = tree.Lookup("f") sig, = f.signatures p_x, = sig.params self.assertEquals(p_x.type, pytd.TypeParameter("T", None)) cls = tree.Lookup("A") cls_parent, = cls.parents f_cls, = cls.methods sig_cls, = f_cls.signatures # AdjustSelf has not been called yet, so self may not have the right type _, p_x_cls = sig_cls.params self.assertEquals(cls_parent.parameters, (pytd.TypeParameter("T", None), )) self.assertEquals(p_x_cls.type, pytd.TypeParameter("T2", None)) # The parser should not have attempted to insert templates! It does # not know about imported type parameters. self.assertEquals(sig.template, ()) self.assertEquals(cls.template, ()) self.assertEquals(sig_cls.template, ())
def add_type_var(self, name, param_list): """Add a type variable with the given name and parameter list.""" params = _validate_params(param_list) if (not params.required or not isinstance(params.required[0], pytd.Parameter)): raise ParseError("TypeVar's first arg should be a string") # Allow and ignore any other arguments (types, covariant=..., etc) name_param = params.required[0].name if name != name_param: raise ParseError("TypeVar name needs to be %r (not %r)" % (name_param, name)) if not self._current_condition.active: return self._type_params.append(pytd.TypeParameter(name, scope=None))
def testVerifyHeterogeneousTuple(self): # Error: does not inherit from Generic base = pytd.ClassType("tuple") base.cls = pytd.Class("tuple", None, (), (), (), (), None, ()) t1 = pytd.TupleType(base, (pytd.NamedType("str"), pytd.NamedType("float"))) self.assertRaises(visitors.ContainerError, lambda: t1.Visit(visitors.VerifyContainers())) # Error: Generic[str, float] gen = pytd.ClassType("typing.Generic") gen.cls = pytd.Class("typing.Generic", None, (), (), (), (), None, ()) t2 = pytd.TupleType(gen, (pytd.NamedType("str"), pytd.NamedType("float"))) self.assertRaises(visitors.ContainerError, lambda: t2.Visit(visitors.VerifyContainers())) # Okay param = pytd.TypeParameter("T") parent = pytd.GenericType(gen, (param,)) base.cls = pytd.Class( "tuple", None, (parent,), (), (), (), None, (pytd.TemplateItem(param),)) t3 = pytd.TupleType(base, (pytd.NamedType("str"), pytd.NamedType("float"))) t3.Visit(visitors.VerifyContainers())
def add_type_var(self, name, name_arg, args): """Add a type variable, <name> = TypeVar(<name_arg>, <args>).""" if name != name_arg: raise ParseError("TypeVar name needs to be %r (not %r)" % ( name_arg, name)) # 'bound' is the only keyword argument we currently use. # TODO(rechen): We should enforce the PEP 484 guideline that # len(constraints) != 1. However, this guideline is currently violated # in typeshed (see https://github.com/python/typeshed/pull/806). constraints, named_args = args named_args = dict(named_args) if named_args else {} extra = set(named_args) - {"bound", "covariant", "contravariant"} if extra: raise ParseError("Unrecognized keyword(s): %s" % ", ".join(extra)) if not self._current_condition.active: return self._type_params.append(pytd.TypeParameter( name=name, constraints=() if constraints is None else tuple(constraints), bound=named_args.get("bound")))
def _namedtuple_new(self, name, fields): """Build a __new__ method for a namedtuple with the given fields. For a namedtuple defined as NamedTuple("_", [("foo", int), ("bar", str)]), generates the method def __new__(cls: Type[_T], foo: int, bar: str) -> _T: ... where _T is a TypeVar bounded by the class type. Args: name: The class name. fields: A list of (name, type) pairs representing the namedtuple fields. Returns: A _NameAndSig object for a __new__ method. """ type_param = pytd.TypeParameter("_T" + name, bound=pytd.NamedType(name)) self._type_params.append(type_param) cls_arg = ( "cls", pytd.GenericType(pytd.NamedType("type"), (type_param,)), None) args = [cls_arg] + [(n, t, None) for n, t in fields] return self.new_function((), "__new__", args, type_param, ())
def _make_new(self, name: str, fields: List[Tuple[str, Any]]) -> function.NameAndSig: """Build a __new__ method for a namedtuple with the given fields. For a namedtuple defined as NamedTuple("_", [("foo", int), ("bar", str)]), generates the method def __new__(cls: Type[_T], foo: int, bar: str) -> _T: ... where _T is a TypeVar bounded by the class type. Args: name: The class name. fields: A list of (name, type) pairs representing the namedtuple fields. Returns: A function.NameAndSig object for a __new__ method. """ type_param = pytd.TypeParameter("_T" + name, bound=pytd.NamedType(name)) self.type_param = type_param cls_arg = ("cls", pytdgen.pytd_type(type_param)) args = [cls_arg] + fields return function.NameAndSig.make("__new__", args, type_param)
def p_funcdef(self, p): """funcdef : DEF NAME LPAREN params RPAREN return raises signature maybe_body""" _, _, name, _, params, _, return_type, raises, _, body = p # TODO(kramm): Output a warning if we already encountered a signature # with these types (but potentially different argument names) if name == '__init__' and isinstance(return_type, pytd.AnythingType): ret = pytd.NamedType('NoneType') else: ret = return_type signature = pytd.Signature(params=tuple(params.required), return_type=ret, exceptions=tuple(raises), template=(), has_optional=params.has_optional) typeparams = { name: pytd.TypeParameter(name) for name in self.context.typevars } used_typeparams = set() signature = signature.Visit( visitors.ReplaceTypes(typeparams, used_typeparams)) if used_typeparams: signature = signature.Replace(template=tuple( pytd.TemplateItem(typeparams[name]) for name in used_typeparams)) for mutator in body: try: signature = signature.Visit(mutator) except NotImplementedError as e: make_syntax_error(self, e.message, p) if not mutator.successful: make_syntax_error(self, 'No parameter named %s' % mutator.name, p) p[0] = NameAndSig(name=name, signature=signature, external_code=False)
def _typeparam_to_def(self, node, v, name): constraints = tuple(c.get_instance_type(node) for c in v.constraints) bound = v.bound and v.bound.get_instance_type(node) return pytd.TypeParameter(name, constraints=constraints, bound=bound)