def load(self, options): """Read builtins.pytd and typing.pytd, and return the parsed modules.""" t = self._parse_predefined("typing", options) b = self._parse_predefined("builtins", options) b = b.Visit( visitors.LookupExternalTypes({"typing": t}, self_name="builtins")) t = t.Visit(visitors.LookupBuiltins(b)) b = b.Visit(visitors.NamedTypeToClassType()) t = t.Visit(visitors.NamedTypeToClassType()) b = b.Visit(visitors.AdjustTypeParameters()) t = t.Visit(visitors.AdjustTypeParameters()) b = b.Visit(visitors.CanonicalOrderingVisitor()) t = t.Visit(visitors.CanonicalOrderingVisitor()) b.Visit( visitors.FillInLocalPointers({ "": b, "typing": t, "builtins": b })) t.Visit( visitors.FillInLocalPointers({ "": t, "typing": t, "builtins": b })) b.Visit(visitors.VerifyLookup()) t.Visit(visitors.VerifyLookup()) b.Visit(visitors.VerifyContainers()) t.Visit(visitors.VerifyContainers()) return b, t
def _process_module(self, module_name, filename, ast): """Create a module from a loaded ast and save it to the loader cache. Args: module_name: The fully qualified name of the module being imported. filename: The file the ast was generated from. ast: The pytd.TypeDeclUnit representing the module. Returns: The ast (pytd.TypeDeclUnit) as represented in this loader. """ module = Module(module_name, filename, ast) self._modules[module_name] = module try: module.ast = self._postprocess_pyi(module.ast) # Now that any imported TypeVar instances have been resolved, adjust type # parameters in classes and functions. module.ast = module.ast.Visit(visitors.AdjustTypeParameters()) # Now we can fill in internal cls pointers to ClassType nodes in the # module. This code executes when the module is first loaded, which # happens before any others use it to resolve dependencies, so there are # no external pointers into the module at this point. module.ast.Visit( visitors.FillInLocalPointers({"": module.ast, module_name: module.ast})) except: # don't leave half-resolved modules around del self._modules[module_name] raise self.add_module_prefixes(module_name) return module.ast
def parse_and_solve(self, src): ast = self.parse(src) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.AdjustTypeParameters()) types, _ = convert_structural.solve(ast, builtins_pytd=self.builtins_pytd) # Drop "__builtin__" prefix, for more readable tests. return {k: {v.rpartition("__builtin__.")[2] for v in l} for k, l in types.items()}
def ParseWithBuiltins(self, src): ast = parser.parse_string(textwrap.dedent(src), options=self.options) ast = ast.Visit(visitors.LookupExternalTypes( {"builtins": self.loader.builtins, "typing": self.loader.typing})) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.AdjustTypeParameters()) ast.Visit(visitors.FillInLocalPointers({ "": ast, "builtins": self.loader.builtins})) ast.Visit(visitors.VerifyVisitor()) return ast
def GetBuiltinsAndTyping(python_version): # Deprecated. Use load_pytd instead. """Get __builtin__.pytd and typing.pytd.""" assert python_version global _cached_builtins_pytd if _cached_builtins_pytd.cache: assert _cached_builtins_pytd.version == python_version else: t = parser.parse_string(_FindBuiltinFile("typing", python_version), name="typing", python_version=python_version) b = parser.parse_string(_FindBuiltinFile("__builtin__", python_version), name="__builtin__", python_version=python_version) b = b.Visit( visitors.LookupExternalTypes({"typing": t}, full_names=True, self_name="__builtin__")) t = t.Visit(visitors.LookupBuiltins(b)) b = b.Visit(visitors.NamedTypeToClassType()) t = t.Visit(visitors.NamedTypeToClassType()) b = b.Visit(visitors.AdjustTypeParameters()) t = t.Visit(visitors.AdjustTypeParameters()) b = b.Visit(visitors.CanonicalOrderingVisitor()) t = t.Visit(visitors.CanonicalOrderingVisitor()) b.Visit( visitors.FillInLocalPointers({ "": b, "typing": t, "__builtin__": b })) t.Visit( visitors.FillInLocalPointers({ "": t, "typing": t, "__builtin__": b })) b.Visit(visitors.VerifyLookup()) t.Visit(visitors.VerifyLookup()) b.Visit(visitors.VerifyContainers()) t.Visit(visitors.VerifyContainers()) _cached_builtins_pytd = Cache(python_version, (b, t)) return _cached_builtins_pytd.cache
def ParseWithBuiltins(self, src): ast = parser.parse_string(textwrap.dedent(src), python_version=self.PYTHON_VERSION) ast = ast.Visit(visitors.LookupExternalTypes( {"__builtin__": self.loader.builtins, "typing": self.loader.typing})) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.AdjustTypeParameters()) ast.Visit(visitors.FillInLocalPointers({ "": ast, "__builtin__": self.loader.builtins})) ast.Visit(visitors.VerifyVisitor()) return ast
def Parse(self, src, name=None, version=None, platform=None): version = version or self.PYTHON_VERSION tree = parser.parse_string( textwrap.dedent(src), name=name, python_version=version, platform=platform) tree = tree.Visit(visitors.NamedTypeToClassType()) tree = tree.Visit(visitors.AdjustTypeParameters()) # Convert back to named types for easier testing tree = tree.Visit(visitors.ClassTypeToNamedType()) tree.Visit(visitors.VerifyVisitor()) return tree
def Parse(self, src, name=None, version=None, platform=None): if version: self.options.python_version = version if platform: self.options.platform = platform tree = parser.parse_string( textwrap.dedent(src), name=name, options=self.options) tree = tree.Visit(visitors.NamedTypeToClassType()) tree = tree.Visit(visitors.AdjustTypeParameters()) # Convert back to named types for easier testing tree = tree.Visit(visitors.ClassTypeToNamedType()) tree.Visit(visitors.VerifyVisitor()) return tree
def testGetBasesInMRO(self): ast = parser.parse_string(textwrap.dedent(""" from typing import Generic, TypeVar T = TypeVar("T") class Foo(Generic[T]): pass class Bar(Foo[int]): pass """), python_version=self.PYTHON_VERSION) ast = ast.Visit(visitors.AdjustTypeParameters()) loader = load_pytd.Loader(None, self.PYTHON_VERSION) ast = loader.resolve_ast(ast) bases = mro.GetBasesInMRO(ast.Lookup("Bar"), lookup_ast=ast) self.assertListEqual(["Foo", "typing.Generic", "__builtin__.object"], [t.name for t in bases])
def GetBuiltinsAndTyping(python_version): # Deprecated. Use load_pytd instead. """Get builtins.pytd and typing.pytd.""" assert python_version if python_version not in _cached_builtins_pytd: t = parser.parse_string(_FindBuiltinFile("typing", python_version), name="typing", python_version=python_version) b = parser.parse_string(_FindBuiltinFile("builtins", python_version), name="builtins", python_version=python_version) b = b.Visit( visitors.LookupExternalTypes({"typing": t}, self_name="builtins")) t = t.Visit(visitors.LookupBuiltins(b)) b = b.Visit(visitors.NamedTypeToClassType()) t = t.Visit(visitors.NamedTypeToClassType()) b = b.Visit(visitors.AdjustTypeParameters()) t = t.Visit(visitors.AdjustTypeParameters()) b = b.Visit(visitors.CanonicalOrderingVisitor()) t = t.Visit(visitors.CanonicalOrderingVisitor()) b.Visit( visitors.FillInLocalPointers({ "": b, "typing": t, "builtins": b })) t.Visit( visitors.FillInLocalPointers({ "": t, "typing": t, "builtins": b })) b.Visit(visitors.VerifyLookup()) t.Visit(visitors.VerifyLookup()) b.Visit(visitors.VerifyContainers()) t.Visit(visitors.VerifyContainers()) _cached_builtins_pytd[python_version] = (b, t) return _cached_builtins_pytd[python_version]
def test_get_bases_in_mro(self): ast = parser.parse_string(textwrap.dedent(""" from typing import Generic, TypeVar T = TypeVar("T") class Foo(Generic[T]): pass class Bar(Foo[int]): pass """), options=self.options) ast = ast.Visit(visitors.AdjustTypeParameters()) loader = load_pytd.Loader( config.Options.create(python_version=self.python_version)) ast = loader.resolve_ast(ast) bases = mro.GetBasesInMRO(ast.Lookup("Bar"), lookup_ast=ast) self.assertListEqual(["Foo", "typing.Generic", "builtins.object"], [t.name for t in bases])
def compute_types(self, defs): classes = (tuple(self.pytd_classes_for_unknowns()) + tuple(self.pytd_classes_for_call_traces()) + self.pytd_classes_for_namedtuple_instances()) functions = tuple(self.pytd_functions_for_call_traces()) aliases = tuple(self.pytd_aliases()) ty = pytd_utils.Concat( self.pytd_for_types(defs), pytd_utils.CreateModule("unknowns", classes=classes, functions=functions, aliases=aliases)) ty = ty.Visit(optimize.CombineReturnsAndExceptions()) ty = ty.Visit(optimize.PullInMethodClasses()) ty = ty.Visit(visitors.DefaceUnresolved( [ty, self.loader.concat_all()], "~unknown")) return ty.Visit(visitors.AdjustTypeParameters())
def _process_module(self, module_name, filename, mod_ast): """Create a module from a loaded ast and save it to the loader cache. Args: module_name: The fully qualified name of the module being imported. May be None. filename: The file the ast was generated from. May be None. mod_ast: The pytd.TypeDeclUnit representing the module. Returns: The ast (pytd.TypeDeclUnit) as represented in this loader. """ module = Module(module_name, filename, mod_ast) # Builtins need to be resolved before the module is cached so that they are # not mistaken for local types. External types can be left unresolved # because they are unambiguous. self._resolver.allow_singletons = False module.ast = self._resolver.resolve_builtin_types(module.ast) self._modules[module_name] = module try: self._resolver.allow_singletons = True module.ast = self._resolve_external_and_local_types(module.ast) # We need to resolve builtin singletons after we have made sure they are # not shadowed by a local or a star import. module.ast = self._resolver.resolve_builtin_types(module.ast) self._resolver.allow_singletons = False # Now that any imported TypeVar instances have been resolved, adjust type # parameters in classes and functions. module.ast = module.ast.Visit(visitors.AdjustTypeParameters()) # Now we can fill in internal cls pointers to ClassType nodes in the # module. This code executes when the module is first loaded, which # happens before any others use it to resolve dependencies, so there are # no external pointers into the module at this point. module_map = {"": module.ast, module_name: module.ast} module.ast.Visit(visitors.FillInLocalPointers(module_map)) except: # don't leave half-resolved modules around del self._modules[module_name] raise if module_name: self.add_module_prefixes(module_name) return module.ast
def PrepareForExport(module_name, python_version, ast, loader): """Prepare an ast as if it was parsed and loaded. External dependencies will not be resolved, as the ast generated by this method is supposed to be exported. Args: module_name: The module_name as a string for the returned ast. python_version: A tuple of (major, minor) python version as string (see config.python_version). ast: pytd.TypeDeclUnit, is only used if src is None. loader: A load_pytd.Loader instance. Returns: A pytd.TypeDeclUnit representing the supplied AST as it would look after being written to a file and parsed. """ # This is a workaround for functionality which crept into places it doesn't # belong. Ideally this would call some transformation Visitors on ast to # transform it into the same ast we get after parsing and loading (compare # load_pytd.Loader.load_file). Unfortunately parsing has some special cases, # e.g. '__init__' return type and '__new__' being a 'staticmethod', which # need to be moved to visitors before we can do this. Printing an ast also # applies transformations, # e.g. visitors.PrintVisitor._FormatContainerContents, which need to move to # their own visitors so they can be applied without printing. src = pytd_utils.Print(ast) ast = parser.parse_string(src=src, name=module_name, python_version=python_version) ast = ast.Visit(visitors.LookupBuiltins(loader.builtins, full_names=False)) ast = ast.Visit( visitors.ExpandCompatibleBuiltins(loader.builtins, python_version)) ast = ast.Visit(visitors.LookupLocalTypes()) ast = ast.Visit(visitors.AdjustTypeParameters()) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.FillInLocalPointers({"": ast, module_name: ast})) ast = ast.Visit(visitors.CanonicalOrderingVisitor()) ast = ast.Visit( visitors.ClassTypeToLateType( ignore=[module_name + ".", "__builtin__.", "typing."])) return ast
def LinkAgainstSimpleBuiltins(self, ast): ast = ast.Visit(visitors.AdjustTypeParameters()) ast = visitors.LookupClasses(ast, self.mini_builtins) return ast