def _process_module(self, module_name, filename, ast): """Create a module from a loaded ast and save it to the loader cache. Args: module_name: The fully qualified name of the module being imported. filename: The file the ast was generated from. ast: The pytd.TypeDeclUnit representing the module. Returns: The ast (pytd.TypeDeclUnit) as represented in this loader. """ module = Module(module_name, filename, ast) self._modules[module_name] = module try: module.ast = self._postprocess_pyi(module.ast) # Now that any imported TypeVar instances have been resolved, adjust type # parameters in classes and functions. module.ast = module.ast.Visit(visitors.AdjustTypeParameters()) # Now we can fill in internal cls pointers to ClassType nodes in the # module. This code executes when the module is first loaded, which # happens before any others use it to resolve dependencies, so there are # no external pointers into the module at this point. module.ast.Visit( visitors.FillInLocalPointers({"": module.ast, module_name: module.ast})) except: # don't leave half-resolved modules around del self._modules[module_name] raise return module.ast
def LinkAgainstSimpleBuiltins(self, ast): ast = ast.Visit(visitors.NamedTypeToClassType()) ast = visitors.AdjustTypeParameters(ast) ast.Visit(visitors.FillInModuleClasses({"": ast})) ast = ast.Visit(visitors.LookupFullNames([self.mini_builtins])) ast.Visit(visitors.VerifyLookup()) return ast
def parse_and_solve(self, src): ast = self.parse(src) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.AdjustTypeParameters()) types, _ = convert_structural.solve(ast, builtins_pytd=self.builtins_pytd) # Drop "__builtin__" prefix, for more readable tests. return {k: {v.rpartition("__builtin__.")[2] for v in l} for k, l in types.items()}
def Parse(self, src, version=None, platform=None): # TODO(kramm): Using self.parser here breaks tests. Why? tree = parser.TypeDeclParser(version=version, platform=platform).Parse( textwrap.dedent(src)) tree = tree.Visit(visitors.NamedTypeToClassType()) tree = visitors.AdjustTypeParameters(tree) # Convert back to named types for easier testing tree = tree.Visit(visitors.ClassTypeToNamedType()) tree.Visit(visitors.VerifyVisitor()) return tree
def ParseWithBuiltins(self, src): ast = parser.TypeDeclParser().Parse(textwrap.dedent(src)) b, t = builtins.GetBuiltinsAndTyping() ast = ast.Visit(visitors.LookupExternalTypes( {"__builtin__": b, "typing": t}, full_names=True)) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = visitors.AdjustTypeParameters(ast) ast.Visit(visitors.FillInModuleClasses({"": ast})) ast.Visit(visitors.VerifyVisitor()) return ast
def GetBuiltinsAndTyping(python_version): """Get __builtin__.pytd and typing.pytd.""" assert python_version global _cached_builtins_pytd if _cached_builtins_pytd.cache: assert _cached_builtins_pytd.version == python_version else: t = parser.parse_string(_FindBuiltinFile("typing", python_version), name="typing", python_version=python_version) b = parser.parse_string(_FindBuiltinFile("__builtin__", python_version), name="__builtin__", python_version=python_version) b = b.Visit( visitors.LookupExternalTypes({"typing": t}, full_names=True, self_name="__builtin__")) t = t.Visit(visitors.LookupBuiltins(b)) b = b.Visit(visitors.NamedTypeToClassType()) t = t.Visit(visitors.NamedTypeToClassType()) b = b.Visit(visitors.AdjustTypeParameters()) t = t.Visit(visitors.AdjustTypeParameters()) b = b.Visit(visitors.CanonicalOrderingVisitor()) t = t.Visit(visitors.CanonicalOrderingVisitor()) b.Visit( visitors.FillInLocalPointers({ "": b, "typing": t, "__builtin__": b })) t.Visit( visitors.FillInLocalPointers({ "": t, "typing": t, "__builtin__": b })) b.Visit(visitors.VerifyLookup()) t.Visit(visitors.VerifyLookup()) b.Visit(visitors.VerifyContainers()) t.Visit(visitors.VerifyContainers()) _cached_builtins_pytd = Cache(python_version, (b, t)) return _cached_builtins_pytd.cache
def Parse(self, src, name=None, version=None, platform=None): version = version or self.PYTHON_VERSION # TODO(kramm): Using self.parser here breaks tests. Why? tree = parser.parse_string(textwrap.dedent(src), name=name, python_version=version, platform=platform) tree = tree.Visit(visitors.NamedTypeToClassType()) tree = tree.Visit(visitors.AdjustTypeParameters()) # Convert back to named types for easier testing tree = tree.Visit(visitors.ClassTypeToNamedType()) tree.Visit(visitors.VerifyVisitor()) return tree
def ParseWithBuiltins(self, src): ast = parser.parse_string(textwrap.dedent(src)) b, t = builtins.GetBuiltinsAndTyping() ast = ast.Visit( visitors.LookupExternalTypes({ "__builtin__": b, "typing": t }, full_names=True)) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.AdjustTypeParameters()) ast.Visit(visitors.FillInLocalPointers({"": ast, "__builtin__": b})) ast.Visit(visitors.VerifyVisitor()) return ast
def GetBuiltinsAndTyping(): """Get __builtin__.pytd and typing.pytd.""" global _cached_builtins_pytd if not _cached_builtins_pytd: t = parser.TypeDeclParser().Parse(_FindBuiltinFile("typing"), name="typing") t = t.Visit(visitors.AddNamePrefix()) b = parser.TypeDeclParser().Parse(_FindBuiltinFile("__builtin__"), name="__builtin__") b = b.Visit(visitors.AddNamePrefix()) b = b.Visit(visitors.NamedTypeToClassType()) b = b.Visit( visitors.LookupExternalTypes({"typing": t}, full_names=True, self_name="__builtin__")) t = t.Visit(visitors.LookupBuiltins(b)) t = t.Visit(visitors.NamedTypeToClassType()) b = visitors.AdjustTypeParameters(b) t = visitors.AdjustTypeParameters(t) b.Visit( visitors.FillInModuleClasses({ "": b, "typing": t, "__builtin__": b })) t.Visit( visitors.FillInModuleClasses({ "": t, "typing": t, "__builtin__": b })) b.Visit(visitors.VerifyLookup()) t.Visit(visitors.VerifyLookup()) b.Visit(visitors.VerifyContainers()) t.Visit(visitors.VerifyContainers()) _cached_builtins_pytd = b, t return _cached_builtins_pytd
def compute_types(self, defs): ty = pytd_utils.Concat( self.pytd_for_types(defs), pytd.TypeDeclUnit( "unknowns", constants=tuple(), type_params=tuple(self.pytd_typevars()), classes=tuple(self.pytd_classes_for_unknowns()) + tuple(self.pytd_classes_for_call_traces()), functions=tuple(self.pytd_functions_for_call_traces()), aliases=tuple(self.pytd_aliases()))) ty = ty.Visit(optimize.PullInMethodClasses()) ty = ty.Visit(visitors.DefaceUnresolved( [ty, self.loader.concat_all()], "~unknown")) return ty.Visit(visitors.AdjustTypeParameters())
def GetBuiltinsAndTyping(): """Get __builtin__.pytd and typing.pytd.""" global _cached_builtins_pytd if not _cached_builtins_pytd: t = parser.parse_string(_FindBuiltinFile("typing"), name="typing") b = parser.parse_string(_FindBuiltinFile("__builtin__"), name="__builtin__") b = b.Visit( visitors.LookupExternalTypes({"typing": t}, full_names=True, self_name="__builtin__")) t = t.Visit(visitors.LookupBuiltins(b)) b = b.Visit(visitors.NamedTypeToClassType()) t = t.Visit(visitors.NamedTypeToClassType()) b = b.Visit(visitors.AdjustTypeParameters()) t = t.Visit(visitors.AdjustTypeParameters()) b = b.Visit(visitors.CanonicalOrderingVisitor()) t = t.Visit(visitors.CanonicalOrderingVisitor()) b.Visit( visitors.FillInLocalPointers({ "": b, "typing": t, "__builtin__": b })) t.Visit( visitors.FillInLocalPointers({ "": t, "typing": t, "__builtin__": b })) b.Visit(visitors.VerifyLookup()) t.Visit(visitors.VerifyLookup()) b.Visit(visitors.VerifyContainers()) t.Visit(visitors.VerifyContainers()) _cached_builtins_pytd = b, t return _cached_builtins_pytd
def ParseWithBuiltins(self, src): ast = parser.parse_string(textwrap.dedent(src), python_version=self.PYTHON_VERSION) ast = ast.Visit( visitors.LookupExternalTypes( { "__builtin__": self.loader.builtins, "typing": self.loader.typing }, full_names=True)) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.AdjustTypeParameters()) ast.Visit( visitors.FillInLocalPointers({ "": ast, "__builtin__": self.loader.builtins })) ast.Visit(visitors.VerifyVisitor()) return ast
def _load_file(self, module_name, filename, ast=None): """Load (or retrieve from cache) a module and resolve its dependencies.""" self._concatenated = None # invalidate existing = self._modules.get(module_name) if existing: if existing.filename != filename: raise AssertionError( "%s exists as both %s and %s" % (module_name, filename, existing.filename)) return existing.ast if not ast: ast = builtins.ParsePyTD( filename=filename, module=module_name, python_version=self.options.python_version) ast = self._postprocess_pyi(ast) module = Module(module_name, filename, ast) self._modules[module_name] = module try: module.ast = self._load_and_resolve_ast_dependencies( module.ast, module_name) # Now that any imported TypeVar instances have been resolved, adjust type # parameters in classes and functions. module.ast = visitors.AdjustTypeParameters(module.ast) # Now we can fill in internal cls pointers to ClassType nodes in the # module. This code executes when the module is first loaded, which # happens before any others use it to resolve dependencies, so there are # no external pointers into the module at this point. module.ast.Visit( visitors.FillInModuleClasses({ "": module.ast, module_name: module.ast })) # TODO(rechen): Once generics are supported in inline type annotations, a # VerifyContainers check should also be done on the final ast. module.ast.Visit(visitors.VerifyContainers()) except: del self._modules[ module_name] # don't leave half-resolved modules around raise return module.ast
def PrepareForExport(module_name, python_version, ast, loader): """Prepare an ast as if it was parsed and loaded. External dependencies will not be resolved, as the ast generated by this method is supposed to be exported. Args: module_name: The module_name as a string for the returned ast. python_version: A tuple of (major, minor) python version as string (see config.python_version). ast: pytd.TypeDeclUnit, is only used if src is None. loader: A load_pytd.Loader instance. Returns: A pytd.TypeDeclUnit representing the supplied AST as it would look after being written to a file and parsed. """ # This is a workaround for functionality which crept into places it doesn't # belong. Ideally this would call some transformation Visitors on ast to # transform it into the same ast we get after parsing and loading (compare # load_pytd.Loader.load_file). Unfortunately parsing has some special cases, # e.g. '__init__' return type and '__new__' being a 'staticmethod', which # need to be moved to visitors before we can do this. Printing an ast also # applies transformations, # e.g. visitors.PrintVisitor._FormatContainerContents, which need to move to # their own visitors so they can be applied without printing. src = pytd_utils.Print(ast) ast = pytd_builtins.ParsePyTD(src=src, module=module_name, python_version=python_version) ast = ast.Visit(visitors.LookupBuiltins(loader.builtins, full_names=False)) ast = ast.Visit(visitors.ExpandCompatibleBuiltins(loader.builtins)) ast = ast.Visit(visitors.LookupLocalTypes()) ast = ast.Visit(visitors.AdjustTypeParameters()) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.FillInLocalPointers({"": ast, module_name: ast})) ast = ast.Visit(visitors.CanonicalOrderingVisitor()) ast = ast.Visit( visitors.ClassTypeToLateType( ignore=[module_name + ".", "__builtin__.", "typing."])) return ast
def LinkAgainstSimpleBuiltins(self, ast): ast = ast.Visit(visitors.AdjustTypeParameters()) ast = visitors.LookupClasses(ast, self.mini_builtins) return ast