def testMaybeFillInLocalPointers(self): src = textwrap.dedent(""" class A(object): def a(self, a: A, b: B) -> A or B: raise A() raise B() """) tree = self.Parse(src) ty_a = pytd.ClassType("A") ty_a.Visit(visitors.FillInLocalPointers({"": tree})) self.assertIsNotNone(ty_a.cls) ty_b = pytd.ClassType("B") ty_b.Visit(visitors.FillInLocalPointers({"": tree})) self.assertIsNone(ty_b.cls)
def _process_module(self, module_name, filename, ast): """Create a module from a loaded ast and save it to the loader cache. Args: module_name: The fully qualified name of the module being imported. filename: The file the ast was generated from. ast: The pytd.TypeDeclUnit representing the module. Returns: The ast (pytd.TypeDeclUnit) as represented in this loader. """ module = Module(module_name, filename, ast) self._modules[module_name] = module try: module.ast = self._postprocess_pyi(module.ast) # Now that any imported TypeVar instances have been resolved, adjust type # parameters in classes and functions. module.ast = module.ast.Visit(visitors.AdjustTypeParameters()) # Now we can fill in internal cls pointers to ClassType nodes in the # module. This code executes when the module is first loaded, which # happens before any others use it to resolve dependencies, so there are # no external pointers into the module at this point. module.ast.Visit( visitors.FillInLocalPointers({"": module.ast, module_name: module.ast})) except: # don't leave half-resolved modules around del self._modules[module_name] raise return module.ast
def _FillLocalReferences(serializable_ast, module_map): local_filler = visitors.FillInLocalPointers(module_map) if serializable_ast.class_type_nodes: for node in serializable_ast.class_type_nodes: local_filler.EnterClassType(node) if node.cls is None: raise AssertionError("This should not happen: %s" % str(node)) else: serializable_ast.ast.Visit(local_filler)
def GetBuiltinsAndTyping(python_version): """Get __builtin__.pytd and typing.pytd.""" assert python_version global _cached_builtins_pytd if _cached_builtins_pytd.cache: assert _cached_builtins_pytd.version == python_version else: t = parser.parse_string(_FindBuiltinFile("typing", python_version), name="typing", python_version=python_version) b = parser.parse_string(_FindBuiltinFile("__builtin__", python_version), name="__builtin__", python_version=python_version) b = b.Visit( visitors.LookupExternalTypes({"typing": t}, full_names=True, self_name="__builtin__")) t = t.Visit(visitors.LookupBuiltins(b)) b = b.Visit(visitors.NamedTypeToClassType()) t = t.Visit(visitors.NamedTypeToClassType()) b = b.Visit(visitors.AdjustTypeParameters()) t = t.Visit(visitors.AdjustTypeParameters()) b = b.Visit(visitors.CanonicalOrderingVisitor()) t = t.Visit(visitors.CanonicalOrderingVisitor()) b.Visit( visitors.FillInLocalPointers({ "": b, "typing": t, "__builtin__": b })) t.Visit( visitors.FillInLocalPointers({ "": t, "typing": t, "__builtin__": b })) b.Visit(visitors.VerifyLookup()) t.Visit(visitors.VerifyLookup()) b.Visit(visitors.VerifyContainers()) t.Visit(visitors.VerifyContainers()) _cached_builtins_pytd = Cache(python_version, (b, t)) return _cached_builtins_pytd.cache
def ParseWithBuiltins(self, src): ast = parser.parse_string(textwrap.dedent(src)) b, t = builtins.GetBuiltinsAndTyping() ast = ast.Visit( visitors.LookupExternalTypes({ "__builtin__": b, "typing": t }, full_names=True)) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.AdjustTypeParameters()) ast.Visit(visitors.FillInLocalPointers({"": ast, "__builtin__": b})) ast.Visit(visitors.VerifyVisitor()) return ast
def GetBuiltinsAndTyping(): """Get __builtin__.pytd and typing.pytd.""" global _cached_builtins_pytd if not _cached_builtins_pytd: t = parser.parse_string(_FindBuiltinFile("typing"), name="typing") b = parser.parse_string(_FindBuiltinFile("__builtin__"), name="__builtin__") b = b.Visit( visitors.LookupExternalTypes({"typing": t}, full_names=True, self_name="__builtin__")) t = t.Visit(visitors.LookupBuiltins(b)) b = b.Visit(visitors.NamedTypeToClassType()) t = t.Visit(visitors.NamedTypeToClassType()) b = b.Visit(visitors.AdjustTypeParameters()) t = t.Visit(visitors.AdjustTypeParameters()) b = b.Visit(visitors.CanonicalOrderingVisitor()) t = t.Visit(visitors.CanonicalOrderingVisitor()) b.Visit( visitors.FillInLocalPointers({ "": b, "typing": t, "__builtin__": b })) t.Visit( visitors.FillInLocalPointers({ "": t, "typing": t, "__builtin__": b })) b.Visit(visitors.VerifyLookup()) t.Visit(visitors.VerifyLookup()) b.Visit(visitors.VerifyContainers()) t.Visit(visitors.VerifyContainers()) _cached_builtins_pytd = b, t return _cached_builtins_pytd
def ParseWithBuiltins(self, src): ast = parser.parse_string(textwrap.dedent(src), python_version=self.PYTHON_VERSION) ast = ast.Visit( visitors.LookupExternalTypes( { "__builtin__": self.loader.builtins, "typing": self.loader.typing }, full_names=True)) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.AdjustTypeParameters()) ast.Visit( visitors.FillInLocalPointers({ "": ast, "__builtin__": self.loader.builtins })) ast.Visit(visitors.VerifyVisitor()) return ast
def PrepareForExport(module_name, python_version, ast, loader): """Prepare an ast as if it was parsed and loaded. External dependencies will not be resolved, as the ast generated by this method is supposed to be exported. Args: module_name: The module_name as a string for the returned ast. python_version: A tuple of (major, minor) python version as string (see config.python_version). ast: pytd.TypeDeclUnit, is only used if src is None. loader: A load_pytd.Loader instance. Returns: A pytd.TypeDeclUnit representing the supplied AST as it would look after being written to a file and parsed. """ # This is a workaround for functionality which crept into places it doesn't # belong. Ideally this would call some transformation Visitors on ast to # transform it into the same ast we get after parsing and loading (compare # load_pytd.Loader.load_file). Unfortunately parsing has some special cases, # e.g. '__init__' return type and '__new__' being a 'staticmethod', which # need to be moved to visitors before we can do this. Printing an ast also # applies transformations, # e.g. visitors.PrintVisitor._FormatContainerContents, which need to move to # their own visitors so they can be applied without printing. src = pytd_utils.Print(ast) ast = pytd_builtins.ParsePyTD(src=src, module=module_name, python_version=python_version) ast = ast.Visit(visitors.LookupBuiltins(loader.builtins, full_names=False)) ast = ast.Visit(visitors.ExpandCompatibleBuiltins(loader.builtins)) ast = ast.Visit(visitors.LookupLocalTypes()) ast = ast.Visit(visitors.AdjustTypeParameters()) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.FillInLocalPointers({"": ast, module_name: ast})) ast = ast.Visit(visitors.CanonicalOrderingVisitor()) ast = ast.Visit( visitors.ClassTypeToLateType( ignore=[module_name + ".", "__builtin__.", "typing."])) return ast
def value_to_pytd_type(self, node, v, seen, view): """Get a PyTD type representing this object, as seen at a node. Args: node: The node from which we want to observe this object. v: The object. seen: The set of values seen before while computing the type. view: A Variable -> binding map. Returns: A PyTD type. """ if isinstance(v, (abstract.Empty, abstract.Nothing)): return pytd.NothingType() elif isinstance(v, abstract.TypeParameterInstance): if v.instance.type_parameters[v.name].bindings: # The type parameter was initialized. return pytd_utils.JoinTypes( self.value_to_pytd_type(node, p, seen, view) for p in v.instance.type_parameters[v.name].data) elif v.param.constraints: return pytd_utils.JoinTypes( self.value_instance_to_pytd_type(node, p, None, seen, view) for p in v.param.constraints) else: return pytd.AnythingType() elif isinstance(v, typing.TypeVar): return pytd.NamedType("__builtin__.type") elif isinstance( v, (abstract.InterpreterFunction, abstract.BoundInterpreterFunction)): sig, = abstract.get_signatures(v) return self.value_instance_to_pytd_type( node, self.signature_to_callable(sig, v.vm), None, seen, view) elif isinstance(v, (abstract.PyTDFunction, abstract.BoundPyTDFunction)): signatures = abstract.get_signatures(v) if len(signatures) == 1: val = self.signature_to_callable(signatures[0], v.vm) if not v.vm.annotations_util.get_type_parameters(val): # This is a workaround to make sure we don't put unexpected type # parameters in call traces. return self.value_instance_to_pytd_type( node, val, None, seen, view) return pytd.NamedType("typing.Callable") elif isinstance( v, (special_builtins.IsInstance, abstract.ClassMethod, abstract.StaticMethod, special_builtins.ClassMethodCallable)): return pytd.NamedType("typing.Callable") elif isinstance(v, abstract.Class): param = self.value_instance_to_pytd_type(node, v, None, seen, view) return pytd.GenericType( base_type=pytd.NamedType("__builtin__.type"), parameters=(param, )) elif isinstance(v, abstract.Module): return pytd.NamedType("__builtin__.module") elif isinstance(v, abstract.SimpleAbstractValue): if v.cls: classvalues = self._get_values(node, v.cls, view) cls_types = [] for cls in classvalues: cls_types.append( self.value_instance_to_pytd_type(node, cls, v, seen=seen, view=view)) ret = pytd_utils.JoinTypes(cls_types) ret.Visit( visitors.FillInLocalPointers( {"__builtin__": v.vm.loader.builtins})) return ret else: # We don't know this type's __class__, so return AnythingType to # indicate that we don't know anything about what this is. # This happens e.g. for locals / globals, which are returned from the # code in class declarations. log.info("Using ? for %s", v.name) return pytd.AnythingType() elif isinstance(v, abstract.Union): return pytd.UnionType( tuple( self.value_to_pytd_type(node, o, seen, view) for o in v.options)) elif isinstance(v, special_builtins.SuperInstance): return pytd.NamedType("__builtin__.super") elif isinstance(v, (abstract.Unsolvable, abstract.TypeParameter)): # Arguably, the type of a type parameter is NamedType("typing.TypeVar"), # but pytype doesn't know how to handle that, so let's just go with Any. return pytd.AnythingType() elif isinstance(v, abstract.Unknown): return pytd.NamedType(v.class_name) else: raise NotImplementedError(v.__class__.__name__)
def _finish_ast(self, ast): module_map = self._get_module_map() module_map[""] = ast # The module itself (local lookup) ast.Visit(visitors.FillInLocalPointers(module_map))
def testFillInFunctionTypePointers(self): src = textwrap.dedent("def f(): ...") tree = self.Parse(src) ty = pytd.FunctionType("f", None) ty.Visit(visitors.FillInLocalPointers({"": tree})) self.assertEqual(ty.function, tree.Lookup("f"))