def load(self, options): """Read builtins.pytd and typing.pytd, and return the parsed modules.""" t = self._parse_predefined("typing", options) b = self._parse_predefined("builtins", options) b = b.Visit( visitors.LookupExternalTypes({"typing": t}, self_name="builtins")) t = t.Visit(visitors.LookupBuiltins(b)) b = b.Visit(visitors.NamedTypeToClassType()) t = t.Visit(visitors.NamedTypeToClassType()) b = b.Visit(visitors.AdjustTypeParameters()) t = t.Visit(visitors.AdjustTypeParameters()) b = b.Visit(visitors.CanonicalOrderingVisitor()) t = t.Visit(visitors.CanonicalOrderingVisitor()) b.Visit( visitors.FillInLocalPointers({ "": b, "typing": t, "builtins": b })) t.Visit( visitors.FillInLocalPointers({ "": t, "typing": t, "builtins": b })) b.Visit(visitors.VerifyLookup()) t.Visit(visitors.VerifyLookup()) b.Visit(visitors.VerifyContainers()) t.Visit(visitors.VerifyContainers()) return b, t
def testMaybeFillInLocalPointers(self): src = textwrap.dedent(""" class A(object): def a(self, a: A, b: B) -> A or B: raise A() raise B() """) tree = self.Parse(src) ty_a = pytd.ClassType("A") ty_a.Visit(visitors.FillInLocalPointers({"": tree})) self.assertIsNotNone(ty_a.cls) ty_b = pytd.ClassType("B") ty_b.Visit(visitors.FillInLocalPointers({"": tree})) self.assertIsNone(ty_b.cls)
def test_maybe_fill_in_local_pointers(self): src = textwrap.dedent(""" from typing import Union class A: def a(self, a: A, b: B) -> Union[A, B]: raise A() raise B() """) tree = self.Parse(src) ty_a = pytd.ClassType("A") ty_a.Visit(visitors.FillInLocalPointers({"": tree})) self.assertIsNotNone(ty_a.cls) ty_b = pytd.ClassType("B") ty_b.Visit(visitors.FillInLocalPointers({"": tree})) self.assertIsNone(ty_b.cls)
def _process_module(self, module_name, filename, ast): """Create a module from a loaded ast and save it to the loader cache. Args: module_name: The fully qualified name of the module being imported. filename: The file the ast was generated from. ast: The pytd.TypeDeclUnit representing the module. Returns: The ast (pytd.TypeDeclUnit) as represented in this loader. """ module = Module(module_name, filename, ast) self._modules[module_name] = module try: module.ast = self._postprocess_pyi(module.ast) # Now that any imported TypeVar instances have been resolved, adjust type # parameters in classes and functions. module.ast = module.ast.Visit(visitors.AdjustTypeParameters()) # Now we can fill in internal cls pointers to ClassType nodes in the # module. This code executes when the module is first loaded, which # happens before any others use it to resolve dependencies, so there are # no external pointers into the module at this point. module.ast.Visit( visitors.FillInLocalPointers({"": module.ast, module_name: module.ast})) except: # don't leave half-resolved modules around del self._modules[module_name] raise self.add_module_prefixes(module_name) return module.ast
def GetBuiltinsAndTyping(python_version): # Deprecated. Use load_pytd instead. """Get __builtin__.pytd and typing.pytd.""" assert python_version global _cached_builtins_pytd if _cached_builtins_pytd.cache: assert _cached_builtins_pytd.version == python_version else: t = parser.parse_string(_FindBuiltinFile("typing", python_version), name="typing", python_version=python_version) b = parser.parse_string(_FindBuiltinFile("__builtin__", python_version), name="__builtin__", python_version=python_version) b = b.Visit( visitors.LookupExternalTypes({"typing": t}, full_names=True, self_name="__builtin__")) t = t.Visit(visitors.LookupBuiltins(b)) b = b.Visit(visitors.NamedTypeToClassType()) t = t.Visit(visitors.NamedTypeToClassType()) b = b.Visit(visitors.AdjustTypeParameters()) t = t.Visit(visitors.AdjustTypeParameters()) b = b.Visit(visitors.CanonicalOrderingVisitor()) t = t.Visit(visitors.CanonicalOrderingVisitor()) b.Visit( visitors.FillInLocalPointers({ "": b, "typing": t, "__builtin__": b })) t.Visit( visitors.FillInLocalPointers({ "": t, "typing": t, "__builtin__": b })) b.Visit(visitors.VerifyLookup()) t.Visit(visitors.VerifyLookup()) b.Visit(visitors.VerifyContainers()) t.Visit(visitors.VerifyContainers()) _cached_builtins_pytd = Cache(python_version, (b, t)) return _cached_builtins_pytd.cache
def ParseWithBuiltins(self, src): ast = parser.parse_string(textwrap.dedent(src), options=self.options) ast = ast.Visit(visitors.LookupExternalTypes( {"builtins": self.loader.builtins, "typing": self.loader.typing})) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.AdjustTypeParameters()) ast.Visit(visitors.FillInLocalPointers({ "": ast, "builtins": self.loader.builtins})) ast.Visit(visitors.VerifyVisitor()) return ast
def ParseWithBuiltins(self, src): ast = parser.parse_string(textwrap.dedent(src), python_version=self.PYTHON_VERSION) ast = ast.Visit(visitors.LookupExternalTypes( {"__builtin__": self.loader.builtins, "typing": self.loader.typing})) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.AdjustTypeParameters()) ast.Visit(visitors.FillInLocalPointers({ "": ast, "__builtin__": self.loader.builtins})) ast.Visit(visitors.VerifyVisitor()) return ast
def FillLocalReferences(serializable_ast, module_map): """Fill in local references.""" local_filler = visitors.FillInLocalPointers(module_map) if serializable_ast.class_type_nodes is None: serializable_ast.ast.Visit(local_filler) return serializable_ast.Replace(class_type_nodes=None) else: for node in serializable_ast.class_type_nodes: local_filler.EnterClassType(node) if node.cls is None: raise AssertionError("This should not happen: %s" % str(node)) return serializable_ast
def GetBuiltinsAndTyping(python_version): # Deprecated. Use load_pytd instead. """Get builtins.pytd and typing.pytd.""" assert python_version if python_version not in _cached_builtins_pytd: t = parser.parse_string(_FindBuiltinFile("typing", python_version), name="typing", python_version=python_version) b = parser.parse_string(_FindBuiltinFile("builtins", python_version), name="builtins", python_version=python_version) b = b.Visit( visitors.LookupExternalTypes({"typing": t}, self_name="builtins")) t = t.Visit(visitors.LookupBuiltins(b)) b = b.Visit(visitors.NamedTypeToClassType()) t = t.Visit(visitors.NamedTypeToClassType()) b = b.Visit(visitors.AdjustTypeParameters()) t = t.Visit(visitors.AdjustTypeParameters()) b = b.Visit(visitors.CanonicalOrderingVisitor()) t = t.Visit(visitors.CanonicalOrderingVisitor()) b.Visit( visitors.FillInLocalPointers({ "": b, "typing": t, "builtins": b })) t.Visit( visitors.FillInLocalPointers({ "": t, "typing": t, "builtins": b })) b.Visit(visitors.VerifyLookup()) t.Visit(visitors.VerifyLookup()) b.Visit(visitors.VerifyContainers()) t.Visit(visitors.VerifyContainers()) _cached_builtins_pytd[python_version] = (b, t) return _cached_builtins_pytd[python_version]
def PrepareForExport(module_name, python_version, ast, loader): """Prepare an ast as if it was parsed and loaded. External dependencies will not be resolved, as the ast generated by this method is supposed to be exported. Args: module_name: The module_name as a string for the returned ast. python_version: A tuple of (major, minor) python version as string (see config.python_version). ast: pytd.TypeDeclUnit, is only used if src is None. loader: A load_pytd.Loader instance. Returns: A pytd.TypeDeclUnit representing the supplied AST as it would look after being written to a file and parsed. """ # This is a workaround for functionality which crept into places it doesn't # belong. Ideally this would call some transformation Visitors on ast to # transform it into the same ast we get after parsing and loading (compare # load_pytd.Loader.load_file). Unfortunately parsing has some special cases, # e.g. '__init__' return type and '__new__' being a 'staticmethod', which # need to be moved to visitors before we can do this. Printing an ast also # applies transformations, # e.g. visitors.PrintVisitor._FormatContainerContents, which need to move to # their own visitors so they can be applied without printing. src = pytd_utils.Print(ast) ast = parser.parse_string(src=src, name=module_name, python_version=python_version) ast = ast.Visit(visitors.LookupBuiltins(loader.builtins, full_names=False)) ast = ast.Visit( visitors.ExpandCompatibleBuiltins(loader.builtins, python_version)) ast = ast.Visit(visitors.LookupLocalTypes()) ast = ast.Visit(visitors.AdjustTypeParameters()) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.FillInLocalPointers({"": ast, module_name: ast})) ast = ast.Visit(visitors.CanonicalOrderingVisitor()) ast = ast.Visit( visitors.ClassTypeToLateType( ignore=[module_name + ".", "__builtin__.", "typing."])) return ast
def _process_module(self, module_name, filename, mod_ast): """Create a module from a loaded ast and save it to the loader cache. Args: module_name: The fully qualified name of the module being imported. May be None. filename: The file the ast was generated from. May be None. mod_ast: The pytd.TypeDeclUnit representing the module. Returns: The ast (pytd.TypeDeclUnit) as represented in this loader. """ module = Module(module_name, filename, mod_ast) # Builtins need to be resolved before the module is cached so that they are # not mistaken for local types. External types can be left unresolved # because they are unambiguous. self._resolver.allow_singletons = False module.ast = self._resolver.resolve_builtin_types(module.ast) self._modules[module_name] = module try: self._resolver.allow_singletons = True module.ast = self._resolve_external_and_local_types(module.ast) # We need to resolve builtin singletons after we have made sure they are # not shadowed by a local or a star import. module.ast = self._resolver.resolve_builtin_types(module.ast) self._resolver.allow_singletons = False # Now that any imported TypeVar instances have been resolved, adjust type # parameters in classes and functions. module.ast = module.ast.Visit(visitors.AdjustTypeParameters()) # Now we can fill in internal cls pointers to ClassType nodes in the # module. This code executes when the module is first loaded, which # happens before any others use it to resolve dependencies, so there are # no external pointers into the module at this point. module_map = {"": module.ast, module_name: module.ast} module.ast.Visit(visitors.FillInLocalPointers(module_map)) except: # don't leave half-resolved modules around del self._modules[module_name] raise if module_name: self.add_module_prefixes(module_name) return module.ast
def value_to_pytd_type(self, node, v, seen, view): """Get a PyTD type representing this object, as seen at a node. Args: node: The node from which we want to observe this object. v: The object. seen: The set of values seen before while computing the type. view: A Variable -> binding map. Returns: A PyTD type. """ if isinstance(v, (abstract.Empty, typing_overlay.NoReturn)): return pytd.NothingType() elif isinstance(v, abstract.TypeParameterInstance): if v.instance.get_instance_type_parameter(v.full_name).bindings: # The type parameter was initialized. Set the view to None, since we # don't include v.instance in the view. return pytd_utils.JoinTypes( self.value_to_pytd_type(node, p, seen, None) for p in v.instance.get_instance_type_parameter(v.full_name).data) elif v.param.constraints: return pytd_utils.JoinTypes( self.value_instance_to_pytd_type(node, p, None, seen, view) for p in v.param.constraints) elif v.param.bound: return self.value_instance_to_pytd_type( node, v.param.bound, None, seen, view) else: return pytd.AnythingType() elif isinstance(v, typing_overlay.TypeVar): return pytd.NamedType("__builtin__.type") elif isinstance(v, abstract.FUNCTION_TYPES): try: signatures = abstract_utils.get_signatures(v) except NotImplementedError: return pytd.NamedType("typing.Callable") if len(signatures) == 1: val = self.signature_to_callable(signatures[0], self.vm) if (not isinstance(v, abstract.PYTD_FUNCTION_TYPES) or not self.vm.annotations_util.get_type_parameters(val)): # This is a workaround to make sure we don't put unexpected type # parameters in call traces. return self.value_instance_to_pytd_type(node, val, None, seen, view) return pytd.NamedType("typing.Callable") elif isinstance(v, (abstract.ClassMethod, abstract.StaticMethod)): return self.value_to_pytd_type(node, v.method, seen, view) elif isinstance(v, (special_builtins.IsInstance, special_builtins.ClassMethodCallable)): return pytd.NamedType("typing.Callable") elif isinstance(v, mixin.Class): param = self.value_instance_to_pytd_type(node, v, None, seen, view) return pytd.GenericType(base_type=pytd.NamedType("__builtin__.type"), parameters=(param,)) elif isinstance(v, abstract.Module): return pytd.NamedType("__builtin__.module") elif isinstance(v, abstract.SimpleAbstractValue): if v.cls: ret = self.value_instance_to_pytd_type( node, v.cls, v, seen=seen, view=view) ret.Visit(visitors.FillInLocalPointers( {"__builtin__": self.vm.loader.builtins})) return ret else: # We don't know this type's __class__, so return AnythingType to # indicate that we don't know anything about what this is. # This happens e.g. for locals / globals, which are returned from the # code in class declarations. log.info("Using ? for %s", v.name) return pytd.AnythingType() elif isinstance(v, abstract.Union): return pytd.UnionType(tuple(self.value_to_pytd_type(node, o, seen, view) for o in v.options)) elif isinstance(v, special_builtins.SuperInstance): return pytd.NamedType("__builtin__.super") elif isinstance(v, (abstract.Unsolvable, abstract.TypeParameter)): # Arguably, the type of a type parameter is NamedType("typing.TypeVar"), # but pytype doesn't know how to handle that, so let's just go with Any. return pytd.AnythingType() elif isinstance(v, abstract.Unknown): return pytd.NamedType(v.class_name) elif isinstance(v, abstract.BuildClass): return pytd.NamedType("typing.Callable") else: raise NotImplementedError(v.__class__.__name__)
def testFillInFunctionTypePointers(self): src = textwrap.dedent("def f(): ...") tree = self.Parse(src) ty = pytd.FunctionType("f", None) ty.Visit(visitors.FillInLocalPointers({"": tree})) self.assertEqual(ty.function, tree.Lookup("f"))
def value_to_pytd_type(self, node, v, seen, view): """Get a PyTD type representing this object, as seen at a node. Args: node: The node from which we want to observe this object. v: The object. seen: The set of values seen before while computing the type. view: A Variable -> binding map. Returns: A PyTD type. """ if isinstance(v, (abstract.Empty, typing_overlay.NoReturn)): return pytd.NothingType() elif isinstance(v, abstract.TypeParameterInstance): if (v.module in self._scopes or v.instance is abstract_utils.DUMMY_CONTAINER): return self._typeparam_to_def(node, v.param, v.param.name) elif v.instance.get_instance_type_parameter(v.full_name).bindings: # The type parameter was initialized. Set the view to None, since we # don't include v.instance in the view. return pytd_utils.JoinTypes( self.value_to_pytd_type(node, p, seen, None) for p in v.instance.get_instance_type_parameter(v.full_name).data) elif v.param.constraints: return pytd_utils.JoinTypes( self.value_instance_to_pytd_type(node, p, None, seen, view) for p in v.param.constraints) elif v.param.bound: return self.value_instance_to_pytd_type( node, v.param.bound, None, seen, view) else: return pytd.AnythingType() elif isinstance(v, typing_overlay.TypeVar): return pytd.NamedType("builtins.type") elif isinstance(v, dataclass_overlay.FieldInstance): if not v.default: return pytd.AnythingType() return pytd_utils.JoinTypes( self.value_to_pytd_type(node, d, seen, view) for d in v.default.data) elif isinstance(v, attr_overlay.AttribInstance): ret = self.value_to_pytd_type(node, v.typ, seen, view) md = metadata.to_pytd(v.to_metadata()) return pytd.Annotated(ret, ("'pytype_metadata'", md)) elif isinstance(v, special_builtins.PropertyInstance): return pytd.NamedType("builtins.property") elif isinstance(v, typed_dict.TypedDict): return pytd.NamedType(v.props.name) elif isinstance(v, abstract.FUNCTION_TYPES): try: signatures = function.get_signatures(v) except NotImplementedError: return pytd.NamedType("typing.Callable") if len(signatures) == 1: val = self.signature_to_callable(signatures[0]) if not isinstance( v, abstract.PYTD_FUNCTION_TYPES) or not val.formal: # This is a workaround to make sure we don't put unexpected type # parameters in call traces. return self.value_instance_to_pytd_type( node, val, None, seen, view) return pytd.NamedType("typing.Callable") elif isinstance(v, (abstract.ClassMethod, abstract.StaticMethod)): return self.value_to_pytd_type(node, v.method, seen, view) elif isinstance(v, (special_builtins.IsInstance, special_builtins.ClassMethodCallable)): return pytd.NamedType("typing.Callable") elif isinstance(v, abstract.Class): param = self.value_instance_to_pytd_type(node, v, None, seen, view) return pytd.GenericType(base_type=pytd.NamedType("builtins.type"), parameters=(param, )) elif isinstance(v, abstract.Module): return pytd.Alias(v.name, pytd.Module(v.name, module_name=v.full_name)) elif (self._output_mode >= Converter.OutputMode.LITERAL and isinstance(v, abstract.ConcreteValue) and isinstance(v.pyval, (int, str, bytes))): # LITERAL mode is used only for pretty-printing, so we just stringify the # inner value rather than properly converting it. return pytd.Literal(repr(v.pyval)) elif isinstance(v, abstract.SimpleValue): ret = self.value_instance_to_pytd_type(node, v.cls, v, seen=seen, view=view) ret.Visit( visitors.FillInLocalPointers( {"builtins": self.ctx.loader.builtins})) return ret elif isinstance(v, abstract.Union): return pytd_utils.JoinTypes( self.value_to_pytd_type(node, o, seen, view) for o in v.options) elif isinstance(v, special_builtins.SuperInstance): return pytd.NamedType("builtins.super") elif isinstance(v, abstract.TypeParameter): # Arguably, the type of a type parameter is NamedType("typing.TypeVar"), # but pytype doesn't know how to handle that, so let's just go with Any # unless self._detailed is set. if self._detailed: return pytd.NamedType("typing.TypeVar") else: return pytd.AnythingType() elif isinstance(v, abstract.Unsolvable): return pytd.AnythingType() elif isinstance(v, abstract.Unknown): return pytd.NamedType(v.class_name) elif isinstance(v, abstract.BuildClass): return pytd.NamedType("typing.Callable") elif isinstance(v, abstract.FinalAnnotation): param = self.value_to_pytd_type(node, v.annotation, seen, view) return pytd.GenericType(base_type=pytd.NamedType("typing.Final"), parameters=(param, )) else: raise NotImplementedError(v.__class__.__name__)
def _finish_pyi(self, pyval, ast=None): module_map = self._get_module_map() module_map[""] = ast or pyval # The module itself (local lookup) pyval.Visit(visitors.FillInLocalPointers(module_map))
def _finish_ast(self, ast): module_map = self._get_module_map() module_map[""] = ast # The module itself (local lookup) ast.Visit(visitors.FillInLocalPointers(module_map))
def value_to_pytd_type(self, node, v, seen, view): """Get a PyTD type representing this object, as seen at a node. Args: node: The node from which we want to observe this object. v: The object. seen: The set of values seen before while computing the type. view: A Variable -> binding map. Returns: A PyTD type. """ if isinstance(v, (abstract.Empty, typing_overlay.NoReturn)): return pytd.NothingType() elif isinstance(v, abstract.TypeParameterInstance): if v.module in self._scopes: return self._typeparam_to_def(node, v.param, v.param.name) elif v.instance.get_instance_type_parameter(v.full_name).bindings: # The type parameter was initialized. Set the view to None, since we # don't include v.instance in the view. return pytd_utils.JoinTypes( self.value_to_pytd_type(node, p, seen, None) for p in v.instance.get_instance_type_parameter(v.full_name).data) elif v.param.constraints: return pytd_utils.JoinTypes( self.value_instance_to_pytd_type(node, p, None, seen, view) for p in v.param.constraints) elif v.param.bound: return self.value_instance_to_pytd_type( node, v.param.bound, None, seen, view) else: return pytd.AnythingType() elif isinstance(v, typing_overlay.TypeVar): return pytd.NamedType("builtins.type") elif isinstance(v, dataclass_overlay.FieldInstance): if not v.default: return pytd.AnythingType() return pytd_utils.JoinTypes( self.value_to_pytd_type(node, d, seen, view) for d in v.default.data) elif isinstance(v, abstract.FUNCTION_TYPES): try: signatures = abstract_utils.get_signatures(v) except NotImplementedError: return pytd.NamedType("typing.Callable") if len(signatures) == 1: val = self.signature_to_callable(signatures[0]) if not isinstance( v, abstract.PYTD_FUNCTION_TYPES) or not val.formal: # This is a workaround to make sure we don't put unexpected type # parameters in call traces. return self.value_instance_to_pytd_type( node, val, None, seen, view) return pytd.NamedType("typing.Callable") elif isinstance(v, (abstract.ClassMethod, abstract.StaticMethod)): return self.value_to_pytd_type(node, v.method, seen, view) elif isinstance(v, (special_builtins.IsInstance, special_builtins.ClassMethodCallable)): return pytd.NamedType("typing.Callable") elif isinstance(v, class_mixin.Class): param = self.value_instance_to_pytd_type(node, v, None, seen, view) return pytd.GenericType(base_type=pytd.NamedType("builtins.type"), parameters=(param, )) elif isinstance(v, abstract.Module): return pytd.NamedType("builtins.module") elif (self._output_mode >= Converter.OutputMode.LITERAL and isinstance(v, abstract.ConcreteValue) and isinstance(v.pyval, (int, str, bytes))): # LITERAL mode is used only for pretty-printing, so we just stringify the # inner value rather than properly converting it. return pytd.Literal(repr(v.pyval)) elif isinstance(v, abstract.SimpleValue): if v.cls: ret = self.value_instance_to_pytd_type(node, v.cls, v, seen=seen, view=view) ret.Visit( visitors.FillInLocalPointers( {"builtins": self.vm.loader.builtins})) return ret else: # We don't know this type's __class__, so return AnythingType to # indicate that we don't know anything about what this is. # This happens e.g. for locals / globals, which are returned from the # code in class declarations. log.info("Using Any for %s", v.name) return pytd.AnythingType() elif isinstance(v, abstract.Union): opts = [] for o in v.options: # NOTE: Guarding printing of type parameters behind _detailed until # round-tripping is working properly. if self._detailed and isinstance(o, abstract.TypeParameter): opt = self._typeparam_to_def(node, o, o.name) else: opt = self.value_to_pytd_type(node, o, seen, view) opts.append(opt) return pytd.UnionType(tuple(opts)) elif isinstance(v, special_builtins.SuperInstance): return pytd.NamedType("builtins.super") elif isinstance(v, abstract.TypeParameter): # Arguably, the type of a type parameter is NamedType("typing.TypeVar"), # but pytype doesn't know how to handle that, so let's just go with Any # unless self._detailed is set. if self._detailed: return pytd.NamedType("typing.TypeVar") else: return pytd.AnythingType() elif isinstance(v, abstract.Unsolvable): return pytd.AnythingType() elif isinstance(v, abstract.Unknown): return pytd.NamedType(v.class_name) elif isinstance(v, abstract.BuildClass): return pytd.NamedType("typing.Callable") else: raise NotImplementedError(v.__class__.__name__)
def _resolve_classtype_pointers(self, mod_ast, *, lookup_ast=None): module_map = self._modules.get_module_map() module_map[""] = lookup_ast or mod_ast # The module itself (local lookup) mod_ast.Visit(visitors.FillInLocalPointers(module_map))