def load(self, options):
"""Read builtins.pytd and typing.pytd, and return the parsed modules."""
t = self._parse_predefined("typing", options)
b = self._parse_predefined("builtins", options)
b = b.Visit(
visitors.LookupExternalTypes({"typing": t}, self_name="builtins"))
t = t.Visit(visitors.LookupBuiltins(b))
b = b.Visit(visitors.NamedTypeToClassType())
t = t.Visit(visitors.NamedTypeToClassType())
b = b.Visit(visitors.AdjustTypeParameters())
t = t.Visit(visitors.AdjustTypeParameters())
b = b.Visit(visitors.CanonicalOrderingVisitor())
t = t.Visit(visitors.CanonicalOrderingVisitor())
b.Visit(
visitors.FillInLocalPointers({
"": b,
"typing": t,
"builtins": b
}))
t.Visit(
visitors.FillInLocalPointers({
"": t,
"typing": t,
"builtins": b
}))
b.Visit(visitors.VerifyLookup())
t.Visit(visitors.VerifyLookup())
b.Visit(visitors.VerifyContainers())
t.Visit(visitors.VerifyContainers())
return b, t
def _process_module(self, module_name, filename, ast):
"""Create a module from a loaded ast and save it to the loader cache.
Args:
module_name: The fully qualified name of the module being imported.
filename: The file the ast was generated from.
ast: The pytd.TypeDeclUnit representing the module.
Returns:
The ast (pytd.TypeDeclUnit) as represented in this loader.
"""
module = Module(module_name, filename, ast)
self._modules[module_name] = module
try:
module.ast = self._postprocess_pyi(module.ast)
# Now that any imported TypeVar instances have been resolved, adjust type
# parameters in classes and functions.
module.ast = module.ast.Visit(visitors.AdjustTypeParameters())
# Now we can fill in internal cls pointers to ClassType nodes in the
# module. This code executes when the module is first loaded, which
# happens before any others use it to resolve dependencies, so there are
# no external pointers into the module at this point.
module.ast.Visit(
visitors.FillInLocalPointers({"": module.ast,
module_name: module.ast}))
except:
# don't leave half-resolved modules around
del self._modules[module_name]
raise
self.add_module_prefixes(module_name)
return module.ast
def parse_and_solve(self, src):
ast = self.parse(src)
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = ast.Visit(visitors.AdjustTypeParameters())
types, _ = convert_structural.solve(ast, builtins_pytd=self.builtins_pytd)
# Drop "__builtin__" prefix, for more readable tests.
return {k: {v.rpartition("__builtin__.")[2] for v in l}
for k, l in types.items()}
def ParseWithBuiltins(self, src):
ast = parser.parse_string(textwrap.dedent(src), options=self.options)
ast = ast.Visit(visitors.LookupExternalTypes(
{"builtins": self.loader.builtins, "typing": self.loader.typing}))
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = ast.Visit(visitors.AdjustTypeParameters())
ast.Visit(visitors.FillInLocalPointers({
"": ast, "builtins": self.loader.builtins}))
ast.Visit(visitors.VerifyVisitor())
return ast
def GetBuiltinsAndTyping(python_version): # Deprecated. Use load_pytd instead.
"""Get __builtin__.pytd and typing.pytd."""
assert python_version
global _cached_builtins_pytd
if _cached_builtins_pytd.cache:
assert _cached_builtins_pytd.version == python_version
else:
t = parser.parse_string(_FindBuiltinFile("typing", python_version),
name="typing",
python_version=python_version)
b = parser.parse_string(_FindBuiltinFile("__builtin__",
python_version),
name="__builtin__",
python_version=python_version)
b = b.Visit(
visitors.LookupExternalTypes({"typing": t},
full_names=True,
self_name="__builtin__"))
t = t.Visit(visitors.LookupBuiltins(b))
b = b.Visit(visitors.NamedTypeToClassType())
t = t.Visit(visitors.NamedTypeToClassType())
b = b.Visit(visitors.AdjustTypeParameters())
t = t.Visit(visitors.AdjustTypeParameters())
b = b.Visit(visitors.CanonicalOrderingVisitor())
t = t.Visit(visitors.CanonicalOrderingVisitor())
b.Visit(
visitors.FillInLocalPointers({
"": b,
"typing": t,
"__builtin__": b
}))
t.Visit(
visitors.FillInLocalPointers({
"": t,
"typing": t,
"__builtin__": b
}))
b.Visit(visitors.VerifyLookup())
t.Visit(visitors.VerifyLookup())
b.Visit(visitors.VerifyContainers())
t.Visit(visitors.VerifyContainers())
_cached_builtins_pytd = Cache(python_version, (b, t))
return _cached_builtins_pytd.cache
def ParseWithBuiltins(self, src):
ast = parser.parse_string(textwrap.dedent(src),
python_version=self.PYTHON_VERSION)
ast = ast.Visit(visitors.LookupExternalTypes(
{"__builtin__": self.loader.builtins, "typing": self.loader.typing}))
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = ast.Visit(visitors.AdjustTypeParameters())
ast.Visit(visitors.FillInLocalPointers({
"": ast, "__builtin__": self.loader.builtins}))
ast.Visit(visitors.VerifyVisitor())
return ast
def Parse(self, src, name=None, version=None, platform=None):
version = version or self.PYTHON_VERSION
tree = parser.parse_string(
textwrap.dedent(src), name=name, python_version=version,
platform=platform)
tree = tree.Visit(visitors.NamedTypeToClassType())
tree = tree.Visit(visitors.AdjustTypeParameters())
# Convert back to named types for easier testing
tree = tree.Visit(visitors.ClassTypeToNamedType())
tree.Visit(visitors.VerifyVisitor())
return tree
def Parse(self, src, name=None, version=None, platform=None):
if version:
self.options.python_version = version
if platform:
self.options.platform = platform
tree = parser.parse_string(
textwrap.dedent(src), name=name, options=self.options)
tree = tree.Visit(visitors.NamedTypeToClassType())
tree = tree.Visit(visitors.AdjustTypeParameters())
# Convert back to named types for easier testing
tree = tree.Visit(visitors.ClassTypeToNamedType())
tree.Visit(visitors.VerifyVisitor())
return tree
def testGetBasesInMRO(self):
ast = parser.parse_string(textwrap.dedent("""
from typing import Generic, TypeVar
T = TypeVar("T")
class Foo(Generic[T]): pass
class Bar(Foo[int]): pass
"""), python_version=self.PYTHON_VERSION)
ast = ast.Visit(visitors.AdjustTypeParameters())
loader = load_pytd.Loader(None, self.PYTHON_VERSION)
ast = loader.resolve_ast(ast)
bases = mro.GetBasesInMRO(ast.Lookup("Bar"), lookup_ast=ast)
self.assertListEqual(["Foo", "typing.Generic", "__builtin__.object"],
[t.name for t in bases])
def GetBuiltinsAndTyping(python_version): # Deprecated. Use load_pytd instead.
"""Get builtins.pytd and typing.pytd."""
assert python_version
if python_version not in _cached_builtins_pytd:
t = parser.parse_string(_FindBuiltinFile("typing", python_version),
name="typing",
python_version=python_version)
b = parser.parse_string(_FindBuiltinFile("builtins", python_version),
name="builtins",
python_version=python_version)
b = b.Visit(
visitors.LookupExternalTypes({"typing": t}, self_name="builtins"))
t = t.Visit(visitors.LookupBuiltins(b))
b = b.Visit(visitors.NamedTypeToClassType())
t = t.Visit(visitors.NamedTypeToClassType())
b = b.Visit(visitors.AdjustTypeParameters())
t = t.Visit(visitors.AdjustTypeParameters())
b = b.Visit(visitors.CanonicalOrderingVisitor())
t = t.Visit(visitors.CanonicalOrderingVisitor())
b.Visit(
visitors.FillInLocalPointers({
"": b,
"typing": t,
"builtins": b
}))
t.Visit(
visitors.FillInLocalPointers({
"": t,
"typing": t,
"builtins": b
}))
b.Visit(visitors.VerifyLookup())
t.Visit(visitors.VerifyLookup())
b.Visit(visitors.VerifyContainers())
t.Visit(visitors.VerifyContainers())
_cached_builtins_pytd[python_version] = (b, t)
return _cached_builtins_pytd[python_version]
def test_get_bases_in_mro(self):
ast = parser.parse_string(textwrap.dedent("""
from typing import Generic, TypeVar
T = TypeVar("T")
class Foo(Generic[T]): pass
class Bar(Foo[int]): pass
"""),
options=self.options)
ast = ast.Visit(visitors.AdjustTypeParameters())
loader = load_pytd.Loader(
config.Options.create(python_version=self.python_version))
ast = loader.resolve_ast(ast)
bases = mro.GetBasesInMRO(ast.Lookup("Bar"), lookup_ast=ast)
self.assertListEqual(["Foo", "typing.Generic", "builtins.object"],
[t.name for t in bases])
def compute_types(self, defs):
classes = (tuple(self.pytd_classes_for_unknowns()) +
tuple(self.pytd_classes_for_call_traces()) +
self.pytd_classes_for_namedtuple_instances())
functions = tuple(self.pytd_functions_for_call_traces())
aliases = tuple(self.pytd_aliases())
ty = pytd_utils.Concat(
self.pytd_for_types(defs),
pytd_utils.CreateModule("unknowns", classes=classes,
functions=functions, aliases=aliases))
ty = ty.Visit(optimize.CombineReturnsAndExceptions())
ty = ty.Visit(optimize.PullInMethodClasses())
ty = ty.Visit(visitors.DefaceUnresolved(
[ty, self.loader.concat_all()], "~unknown"))
return ty.Visit(visitors.AdjustTypeParameters())
def _process_module(self, module_name, filename, mod_ast):
"""Create a module from a loaded ast and save it to the loader cache.
Args:
module_name: The fully qualified name of the module being imported.
May be None.
filename: The file the ast was generated from. May be None.
mod_ast: The pytd.TypeDeclUnit representing the module.
Returns:
The ast (pytd.TypeDeclUnit) as represented in this loader.
"""
module = Module(module_name, filename, mod_ast)
# Builtins need to be resolved before the module is cached so that they are
# not mistaken for local types. External types can be left unresolved
# because they are unambiguous.
self._resolver.allow_singletons = False
module.ast = self._resolver.resolve_builtin_types(module.ast)
self._modules[module_name] = module
try:
self._resolver.allow_singletons = True
module.ast = self._resolve_external_and_local_types(module.ast)
# We need to resolve builtin singletons after we have made sure they are
# not shadowed by a local or a star import.
module.ast = self._resolver.resolve_builtin_types(module.ast)
self._resolver.allow_singletons = False
# Now that any imported TypeVar instances have been resolved, adjust type
# parameters in classes and functions.
module.ast = module.ast.Visit(visitors.AdjustTypeParameters())
# Now we can fill in internal cls pointers to ClassType nodes in the
# module. This code executes when the module is first loaded, which
# happens before any others use it to resolve dependencies, so there are
# no external pointers into the module at this point.
module_map = {"": module.ast, module_name: module.ast}
module.ast.Visit(visitors.FillInLocalPointers(module_map))
except:
# don't leave half-resolved modules around
del self._modules[module_name]
raise
if module_name:
self.add_module_prefixes(module_name)
return module.ast
def PrepareForExport(module_name, python_version, ast, loader):
"""Prepare an ast as if it was parsed and loaded.
External dependencies will not be resolved, as the ast generated by this
method is supposed to be exported.
Args:
module_name: The module_name as a string for the returned ast.
python_version: A tuple of (major, minor) python version as string
(see config.python_version).
ast: pytd.TypeDeclUnit, is only used if src is None.
loader: A load_pytd.Loader instance.
Returns:
A pytd.TypeDeclUnit representing the supplied AST as it would look after
being written to a file and parsed.
"""
# This is a workaround for functionality which crept into places it doesn't
# belong. Ideally this would call some transformation Visitors on ast to
# transform it into the same ast we get after parsing and loading (compare
# load_pytd.Loader.load_file). Unfortunately parsing has some special cases,
# e.g. '__init__' return type and '__new__' being a 'staticmethod', which
# need to be moved to visitors before we can do this. Printing an ast also
# applies transformations,
# e.g. visitors.PrintVisitor._FormatContainerContents, which need to move to
# their own visitors so they can be applied without printing.
src = pytd_utils.Print(ast)
ast = parser.parse_string(src=src,
name=module_name,
python_version=python_version)
ast = ast.Visit(visitors.LookupBuiltins(loader.builtins, full_names=False))
ast = ast.Visit(
visitors.ExpandCompatibleBuiltins(loader.builtins, python_version))
ast = ast.Visit(visitors.LookupLocalTypes())
ast = ast.Visit(visitors.AdjustTypeParameters())
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = ast.Visit(visitors.FillInLocalPointers({"": ast, module_name: ast}))
ast = ast.Visit(visitors.CanonicalOrderingVisitor())
ast = ast.Visit(
visitors.ClassTypeToLateType(
ignore=[module_name + ".", "__builtin__.", "typing."]))
return ast
def LinkAgainstSimpleBuiltins(self, ast):
ast = ast.Visit(visitors.AdjustTypeParameters())
ast = visitors.LookupClasses(ast, self.mini_builtins)
return ast
