def _process_module(self, module_name, filename, ast):
"""Create a module from a loaded ast and save it to the loader cache.
Args:
module_name: The fully qualified name of the module being imported.
filename: The file the ast was generated from.
ast: The pytd.TypeDeclUnit representing the module.
Returns:
The ast (pytd.TypeDeclUnit) as represented in this loader.
"""
module = Module(module_name, filename, ast)
self._modules[module_name] = module
try:
module.ast = self._postprocess_pyi(module.ast)
# Now that any imported TypeVar instances have been resolved, adjust type
# parameters in classes and functions.
module.ast = module.ast.Visit(visitors.AdjustTypeParameters())
# Now we can fill in internal cls pointers to ClassType nodes in the
# module. This code executes when the module is first loaded, which
# happens before any others use it to resolve dependencies, so there are
# no external pointers into the module at this point.
module.ast.Visit(
visitors.FillInLocalPointers({"": module.ast,
module_name: module.ast}))
except:
# don't leave half-resolved modules around
del self._modules[module_name]
raise
return module.ast
def LinkAgainstSimpleBuiltins(self, ast):
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = visitors.AdjustTypeParameters(ast)
ast.Visit(visitors.FillInModuleClasses({"": ast}))
ast = ast.Visit(visitors.LookupFullNames([self.mini_builtins]))
ast.Visit(visitors.VerifyLookup())
return ast
def parse_and_solve(self, src):
ast = self.parse(src)
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = ast.Visit(visitors.AdjustTypeParameters())
types, _ = convert_structural.solve(ast, builtins_pytd=self.builtins_pytd)
# Drop "__builtin__" prefix, for more readable tests.
return {k: {v.rpartition("__builtin__.")[2] for v in l}
for k, l in types.items()}
def Parse(self, src, version=None, platform=None):
# TODO(kramm): Using self.parser here breaks tests. Why?
tree = parser.TypeDeclParser(version=version, platform=platform).Parse(
textwrap.dedent(src))
tree = tree.Visit(visitors.NamedTypeToClassType())
tree = visitors.AdjustTypeParameters(tree)
# Convert back to named types for easier testing
tree = tree.Visit(visitors.ClassTypeToNamedType())
tree.Visit(visitors.VerifyVisitor())
return tree
def ParseWithBuiltins(self, src):
ast = parser.TypeDeclParser().Parse(textwrap.dedent(src))
b, t = builtins.GetBuiltinsAndTyping()
ast = ast.Visit(visitors.LookupExternalTypes(
{"__builtin__": b, "typing": t}, full_names=True))
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = visitors.AdjustTypeParameters(ast)
ast.Visit(visitors.FillInModuleClasses({"": ast}))
ast.Visit(visitors.VerifyVisitor())
return ast
def GetBuiltinsAndTyping(python_version):
"""Get __builtin__.pytd and typing.pytd."""
assert python_version
global _cached_builtins_pytd
if _cached_builtins_pytd.cache:
assert _cached_builtins_pytd.version == python_version
else:
t = parser.parse_string(_FindBuiltinFile("typing", python_version),
name="typing",
python_version=python_version)
b = parser.parse_string(_FindBuiltinFile("__builtin__",
python_version),
name="__builtin__",
python_version=python_version)
b = b.Visit(
visitors.LookupExternalTypes({"typing": t},
full_names=True,
self_name="__builtin__"))
t = t.Visit(visitors.LookupBuiltins(b))
b = b.Visit(visitors.NamedTypeToClassType())
t = t.Visit(visitors.NamedTypeToClassType())
b = b.Visit(visitors.AdjustTypeParameters())
t = t.Visit(visitors.AdjustTypeParameters())
b = b.Visit(visitors.CanonicalOrderingVisitor())
t = t.Visit(visitors.CanonicalOrderingVisitor())
b.Visit(
visitors.FillInLocalPointers({
"": b,
"typing": t,
"__builtin__": b
}))
t.Visit(
visitors.FillInLocalPointers({
"": t,
"typing": t,
"__builtin__": b
}))
b.Visit(visitors.VerifyLookup())
t.Visit(visitors.VerifyLookup())
b.Visit(visitors.VerifyContainers())
t.Visit(visitors.VerifyContainers())
_cached_builtins_pytd = Cache(python_version, (b, t))
return _cached_builtins_pytd.cache
def Parse(self, src, name=None, version=None, platform=None):
version = version or self.PYTHON_VERSION
# TODO(kramm): Using self.parser here breaks tests. Why?
tree = parser.parse_string(textwrap.dedent(src),
name=name,
python_version=version,
platform=platform)
tree = tree.Visit(visitors.NamedTypeToClassType())
tree = tree.Visit(visitors.AdjustTypeParameters())
# Convert back to named types for easier testing
tree = tree.Visit(visitors.ClassTypeToNamedType())
tree.Visit(visitors.VerifyVisitor())
return tree
def ParseWithBuiltins(self, src):
ast = parser.parse_string(textwrap.dedent(src))
b, t = builtins.GetBuiltinsAndTyping()
ast = ast.Visit(
visitors.LookupExternalTypes({
"__builtin__": b,
"typing": t
},
full_names=True))
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = ast.Visit(visitors.AdjustTypeParameters())
ast.Visit(visitors.FillInLocalPointers({"": ast, "__builtin__": b}))
ast.Visit(visitors.VerifyVisitor())
return ast
def GetBuiltinsAndTyping():
"""Get __builtin__.pytd and typing.pytd."""
global _cached_builtins_pytd
if not _cached_builtins_pytd:
t = parser.TypeDeclParser().Parse(_FindBuiltinFile("typing"),
name="typing")
t = t.Visit(visitors.AddNamePrefix())
b = parser.TypeDeclParser().Parse(_FindBuiltinFile("__builtin__"),
name="__builtin__")
b = b.Visit(visitors.AddNamePrefix())
b = b.Visit(visitors.NamedTypeToClassType())
b = b.Visit(
visitors.LookupExternalTypes({"typing": t},
full_names=True,
self_name="__builtin__"))
t = t.Visit(visitors.LookupBuiltins(b))
t = t.Visit(visitors.NamedTypeToClassType())
b = visitors.AdjustTypeParameters(b)
t = visitors.AdjustTypeParameters(t)
b.Visit(
visitors.FillInModuleClasses({
"": b,
"typing": t,
"__builtin__": b
}))
t.Visit(
visitors.FillInModuleClasses({
"": t,
"typing": t,
"__builtin__": b
}))
b.Visit(visitors.VerifyLookup())
t.Visit(visitors.VerifyLookup())
b.Visit(visitors.VerifyContainers())
t.Visit(visitors.VerifyContainers())
_cached_builtins_pytd = b, t
return _cached_builtins_pytd
def compute_types(self, defs):
ty = pytd_utils.Concat(
self.pytd_for_types(defs),
pytd.TypeDeclUnit(
"unknowns",
constants=tuple(),
type_params=tuple(self.pytd_typevars()),
classes=tuple(self.pytd_classes_for_unknowns()) +
tuple(self.pytd_classes_for_call_traces()),
functions=tuple(self.pytd_functions_for_call_traces()),
aliases=tuple(self.pytd_aliases())))
ty = ty.Visit(optimize.PullInMethodClasses())
ty = ty.Visit(visitors.DefaceUnresolved(
[ty, self.loader.concat_all()], "~unknown"))
return ty.Visit(visitors.AdjustTypeParameters())
def GetBuiltinsAndTyping():
"""Get __builtin__.pytd and typing.pytd."""
global _cached_builtins_pytd
if not _cached_builtins_pytd:
t = parser.parse_string(_FindBuiltinFile("typing"), name="typing")
b = parser.parse_string(_FindBuiltinFile("__builtin__"),
name="__builtin__")
b = b.Visit(
visitors.LookupExternalTypes({"typing": t},
full_names=True,
self_name="__builtin__"))
t = t.Visit(visitors.LookupBuiltins(b))
b = b.Visit(visitors.NamedTypeToClassType())
t = t.Visit(visitors.NamedTypeToClassType())
b = b.Visit(visitors.AdjustTypeParameters())
t = t.Visit(visitors.AdjustTypeParameters())
b = b.Visit(visitors.CanonicalOrderingVisitor())
t = t.Visit(visitors.CanonicalOrderingVisitor())
b.Visit(
visitors.FillInLocalPointers({
"": b,
"typing": t,
"__builtin__": b
}))
t.Visit(
visitors.FillInLocalPointers({
"": t,
"typing": t,
"__builtin__": b
}))
b.Visit(visitors.VerifyLookup())
t.Visit(visitors.VerifyLookup())
b.Visit(visitors.VerifyContainers())
t.Visit(visitors.VerifyContainers())
_cached_builtins_pytd = b, t
return _cached_builtins_pytd
def ParseWithBuiltins(self, src):
ast = parser.parse_string(textwrap.dedent(src),
python_version=self.PYTHON_VERSION)
ast = ast.Visit(
visitors.LookupExternalTypes(
{
"__builtin__": self.loader.builtins,
"typing": self.loader.typing
},
full_names=True))
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = ast.Visit(visitors.AdjustTypeParameters())
ast.Visit(
visitors.FillInLocalPointers({
"": ast,
"__builtin__": self.loader.builtins
}))
ast.Visit(visitors.VerifyVisitor())
return ast
def _load_file(self, module_name, filename, ast=None):
"""Load (or retrieve from cache) a module and resolve its dependencies."""
self._concatenated = None # invalidate
existing = self._modules.get(module_name)
if existing:
if existing.filename != filename:
raise AssertionError(
"%s exists as both %s and %s" %
(module_name, filename, existing.filename))
return existing.ast
if not ast:
ast = builtins.ParsePyTD(
filename=filename,
module=module_name,
python_version=self.options.python_version)
ast = self._postprocess_pyi(ast)
module = Module(module_name, filename, ast)
self._modules[module_name] = module
try:
module.ast = self._load_and_resolve_ast_dependencies(
module.ast, module_name)
# Now that any imported TypeVar instances have been resolved, adjust type
# parameters in classes and functions.
module.ast = visitors.AdjustTypeParameters(module.ast)
# Now we can fill in internal cls pointers to ClassType nodes in the
# module. This code executes when the module is first loaded, which
# happens before any others use it to resolve dependencies, so there are
# no external pointers into the module at this point.
module.ast.Visit(
visitors.FillInModuleClasses({
"": module.ast,
module_name: module.ast
}))
# TODO(rechen): Once generics are supported in inline type annotations, a
# VerifyContainers check should also be done on the final ast.
module.ast.Visit(visitors.VerifyContainers())
except:
del self._modules[
module_name] # don't leave half-resolved modules around
raise
return module.ast
def PrepareForExport(module_name, python_version, ast, loader):
"""Prepare an ast as if it was parsed and loaded.
External dependencies will not be resolved, as the ast generated by this
method is supposed to be exported.
Args:
module_name: The module_name as a string for the returned ast.
python_version: A tuple of (major, minor) python version as string
(see config.python_version).
ast: pytd.TypeDeclUnit, is only used if src is None.
loader: A load_pytd.Loader instance.
Returns:
A pytd.TypeDeclUnit representing the supplied AST as it would look after
being written to a file and parsed.
"""
# This is a workaround for functionality which crept into places it doesn't
# belong. Ideally this would call some transformation Visitors on ast to
# transform it into the same ast we get after parsing and loading (compare
# load_pytd.Loader.load_file). Unfortunately parsing has some special cases,
# e.g. '__init__' return type and '__new__' being a 'staticmethod', which
# need to be moved to visitors before we can do this. Printing an ast also
# applies transformations,
# e.g. visitors.PrintVisitor._FormatContainerContents, which need to move to
# their own visitors so they can be applied without printing.
src = pytd_utils.Print(ast)
ast = pytd_builtins.ParsePyTD(src=src,
module=module_name,
python_version=python_version)
ast = ast.Visit(visitors.LookupBuiltins(loader.builtins, full_names=False))
ast = ast.Visit(visitors.ExpandCompatibleBuiltins(loader.builtins))
ast = ast.Visit(visitors.LookupLocalTypes())
ast = ast.Visit(visitors.AdjustTypeParameters())
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = ast.Visit(visitors.FillInLocalPointers({"": ast, module_name: ast}))
ast = ast.Visit(visitors.CanonicalOrderingVisitor())
ast = ast.Visit(
visitors.ClassTypeToLateType(
ignore=[module_name + ".", "__builtin__.", "typing."]))
return ast
def LinkAgainstSimpleBuiltins(self, ast): ast = ast.Visit(visitors.AdjustTypeParameters()) ast = visitors.LookupClasses(ast, self.mini_builtins) return ast
