def PicklePyi(self, src, module_name):
src = textwrap.dedent(src)
ast = parser.parse_string(
src, options=parser.PyiOptions.from_toplevel_options(self.options))
ast = ast.Visit(
visitors.LookupBuiltins(self.loader.builtins, full_names=False))
return self._Pickle(ast, module_name)
def resolve_builtin_types(self, mod_ast, *, lookup_ast=None):
bltn_lookup = visitors.LookupBuiltins(self.builtins_ast,
full_names=False)
mod_ast = self._lookup(bltn_lookup, mod_ast, lookup_ast)
mod_ast = mod_ast.Visit(
visitors.ExpandCompatibleBuiltins(self.builtins_ast))
return mod_ast
def assertTypesMatchPytd(self, ty, pytd_src):
"""Parses pytd_src and compares with ty."""
pytd_tree = parser.parse_string(
textwrap.dedent(pytd_src),
options=parser.PyiOptions(python_version=self.python_version))
pytd_tree = pytd_tree.Visit(
visitors.LookupBuiltins(self.loader.builtins, full_names=False))
pytd_tree = pytd_tree.Visit(visitors.LookupLocalTypes())
pytd_tree = pytd_tree.Visit(visitors.ClassTypeToNamedType())
pytd_tree = pytd_tree.Visit(
visitors.CanonicalOrderingVisitor(sort_signatures=True))
pytd_tree.Visit(visitors.VerifyVisitor())
ty = ty.Visit(visitors.ClassTypeToNamedType())
ty = ty.Visit(visitors.AdjustSelf())
ty = ty.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True))
ty.Visit(visitors.VerifyVisitor())
ty_src = pytd_utils.Print(ty) + "\n"
pytd_tree_src = pytd_utils.Print(pytd_tree) + "\n"
log.info("========== result ==========")
_LogLines(log.info, ty_src)
log.info("========== expected ==========")
_LogLines(log.info, pytd_tree_src)
log.info("==============================")
# In the diff output, mark expected with "-" and actual with "+".
# (In other words, display a change from "working" to "broken")
self.assertMultiLineEqual(pytd_tree_src, ty_src)
def load(self, options):
"""Read builtins.pytd and typing.pytd, and return the parsed modules."""
t = self._parse_predefined("typing", options)
b = self._parse_predefined("builtins", options)
b = b.Visit(
visitors.LookupExternalTypes({"typing": t}, self_name="builtins"))
t = t.Visit(visitors.LookupBuiltins(b))
b = b.Visit(visitors.NamedTypeToClassType())
t = t.Visit(visitors.NamedTypeToClassType())
b = b.Visit(visitors.AdjustTypeParameters())
t = t.Visit(visitors.AdjustTypeParameters())
b = b.Visit(visitors.CanonicalOrderingVisitor())
t = t.Visit(visitors.CanonicalOrderingVisitor())
b.Visit(
visitors.FillInLocalPointers({
"": b,
"typing": t,
"builtins": b
}))
t.Visit(
visitors.FillInLocalPointers({
"": t,
"typing": t,
"builtins": b
}))
b.Visit(visitors.VerifyLookup())
t.Visit(visitors.VerifyLookup())
b.Visit(visitors.VerifyContainers())
t.Visit(visitors.VerifyContainers())
return b, t
def testExpandCompatibleBuiltins(self):
src = textwrap.dedent("""
from typing import Tuple, Union
def f1(a: float) -> None: ...
def f2() -> float: ...
def f3(a: bool) -> None: ...
def f4() -> bool: ...
def f5(a: Union[bool, int]) -> None: ...
def f6(a: Tuple[bool, int]) -> None: ...
""")
expected = textwrap.dedent("""
from typing import Tuple, Union
def f1(a: Union[float, int]) -> None: ...
def f2() -> float: ...
def f3(a: Union[bool, None]) -> None: ...
def f4() -> bool: ...
def f5(a: Union[bool, None, int]) -> None: ...
def f6(a: Tuple[Union[bool, None], int]) -> None: ...
""")
src_tree, expected_tree = (self.Parse(s).Visit(
visitors.LookupBuiltins(self.loader.builtins))
for s in (src, expected))
new_tree = src_tree.Visit(
visitors.ExpandCompatibleBuiltins(self.loader.builtins))
self.AssertSourceEquals(new_tree, expected_tree)
def _resolve_builtins(self, pyval, ast=None):
builtins_lookup = visitors.LookupBuiltins(self.builtins, full_names=False)
if ast:
builtins_lookup.EnterTypeDeclUnit(ast)
pyval = pyval.Visit(builtins_lookup)
pyval = pyval.Visit(visitors.ExpandCompatibleBuiltins(self.builtins))
return pyval
def _postprocess_pyi(self, ast):
"""Apply all the PYI transformations we need."""
ast = ast.Visit(visitors.LookupBuiltins(self.builtins, full_names=False))
ast = ast.Visit(visitors.ExpandCompatibleBuiltins(self.builtins))
dependencies = self._collect_ast_dependencies(ast)
if dependencies:
self._load_ast_dependencies(dependencies, ast)
ast = self._resolve_external_types(ast)
ast = ast.Visit(visitors.LookupLocalTypes())
return ast
def test_lookup_constant(self):
src1 = textwrap.dedent("""
Foo = ... # type: type
""")
src2 = textwrap.dedent("""
class Bar:
bar = ... # type: foo.Foo
""")
ast1 = self.Parse(src1, name="foo").Visit(
visitors.LookupBuiltins(self.loader.builtins))
ast2 = self.Parse(src2, name="bar")
ast2 = ast2.Visit(visitors.LookupExternalTypes({"foo": ast1, "bar": ast2}))
self.assertEqual(ast2.Lookup("bar.Bar").constants[0],
pytd.Constant(name="bar", type=pytd.AnythingType()))
def _postprocess_pyi(self, ast):
"""Apply all the PYI transformations we need."""
package_name = module_utils.get_package_name(ast.name, ast.is_package)
if package_name:
ast = ast.Visit(visitors.QualifyRelativeNames(package_name))
ast = ast.Visit(
visitors.LookupBuiltins(self.builtins, full_names=False))
ast = ast.Visit(visitors.ExpandCompatibleBuiltins(self.builtins))
dependencies = self._collect_ast_dependencies(ast)
if dependencies:
self._load_ast_dependencies(dependencies, ast)
ast = self._resolve_external_types(ast)
ast = ast.Visit(visitors.LookupLocalTypes())
return ast
def GetBuiltinsAndTyping(python_version): # Deprecated. Use load_pytd instead.
"""Get __builtin__.pytd and typing.pytd."""
assert python_version
global _cached_builtins_pytd
if _cached_builtins_pytd.cache:
assert _cached_builtins_pytd.version == python_version
else:
t = parser.parse_string(_FindBuiltinFile("typing", python_version),
name="typing",
python_version=python_version)
b = parser.parse_string(_FindBuiltinFile("__builtin__",
python_version),
name="__builtin__",
python_version=python_version)
b = b.Visit(
visitors.LookupExternalTypes({"typing": t},
full_names=True,
self_name="__builtin__"))
t = t.Visit(visitors.LookupBuiltins(b))
b = b.Visit(visitors.NamedTypeToClassType())
t = t.Visit(visitors.NamedTypeToClassType())
b = b.Visit(visitors.AdjustTypeParameters())
t = t.Visit(visitors.AdjustTypeParameters())
b = b.Visit(visitors.CanonicalOrderingVisitor())
t = t.Visit(visitors.CanonicalOrderingVisitor())
b.Visit(
visitors.FillInLocalPointers({
"": b,
"typing": t,
"__builtin__": b
}))
t.Visit(
visitors.FillInLocalPointers({
"": t,
"typing": t,
"__builtin__": b
}))
b.Visit(visitors.VerifyLookup())
t.Visit(visitors.VerifyLookup())
b.Visit(visitors.VerifyContainers())
t.Visit(visitors.VerifyContainers())
_cached_builtins_pytd = Cache(python_version, (b, t))
return _cached_builtins_pytd.cache
def PrepareForExport(module_name, python_version, ast, loader):
"""Prepare an ast as if it was parsed and loaded.
External dependencies will not be resolved, as the ast generated by this
method is supposed to be exported.
Args:
module_name: The module_name as a string for the returned ast.
python_version: A tuple of (major, minor) python version as string
(see config.python_version).
ast: pytd.TypeDeclUnit, is only used if src is None.
loader: A load_pytd.Loader instance.
Returns:
A pytd.TypeDeclUnit representing the supplied AST as it would look after
being written to a file and parsed.
"""
# This is a workaround for functionality which crept into places it doesn't
# belong. Ideally this would call some transformation Visitors on ast to
# transform it into the same ast we get after parsing and loading (compare
# load_pytd.Loader.load_file). Unfortunately parsing has some special cases,
# e.g. '__init__' return type and '__new__' being a 'staticmethod', which
# need to be moved to visitors before we can do this. Printing an ast also
# applies transformations,
# e.g. visitors.PrintVisitor._FormatContainerContents, which need to move to
# their own visitors so they can be applied without printing.
src = pytd_utils.Print(ast)
ast = parser.parse_string(src=src,
name=module_name,
python_version=python_version)
ast = ast.Visit(visitors.LookupBuiltins(loader.builtins, full_names=False))
ast = ast.Visit(
visitors.ExpandCompatibleBuiltins(loader.builtins, python_version))
ast = ast.Visit(visitors.LookupLocalTypes())
ast = ast.Visit(visitors.AdjustTypeParameters())
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = ast.Visit(visitors.FillInLocalPointers({"": ast, module_name: ast}))
ast = ast.Visit(visitors.CanonicalOrderingVisitor())
ast = ast.Visit(
visitors.ClassTypeToLateType(
ignore=[module_name + ".", "__builtin__.", "typing."]))
return ast
def test_literal(self):
ast = self.Parse("""
from typing import Literal
x1: Literal[""]
x2: Literal[b""]
x3: Literal[0]
x4: Literal[True]
x5: Literal[None]
""")
ast = ast.Visit(visitors.LookupBuiltins(self.loader.builtins))
self.assertMultiLineEqual(
pytd_utils.Print(ast),
textwrap.dedent("""
from typing import Literal
x1: Literal['']
x2: Literal[b'']
x3: Literal[0]
x4: Literal[True]
x5: None
""").strip())
def GetBuiltinsAndTyping(python_version): # Deprecated. Use load_pytd instead.
"""Get builtins.pytd and typing.pytd."""
assert python_version
if python_version not in _cached_builtins_pytd:
t = parser.parse_string(_FindBuiltinFile("typing", python_version),
name="typing",
python_version=python_version)
b = parser.parse_string(_FindBuiltinFile("builtins", python_version),
name="builtins",
python_version=python_version)
b = b.Visit(
visitors.LookupExternalTypes({"typing": t}, self_name="builtins"))
t = t.Visit(visitors.LookupBuiltins(b))
b = b.Visit(visitors.NamedTypeToClassType())
t = t.Visit(visitors.NamedTypeToClassType())
b = b.Visit(visitors.AdjustTypeParameters())
t = t.Visit(visitors.AdjustTypeParameters())
b = b.Visit(visitors.CanonicalOrderingVisitor())
t = t.Visit(visitors.CanonicalOrderingVisitor())
b.Visit(
visitors.FillInLocalPointers({
"": b,
"typing": t,
"builtins": b
}))
t.Visit(
visitors.FillInLocalPointers({
"": t,
"typing": t,
"builtins": b
}))
b.Visit(visitors.VerifyLookup())
t.Visit(visitors.VerifyLookup())
b.Visit(visitors.VerifyContainers())
t.Visit(visitors.VerifyContainers())
_cached_builtins_pytd[python_version] = (b, t)
return _cached_builtins_pytd[python_version]
def ParseAndResolve(self, src):
ast = self.Parse(src)
return ast.Visit(visitors.LookupBuiltins(self.builtins))
def PicklePyi(self, src, module_name):
src = textwrap.dedent(src)
ast = parser.parse_string(src, python_version=self.python_version)
ast = ast.Visit(
visitors.LookupBuiltins(self.loader.builtins, full_names=False))
return self._Pickle(ast, module_name)
def _resolve_builtins(self, ast):
ast = ast.Visit(
visitors.LookupBuiltins(self.builtins, full_names=False))
ast = ast.Visit(visitors.ExpandCompatibleBuiltins(self.builtins))
return ast
def parse(self, src):
ast = parser.parse_string(textwrap.dedent(src))
ast = ast.Visit(visitors.LookupBuiltins(self.builtins_pytd))
return ast
