def assertTypesMatchPytd(self, ty, pytd_src, version=None):
"""Parses pytd_src and compares with ty."""
# TODO(pludemann): This is a copy of pytd.parse.parser_test_base.Parse()
# TODO(pludemann): Consider using the pytd_tree to call
# assertHasOnlySignatures (or similar) to guard against the
# inferencer adding additional but harmless calls.
pytd_tree = parser.TypeDeclParser(version=version).Parse(
textwrap.dedent(pytd_src))
pytd_tree = pytd_tree.Visit(
visitors.LookupBuiltins(builtins.GetBuiltinsAndTyping()[0]))
pytd_tree = pytd_tree.Visit(
visitors.ClassTypeToNamedType())
pytd_tree = pytd_tree.Visit(
visitors.CanonicalOrderingVisitor(sort_signatures=True))
pytd_tree.Visit(visitors.VerifyVisitor())
ty = ty.Visit(visitors.ClassTypeToNamedType())
ty = ty.Visit(visitors.AdjustSelf(force=True))
ty = ty.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True))
ty.Visit(visitors.VerifyVisitor())
ty_src = pytd.Print(ty) + "\n"
pytd_tree_src = pytd.Print(pytd_tree) + "\n"
log.info("========== result ==========")
_LogLines(log.info, ty_src)
log.info("========== expected ==========")
_LogLines(log.info, pytd_tree_src)
log.info("==============================")
# In the diff output, mark expected with "-" and actual with "+".
# (In other words, display a change from "working" to "broken")
self.assertMultiLineEqual(pytd_tree_src, ty_src)
def assertTypesMatchPytd(self, ty, pytd_src, version=None):
"""Parses pytd_src and compares with ty."""
pytd_tree = parser.parse_string(textwrap.dedent(pytd_src),
python_version=version)
pytd_tree = pytd_tree.Visit(
visitors.LookupBuiltins(builtins.GetBuiltinsAndTyping()[0],
full_names=False))
pytd_tree = pytd_tree.Visit(visitors.LookupLocalTypes())
pytd_tree = pytd_tree.Visit(visitors.ClassTypeToNamedType())
pytd_tree = pytd_tree.Visit(
visitors.CanonicalOrderingVisitor(sort_signatures=True))
pytd_tree.Visit(visitors.VerifyVisitor())
ty = ty.Visit(visitors.ClassTypeToNamedType())
ty = ty.Visit(visitors.AdjustSelf(force=True))
ty = ty.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True))
ty.Visit(visitors.VerifyVisitor())
ty_src = pytd.Print(ty) + "\n"
pytd_tree_src = pytd.Print(pytd_tree) + "\n"
log.info("========== result ==========")
_LogLines(log.info, ty_src)
log.info("========== expected ==========")
_LogLines(log.info, pytd_tree_src)
log.info("==============================")
# In the diff output, mark expected with "-" and actual with "+".
# (In other words, display a change from "working" to "broken")
self.assertMultiLineEqual(pytd_tree_src, ty_src)
def testCanonicalOrderingVisitor(self):
src1 = textwrap.dedent("""
def f() -> ? raises MemoryError, IOError
def f(x: list[a]) -> ?
def f(x: list[b or c]) -> ?
def f(x: list[tuple[d]]) -> ?
""")
src2 = textwrap.dedent("""
def f() -> ? raises IOError, MemoryError
def f(x: list[tuple[d]]) -> ?
def f(x: list[a]) -> ?
def f(x: list[b or c]) -> ?
""")
tree1 = self.Parse(src1)
tree1 = tree1.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True))
tree2 = self.Parse(src2)
tree2 = tree2.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True))
self.AssertSourceEquals(tree1, tree2)
def GetBuiltinsAndTyping(python_version):
"""Get __builtin__.pytd and typing.pytd."""
assert python_version
global _cached_builtins_pytd
if _cached_builtins_pytd.cache:
assert _cached_builtins_pytd.version == python_version
else:
t = parser.parse_string(_FindBuiltinFile("typing", python_version),
name="typing",
python_version=python_version)
b = parser.parse_string(_FindBuiltinFile("__builtin__",
python_version),
name="__builtin__",
python_version=python_version)
b = b.Visit(
visitors.LookupExternalTypes({"typing": t},
full_names=True,
self_name="__builtin__"))
t = t.Visit(visitors.LookupBuiltins(b))
b = b.Visit(visitors.NamedTypeToClassType())
t = t.Visit(visitors.NamedTypeToClassType())
b = b.Visit(visitors.AdjustTypeParameters())
t = t.Visit(visitors.AdjustTypeParameters())
b = b.Visit(visitors.CanonicalOrderingVisitor())
t = t.Visit(visitors.CanonicalOrderingVisitor())
b.Visit(
visitors.FillInLocalPointers({
"": b,
"typing": t,
"__builtin__": b
}))
t.Visit(
visitors.FillInLocalPointers({
"": t,
"typing": t,
"__builtin__": b
}))
b.Visit(visitors.VerifyLookup())
t.Visit(visitors.VerifyLookup())
b.Visit(visitors.VerifyContainers())
t.Visit(visitors.VerifyContainers())
_cached_builtins_pytd = Cache(python_version, (b, t))
return _cached_builtins_pytd.cache
def testCanonicalOrderingVisitor(self):
src1 = textwrap.dedent("""
from typing import TypeVar
def f() -> ?:
raise MemoryError()
raise IOError()
def f(x: list[a]) -> ?
def f(x: list[b or c]) -> ?
def f(x: list[tuple[d]]) -> ?
A = TypeVar("A")
C = TypeVar("C")
B = TypeVar("B")
D = TypeVar("D")
def f(d: A, c: B, b: C, a: D) -> ?
""")
src2 = textwrap.dedent("""
def f() -> ?:
raise IOError()
raise MemoryError()
def f(x: list[tuple[d]]) -> ?
def f(x: list[a]) -> ?
def f(x: list[b or c]) -> ?
A = TypeVar("A")
C = TypeVar("C")
B = TypeVar("B")
D = TypeVar("D")
def f(d: A, c: B, b: C, a: D) -> ?
""")
tree1 = self.Parse(src1)
tree1 = tree1.Visit(
visitors.CanonicalOrderingVisitor(sort_signatures=True))
tree2 = self.Parse(src2)
tree2 = tree2.Visit(
visitors.CanonicalOrderingVisitor(sort_signatures=True))
self.AssertSourceEquals(tree1, tree2)
self.assertEqual(
tree1.Lookup("f").signatures[0].template,
tree2.Lookup("f").signatures[0].template)
def GetBuiltinsAndTyping():
"""Get __builtin__.pytd and typing.pytd."""
global _cached_builtins_pytd
if not _cached_builtins_pytd:
t = parser.parse_string(_FindBuiltinFile("typing"), name="typing")
b = parser.parse_string(_FindBuiltinFile("__builtin__"),
name="__builtin__")
b = b.Visit(
visitors.LookupExternalTypes({"typing": t},
full_names=True,
self_name="__builtin__"))
t = t.Visit(visitors.LookupBuiltins(b))
b = b.Visit(visitors.NamedTypeToClassType())
t = t.Visit(visitors.NamedTypeToClassType())
b = b.Visit(visitors.AdjustTypeParameters())
t = t.Visit(visitors.AdjustTypeParameters())
b = b.Visit(visitors.CanonicalOrderingVisitor())
t = t.Visit(visitors.CanonicalOrderingVisitor())
b.Visit(
visitors.FillInLocalPointers({
"": b,
"typing": t,
"__builtin__": b
}))
t.Visit(
visitors.FillInLocalPointers({
"": t,
"typing": t,
"__builtin__": b
}))
b.Visit(visitors.VerifyLookup())
t.Visit(visitors.VerifyLookup())
b.Visit(visitors.VerifyContainers())
t.Visit(visitors.VerifyContainers())
_cached_builtins_pytd = b, t
return _cached_builtins_pytd
def testBuiltinSuperClasses(self):
src = textwrap.dedent("""
def f(x: list or object, y: complex or memoryview) -> int or bool
""")
expected = textwrap.dedent("""
def f(x, y) -> int
""")
b = builtins.GetBuiltinsPyTD(self.PYTHON_VERSION)
hierarchy = b.Visit(visitors.ExtractSuperClassesByName())
visitor = optimize.FindCommonSuperClasses(
optimize.SuperClassHierarchy(hierarchy))
ast = self.ParseAndResolve(src)
ast = ast.Visit(visitor)
ast = ast.Visit(visitors.DropBuiltinPrefix())
ast = ast.Visit(visitors.CanonicalOrderingVisitor())
self.AssertSourceEquals(ast, expected)
def PrepareForExport(module_name, python_version, ast, loader):
"""Prepare an ast as if it was parsed and loaded.
External dependencies will not be resolved, as the ast generated by this
method is supposed to be exported.
Args:
module_name: The module_name as a string for the returned ast.
python_version: A tuple of (major, minor) python version as string
(see config.python_version).
ast: pytd.TypeDeclUnit, is only used if src is None.
loader: A load_pytd.Loader instance.
Returns:
A pytd.TypeDeclUnit representing the supplied AST as it would look after
being written to a file and parsed.
"""
# This is a workaround for functionality which crept into places it doesn't
# belong. Ideally this would call some transformation Visitors on ast to
# transform it into the same ast we get after parsing and loading (compare
# load_pytd.Loader.load_file). Unfortunately parsing has some special cases,
# e.g. '__init__' return type and '__new__' being a 'staticmethod', which
# need to be moved to visitors before we can do this. Printing an ast also
# applies transformations,
# e.g. visitors.PrintVisitor._FormatContainerContents, which need to move to
# their own visitors so they can be applied without printing.
src = pytd_utils.Print(ast)
ast = pytd_builtins.ParsePyTD(src=src,
module=module_name,
python_version=python_version)
ast = ast.Visit(visitors.LookupBuiltins(loader.builtins, full_names=False))
ast = ast.Visit(visitors.ExpandCompatibleBuiltins(loader.builtins))
ast = ast.Visit(visitors.LookupLocalTypes())
ast = ast.Visit(visitors.AdjustTypeParameters())
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = ast.Visit(visitors.FillInLocalPointers({"": ast, module_name: ast}))
ast = ast.Visit(visitors.CanonicalOrderingVisitor())
ast = ast.Visit(
visitors.ClassTypeToLateType(
ignore=[module_name + ".", "__builtin__.", "typing."]))
return ast
def CanonicalOrdering(n, sort_signatures=False):
"""Convert a PYTD node to a canonical (sorted) ordering."""
# TODO(pludemann): use the original .py to decide the ordering rather
# than an arbitrary sort order
return n.Visit(
visitors.CanonicalOrderingVisitor(sort_signatures=sort_signatures))
def canonical_pyi(pyi, python_version): ast = parser.parse_string(pyi, python_version=python_version) ast = ast.Visit(visitors.ClassTypeToNamedType()) ast = ast.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True)) ast.Visit(visitors.VerifyVisitor()) return pytd.Print(ast)
def CanonicalOrdering(n, sort_signatures=False):
"""Convert a PYTD node to a canonical (sorted) ordering."""
return n.Visit(
visitors.CanonicalOrderingVisitor(sort_signatures=sort_signatures))
def canonical_pyi(pyi):
ast = parser.TypeDeclParser().Parse(pyi)
ast = ast.Visit(visitors.ClassTypeToNamedType())
ast = ast.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True))
ast.Visit(visitors.VerifyVisitor())
return pytd.Print(ast)
