def Optimize(node,
builtins=None,
lossy=False,
use_abcs=False,
max_union=7,
remove_mutable=False,
can_do_lookup=True):
"""Optimize a PYTD tree.
Tries to shrink a PYTD tree by applying various optimizations.
Arguments:
node: A pytd node to be optimized. It won't be modified - this function
will return a new node.
builtins: Definitions of all of the external types in node.
lossy: Allow optimizations that change the meaning of the pytd.
use_abcs: Use abstract base classes to represent unions like
e.g. "float or int" as "Real".
max_union: How many types we allow in a union before we simplify
it to just "object".
remove_mutable: Whether to simplify mutable parameters to normal
parameters.
can_do_lookup: True: We're either allowed to try to resolve NamedType
instances in the AST, or the AST is already resolved. False: Skip any
optimizations that would require NamedTypes to be resolved.
Returns:
An optimized node.
"""
node = node.Visit(RemoveDuplicates())
node = node.Visit(SimplifyUnions())
node = node.Visit(CombineReturnsAndExceptions())
node = node.Visit(Factorize())
node = node.Visit(ApplyOptionalArguments())
node = node.Visit(CombineContainers())
node = node.Visit(SimplifyContainers())
if builtins:
superclasses = builtins.Visit(visitors.ExtractSuperClassesByName())
superclasses.update(node.Visit(visitors.ExtractSuperClassesByName()))
if use_abcs:
superclasses.update(abc_hierarchy.GetSuperClasses())
hierarchy = SuperClassHierarchy(superclasses)
node = node.Visit(SimplifyUnionsWithSuperclasses(hierarchy))
if lossy:
node = node.Visit(FindCommonSuperClasses(hierarchy))
if max_union:
node = node.Visit(CollapseLongUnions(max_union))
node = node.Visit(AdjustReturnAndConstantGenericType())
if remove_mutable:
node = node.Visit(AbsorbMutableParameters())
node = node.Visit(CombineContainers())
node = node.Visit(MergeTypeParameters())
node = node.Visit(visitors.AdjustSelf())
node = node.Visit(SimplifyContainers())
if builtins and can_do_lookup:
node = visitors.LookupClasses(node, builtins, ignore_late_types=True)
node = node.Visit(RemoveInheritedMethods())
node = node.Visit(RemoveRedundantSignatures(hierarchy))
return node
def test_builtin_superclasses(self):
src = pytd_src("""
def f(x: Union[list, object], y: Union[complex, memoryview]) -> Union[int, bool]: ...
""")
expected = pytd_src("""
def f(x: builtins.object, y: builtins.object) -> builtins.int: ...
""")
hierarchy = self.builtins.Visit(visitors.ExtractSuperClassesByName())
hierarchy.update(
self.typing.Visit(visitors.ExtractSuperClassesByName()))
visitor = optimize.FindCommonSuperClasses(
optimize.SuperClassHierarchy(hierarchy))
ast = self.ParseAndResolve(src)
ast = ast.Visit(visitor)
ast = ast.Visit(visitors.CanonicalOrderingVisitor())
self.AssertSourceEquals(ast, expected)
def testBuiltinSuperClasses(self):
src = textwrap.dedent("""
def f(x: list or object, y: complex or memoryview) -> int or bool
""")
expected = textwrap.dedent("""
def f(x: object, y: object) -> int
""")
hierarchy = self.builtins.Visit(visitors.ExtractSuperClassesByName())
hierarchy.update(self.typing.Visit(visitors.ExtractSuperClassesByName()))
visitor = optimize.FindCommonSuperClasses(
optimize.SuperClassHierarchy(hierarchy))
ast = self.ParseAndResolve(src)
ast = ast.Visit(visitor)
ast = ast.Visit(visitors.DropBuiltinPrefix())
ast = ast.Visit(visitors.CanonicalOrderingVisitor())
self.AssertSourceEquals(ast, expected)
def testSimplifyUnionsWithSuperclassesGeneric(self):
src = textwrap.dedent("""
x = ... # type: frozenset[int] or AbstractSet[int]
""")
expected = textwrap.dedent("""
x = ... # type: AbstractSet[int]
""")
hierarchy = self.builtins.Visit(visitors.ExtractSuperClassesByName())
visitor = optimize.SimplifyUnionsWithSuperclasses(
optimize.SuperClassHierarchy(hierarchy))
ast = self.Parse(src)
ast = visitors.LookupClasses(ast, self.builtins)
ast = ast.Visit(visitor)
self.AssertSourceEquals(ast, expected)
def test_simplify_unions_with_superclasses_generic(self):
src = pytd_src("""
x = ... # type: Union[frozenset[int], AbstractSet[int]]
""")
expected = pytd_src("""
x = ... # type: AbstractSet[int]
""")
hierarchy = self.builtins.Visit(visitors.ExtractSuperClassesByName())
visitor = optimize.SimplifyUnionsWithSuperclasses(
optimize.SuperClassHierarchy(hierarchy))
ast = self.Parse(src)
ast = visitors.LookupClasses(ast, self.builtins)
ast = ast.Visit(visitor)
self.AssertSourceEquals(ast, expected)
def testFindCommonSuperClasses(self):
src = textwrap.dedent("""
x = ... # type: int or other.Bar
""")
expected = textwrap.dedent("""
x = ... # type: int or other.Bar
""")
ast = self.Parse(src)
ast = ast.Visit(visitors.ReplaceTypes(
{"other.Bar": pytd.LateType("other.Bar")}))
hierarchy = ast.Visit(visitors.ExtractSuperClassesByName())
ast = ast.Visit(optimize.FindCommonSuperClasses(
optimize.SuperClassHierarchy(hierarchy)))
ast = ast.Visit(visitors.LateTypeToClassType())
self.AssertSourceEquals(ast, expected)
def test_find_common_superclasses(self):
src = pytd_src("""
x = ... # type: Union[int, other.Bar]
""")
expected = pytd_src("""
x = ... # type: Union[int, other.Bar]
""")
ast = self.Parse(src)
ast = ast.Visit(
visitors.ReplaceTypes({"other.Bar": pytd.LateType("other.Bar")}))
hierarchy = ast.Visit(visitors.ExtractSuperClassesByName())
ast = ast.Visit(
optimize.FindCommonSuperClasses(
optimize.SuperClassHierarchy(hierarchy)))
ast = ast.Visit(visitors.LateTypeToClassType())
self.AssertSourceEquals(ast, expected)
def testSimplifyUnionsWithSuperclasses(self):
src = textwrap.dedent("""
x = ... # type: int or bool
y = ... # type: int or bool or float
z = ... # type: list[int] or int
""")
expected = textwrap.dedent("""
x = ... # type: int
y = ... # type: int or float
z = ... # type: list[int] or int
""")
hierarchy = self.builtins.Visit(visitors.ExtractSuperClassesByName())
visitor = optimize.SimplifyUnionsWithSuperclasses(
optimize.SuperClassHierarchy(hierarchy))
ast = self.Parse(src)
ast = visitors.LookupClasses(ast, self.builtins)
ast = ast.Visit(visitor)
self.AssertSourceEquals(ast, expected)
def test_simplify_unions_with_superclasses(self):
src = pytd_src("""
x = ... # type: Union[int, bool]
y = ... # type: Union[int, bool, float]
z = ... # type: Union[list[int], int]
""")
expected = pytd_src("""
x = ... # type: int
y = ... # type: Union[int, float]
z = ... # type: Union[list[int], int]
""")
hierarchy = self.builtins.Visit(visitors.ExtractSuperClassesByName())
visitor = optimize.SimplifyUnionsWithSuperclasses(
optimize.SuperClassHierarchy(hierarchy))
ast = self.Parse(src)
ast = visitors.LookupClasses(ast, self.builtins)
ast = ast.Visit(visitor)
self.AssertSourceEquals(ast, expected)
def test_user_superclass_hierarchy(self):
class_data = pytd_src("""
class AB:
pass
class EFG:
pass
class A(AB, EFG):
pass
class B(AB):
pass
class E(EFG, AB):
pass
class F(EFG):
pass
class G(EFG):
pass
""")
src = pytd_src("""
from typing import Any
def f(x: Union[A, B], y: A, z: B) -> Union[E, F, G]: ...
def g(x: Union[E, F, G, B]) -> Union[E, F]: ...
def h(x) -> Any: ...
""") + class_data
expected = pytd_src("""
from typing import Any
def f(x: AB, y: A, z: B) -> EFG: ...
def g(x: object) -> EFG: ...
def h(x) -> Any: ...
""") + class_data
hierarchy = self.Parse(src).Visit(visitors.ExtractSuperClassesByName())
visitor = optimize.FindCommonSuperClasses(
optimize.SuperClassHierarchy(hierarchy))
new_src = self.ApplyVisitorToString(src, visitor)
self.AssertSourceEquals(new_src, expected)
def testUserSuperClassHierarchy(self):
class_data = textwrap.dedent("""
class AB(object):
pass
class EFG(object):
pass
class A(AB, EFG):
pass
class B(AB):
pass
class E(EFG, AB):
pass
class F(EFG):
pass
class G(EFG):
pass
""")
src = textwrap.dedent("""
def f(x: A or B, y: A, z: B) -> E or F or G
def g(x: E or F or G or B) -> E or F
def h(x) -> ?
""") + class_data
expected = textwrap.dedent("""
def f(x: AB, y: A, z: B) -> EFG
def g(x: object) -> EFG
def h(x) -> ?
""") + class_data
hierarchy = self.Parse(src).Visit(
visitors.ExtractSuperClassesByName())
visitor = optimize.FindCommonSuperClasses(
optimize.SuperClassHierarchy(hierarchy))
new_src = self.ApplyVisitorToString(src, visitor)
self.AssertSourceEquals(new_src, expected)
def testSuperClassesByName(self):
src = textwrap.dedent("""
class A():
pass
class B():
pass
class C(A):
pass
class D(A,B):
pass
class E(C,D,A):
pass
""")
tree = self.Parse(src)
data = tree.Visit(visitors.ExtractSuperClassesByName())
six.assertCountEqual(self, ("classobj",), data["A"])
six.assertCountEqual(self, ("classobj",), data["B"])
six.assertCountEqual(self, ("A",), data["C"])
six.assertCountEqual(self, ("A", "B"), data["D"])
six.assertCountEqual(self, ("A", "C", "D"), data["E"])