def assertTypesMatchPytd(self, ty, pytd_src, version=None):
"""Parses pytd_src and compares with ty."""
# TODO(pludemann): This is a copy of pytd.parse.parser_test_base.Parse()
# TODO(pludemann): Consider using the pytd_tree to call
# assertHasOnlySignatures (or similar) to guard against the
# inferencer adding additional but harmless calls.
pytd_tree = parser.TypeDeclParser(version=version).Parse(
textwrap.dedent(pytd_src))
pytd_tree = pytd_tree.Visit(
visitors.LookupBuiltins(builtins.GetBuiltinsAndTyping()[0]))
pytd_tree = pytd_tree.Visit(
visitors.ClassTypeToNamedType())
pytd_tree = pytd_tree.Visit(
visitors.CanonicalOrderingVisitor(sort_signatures=True))
pytd_tree.Visit(visitors.VerifyVisitor())
ty = ty.Visit(visitors.ClassTypeToNamedType())
ty = ty.Visit(visitors.AdjustSelf(force=True))
ty = ty.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True))
ty.Visit(visitors.VerifyVisitor())
ty_src = pytd.Print(ty) + "\n"
pytd_tree_src = pytd.Print(pytd_tree) + "\n"
log.info("========== result ==========")
_LogLines(log.info, ty_src)
log.info("========== expected ==========")
_LogLines(log.info, pytd_tree_src)
log.info("==============================")
# In the diff output, mark expected with "-" and actual with "+".
# (In other words, display a change from "working" to "broken")
self.assertMultiLineEqual(pytd_tree_src, ty_src)
def assertTypesMatchPytd(self, ty, pytd_src, version=None):
"""Parses pytd_src and compares with ty."""
pytd_tree = parser.parse_string(textwrap.dedent(pytd_src),
python_version=version)
pytd_tree = pytd_tree.Visit(
visitors.LookupBuiltins(builtins.GetBuiltinsAndTyping()[0],
full_names=False))
pytd_tree = pytd_tree.Visit(visitors.LookupLocalTypes())
pytd_tree = pytd_tree.Visit(visitors.ClassTypeToNamedType())
pytd_tree = pytd_tree.Visit(
visitors.CanonicalOrderingVisitor(sort_signatures=True))
pytd_tree.Visit(visitors.VerifyVisitor())
ty = ty.Visit(visitors.ClassTypeToNamedType())
ty = ty.Visit(visitors.AdjustSelf(force=True))
ty = ty.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True))
ty.Visit(visitors.VerifyVisitor())
ty_src = pytd.Print(ty) + "\n"
pytd_tree_src = pytd.Print(pytd_tree) + "\n"
log.info("========== result ==========")
_LogLines(log.info, ty_src)
log.info("========== expected ==========")
_LogLines(log.info, pytd_tree_src)
log.info("==============================")
# In the diff output, mark expected with "-" and actual with "+".
# (In other words, display a change from "working" to "broken")
self.assertMultiLineEqual(pytd_tree_src, ty_src)
def RemoveMutableParameters(ast):
"""Change all mutable parameters in a pytd AST to a non-mutable form."""
ast = ast.Visit(optimize.AbsorbMutableParameters())
ast = ast.Visit(optimize.CombineContainers())
ast = ast.Visit(optimize.MergeTypeParameters())
ast = ast.Visit(visitors.AdjustSelf(force=True))
return ast
def Optimize(node,
builtins=None,
lossy=False,
use_abcs=False,
max_union=7,
remove_mutable=False,
can_do_lookup=True):
"""Optimize a PYTD tree.
Tries to shrink a PYTD tree by applying various optimizations.
Arguments:
node: A pytd node to be optimized. It won't be modified - this function
will return a new node.
builtins: Definitions of all of the external types in node.
lossy: Allow optimizations that change the meaning of the pytd.
use_abcs: Use abstract base classes to represent unions like
e.g. "float or int" as "Real".
max_union: How many types we allow in a union before we simplify
it to just "object".
remove_mutable: Whether to simplify mutable parameters to normal
parameters.
can_do_lookup: True: We're either allowed to try to resolve NamedType
instances in the AST, or the AST is already resolved. False: Skip any
optimizations that would require NamedTypes to be resolved.
Returns:
An optimized node.
"""
node = node.Visit(RemoveDuplicates())
node = node.Visit(SimplifyUnions())
node = node.Visit(CombineReturnsAndExceptions())
node = node.Visit(Factorize())
node = node.Visit(ApplyOptionalArguments())
node = node.Visit(CombineContainers())
node = node.Visit(SimplifyContainers())
if builtins:
superclasses = builtins.Visit(visitors.ExtractSuperClassesByName())
superclasses.update(node.Visit(visitors.ExtractSuperClassesByName()))
if use_abcs:
superclasses.update(abc_hierarchy.GetSuperClasses())
hierarchy = SuperClassHierarchy(superclasses)
node = node.Visit(SimplifyUnionsWithSuperclasses(hierarchy))
if lossy:
node = node.Visit(FindCommonSuperClasses(hierarchy))
if max_union:
node = node.Visit(CollapseLongUnions(max_union))
node = node.Visit(AdjustReturnAndConstantGenericType())
if remove_mutable:
node = node.Visit(AbsorbMutableParameters())
node = node.Visit(CombineContainers())
node = node.Visit(MergeTypeParameters())
node = node.Visit(visitors.AdjustSelf(force=True))
node = node.Visit(SimplifyContainers())
if builtins and can_do_lookup:
node = visitors.LookupClasses(node, builtins)
node = node.Visit(RemoveInheritedMethods())
node = node.Visit(RemoveRedundantSignatures(hierarchy))
return node
def RemoveMutableParameters(ast):
"""Change all mutable parameters in a pytd AST to a non-mutable form."""
# late import, because optimize uses utils.py.
from pytype.pytd import optimize # pylint: disable=g-import-not-at-top
ast = ast.Visit(optimize.AbsorbMutableParameters())
ast = ast.Visit(optimize.CombineContainers())
ast = ast.Visit(optimize.MergeTypeParameters())
ast = ast.Visit(visitors.AdjustSelf(force=True))
return ast
def VisitClass(self, cls):
"""Visit a class, and change constants to methods where possible."""
new_constants = []
new_methods = list(cls.methods)
for const in cls.constants:
if self._IsSimpleCall(const.type):
c = self._MaybeLookup(const.type)
signatures = c.methods[0].signatures
self._processed_count[c.name] += 1
new_methods.append(pytd.Function(const.name, signatures))
else:
new_constants.append(const) # keep
cls = cls.Replace(constants=tuple(new_constants),
methods=tuple(new_methods))
return cls.Visit(visitors.AdjustSelf(force=True))
def Optimize(node,
lossy=False,
use_abcs=False,
max_union=7,
remove_mutable=False):
"""Optimize a PYTD tree.
Tries to shrink a PYTD tree by applying various optimizations.
Arguments:
node: A pytd node to be optimized. It won't be modified - this function
will return a new node.
lossy: Allow optimizations that change the meaning of the pytd.
use_abcs: Use abstract base classes to represent unions like
e.g. "float or int" as "Real"
max_union: How many types we allow in a union before we simplify
it to just "object".
remove_mutable: Whether to simplify mutable parameters to normal
parameters.
Returns:
An optimized node.
"""
node = node.Visit(RemoveDuplicates())
node = node.Visit(SimplifyUnions())
node = node.Visit(CombineReturnsAndExceptions())
node = node.Visit(Factorize())
node = node.Visit(ApplyOptionalArguments())
node = node.Visit(CombineContainers())
if lossy:
hierarchy = node.Visit(visitors.ExtractSuperClassesByName())
node = node.Visit(FindCommonSuperClasses(hierarchy, use_abcs))
if max_union:
node = node.Visit(CollapseLongParameterUnions(max_union))
node = node.Visit(CollapseLongReturnUnions(max_union))
node = node.Visit(CollapseLongConstantUnions(max_union))
if remove_mutable:
node = node.Visit(AbsorbMutableParameters())
node = node.Visit(CombineContainers())
node = node.Visit(MergeTypeParameters())
node = node.Visit(visitors.AdjustSelf(force=True))
node = visitors.LookupClasses(node, builtins.GetBuiltinsPyTD())
node = node.Visit(RemoveInheritedMethods())
return node
def VisitClass(self, cls):
"""Add superclass methods and constants to this Class."""
if any(base for base in cls.parents
if isinstance(base, pytd.NamedType)):
raise AssertionError("AddInheritedMethods needs a resolved AST")
# Filter out only the types we can reason about.
# TODO(kramm): Do we want handle UnionTypes and GenericTypes at some point?
bases = [
base.cls for base in cls.parents
if isinstance(base, pytd.ClassType)
]
# Don't pull in methods that are named the same as existing methods in
# this class, local methods override parent class methods.
names = {m.name for m in cls.methods} | {c.name for c in cls.constants}
# TODO(kramm): This should do full-blown MRO.
new_methods = cls.methods + tuple(
m for base in bases for m in base.methods if m.name not in names)
new_constants = cls.constants + tuple(
c for base in bases for c in base.constants if c.name not in names)
cls = cls.Replace(methods=new_methods, constants=new_constants)
return cls.Visit(visitors.AdjustSelf(force=True))
def p_classdef(self, p):
"""classdef : CLASS NAME class_parents COLON maybe_class_funcs end_class"""
_, _, name, (template, parents), _, class_funcs, _ = p
methoddefs = [x for x in class_funcs if isinstance(x, NameAndSig)]
constants = [x for x in class_funcs if isinstance(x, pytd.Constant)]
if (set(f.name for f in methoddefs) | set(c.name for c in constants) !=
set(d.name for d in class_funcs)):
# TODO(kramm): raise a syntax error right when the identifier is defined.
raise make_syntax_error(self, 'Duplicate identifier(s)', p)
template_names = {t.name for t in template}
for _, sig, _ in methoddefs:
for t in sig.template:
if t.name in template_names:
raise make_syntax_error(
self, 'Duplicate template parameter %s' % t.name, p)
cls = pytd.Class(name=name,
parents=parents,
methods=tuple(self.MergeSignatures(methoddefs)),
constants=tuple(constants),
template=template)
p[0] = cls.Visit(visitors.AdjustSelf())
