def assertTypesMatchPytd(self, ty, pytd_src, version=None): """Parses pytd_src and compares with ty.""" # TODO(pludemann): This is a copy of pytd.parse.parser_test_base.Parse() # TODO(pludemann): Consider using the pytd_tree to call # assertHasOnlySignatures (or similar) to guard against the # inferencer adding additional but harmless calls. pytd_tree = parser.TypeDeclParser(version=version).Parse( textwrap.dedent(pytd_src)) pytd_tree = pytd_tree.Visit( visitors.LookupBuiltins(builtins.GetBuiltinsAndTyping()[0])) pytd_tree = pytd_tree.Visit( visitors.ClassTypeToNamedType()) pytd_tree = pytd_tree.Visit( visitors.CanonicalOrderingVisitor(sort_signatures=True)) pytd_tree.Visit(visitors.VerifyVisitor()) ty = ty.Visit(visitors.ClassTypeToNamedType()) ty = ty.Visit(visitors.AdjustSelf(force=True)) ty = ty.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True)) ty.Visit(visitors.VerifyVisitor()) ty_src = pytd.Print(ty) + "\n" pytd_tree_src = pytd.Print(pytd_tree) + "\n" log.info("========== result ==========") _LogLines(log.info, ty_src) log.info("========== expected ==========") _LogLines(log.info, pytd_tree_src) log.info("==============================") # In the diff output, mark expected with "-" and actual with "+". # (In other words, display a change from "working" to "broken") self.assertMultiLineEqual(pytd_tree_src, ty_src)
def assertTypesMatchPytd(self, ty, pytd_src, version=None): """Parses pytd_src and compares with ty.""" pytd_tree = parser.parse_string(textwrap.dedent(pytd_src), python_version=version) pytd_tree = pytd_tree.Visit( visitors.LookupBuiltins(builtins.GetBuiltinsAndTyping()[0], full_names=False)) pytd_tree = pytd_tree.Visit(visitors.LookupLocalTypes()) pytd_tree = pytd_tree.Visit(visitors.ClassTypeToNamedType()) pytd_tree = pytd_tree.Visit( visitors.CanonicalOrderingVisitor(sort_signatures=True)) pytd_tree.Visit(visitors.VerifyVisitor()) ty = ty.Visit(visitors.ClassTypeToNamedType()) ty = ty.Visit(visitors.AdjustSelf(force=True)) ty = ty.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True)) ty.Visit(visitors.VerifyVisitor()) ty_src = pytd.Print(ty) + "\n" pytd_tree_src = pytd.Print(pytd_tree) + "\n" log.info("========== result ==========") _LogLines(log.info, ty_src) log.info("========== expected ==========") _LogLines(log.info, pytd_tree_src) log.info("==============================") # In the diff output, mark expected with "-" and actual with "+". # (In other words, display a change from "working" to "broken") self.assertMultiLineEqual(pytd_tree_src, ty_src)
def RemoveMutableParameters(ast): """Change all mutable parameters in a pytd AST to a non-mutable form.""" ast = ast.Visit(optimize.AbsorbMutableParameters()) ast = ast.Visit(optimize.CombineContainers()) ast = ast.Visit(optimize.MergeTypeParameters()) ast = ast.Visit(visitors.AdjustSelf(force=True)) return ast
def Optimize(node, builtins=None, lossy=False, use_abcs=False, max_union=7, remove_mutable=False, can_do_lookup=True): """Optimize a PYTD tree. Tries to shrink a PYTD tree by applying various optimizations. Arguments: node: A pytd node to be optimized. It won't be modified - this function will return a new node. builtins: Definitions of all of the external types in node. lossy: Allow optimizations that change the meaning of the pytd. use_abcs: Use abstract base classes to represent unions like e.g. "float or int" as "Real". max_union: How many types we allow in a union before we simplify it to just "object". remove_mutable: Whether to simplify mutable parameters to normal parameters. can_do_lookup: True: We're either allowed to try to resolve NamedType instances in the AST, or the AST is already resolved. False: Skip any optimizations that would require NamedTypes to be resolved. Returns: An optimized node. """ node = node.Visit(RemoveDuplicates()) node = node.Visit(SimplifyUnions()) node = node.Visit(CombineReturnsAndExceptions()) node = node.Visit(Factorize()) node = node.Visit(ApplyOptionalArguments()) node = node.Visit(CombineContainers()) node = node.Visit(SimplifyContainers()) if builtins: superclasses = builtins.Visit(visitors.ExtractSuperClassesByName()) superclasses.update(node.Visit(visitors.ExtractSuperClassesByName())) if use_abcs: superclasses.update(abc_hierarchy.GetSuperClasses()) hierarchy = SuperClassHierarchy(superclasses) node = node.Visit(SimplifyUnionsWithSuperclasses(hierarchy)) if lossy: node = node.Visit(FindCommonSuperClasses(hierarchy)) if max_union: node = node.Visit(CollapseLongUnions(max_union)) node = node.Visit(AdjustReturnAndConstantGenericType()) if remove_mutable: node = node.Visit(AbsorbMutableParameters()) node = node.Visit(CombineContainers()) node = node.Visit(MergeTypeParameters()) node = node.Visit(visitors.AdjustSelf(force=True)) node = node.Visit(SimplifyContainers()) if builtins and can_do_lookup: node = visitors.LookupClasses(node, builtins) node = node.Visit(RemoveInheritedMethods()) node = node.Visit(RemoveRedundantSignatures(hierarchy)) return node
def RemoveMutableParameters(ast): """Change all mutable parameters in a pytd AST to a non-mutable form.""" # late import, because optimize uses utils.py. from pytype.pytd import optimize # pylint: disable=g-import-not-at-top ast = ast.Visit(optimize.AbsorbMutableParameters()) ast = ast.Visit(optimize.CombineContainers()) ast = ast.Visit(optimize.MergeTypeParameters()) ast = ast.Visit(visitors.AdjustSelf(force=True)) return ast
def VisitClass(self, cls): """Visit a class, and change constants to methods where possible.""" new_constants = [] new_methods = list(cls.methods) for const in cls.constants: if self._IsSimpleCall(const.type): c = self._MaybeLookup(const.type) signatures = c.methods[0].signatures self._processed_count[c.name] += 1 new_methods.append(pytd.Function(const.name, signatures)) else: new_constants.append(const) # keep cls = cls.Replace(constants=tuple(new_constants), methods=tuple(new_methods)) return cls.Visit(visitors.AdjustSelf(force=True))
def Optimize(node, lossy=False, use_abcs=False, max_union=7, remove_mutable=False): """Optimize a PYTD tree. Tries to shrink a PYTD tree by applying various optimizations. Arguments: node: A pytd node to be optimized. It won't be modified - this function will return a new node. lossy: Allow optimizations that change the meaning of the pytd. use_abcs: Use abstract base classes to represent unions like e.g. "float or int" as "Real" max_union: How many types we allow in a union before we simplify it to just "object". remove_mutable: Whether to simplify mutable parameters to normal parameters. Returns: An optimized node. """ node = node.Visit(RemoveDuplicates()) node = node.Visit(SimplifyUnions()) node = node.Visit(CombineReturnsAndExceptions()) node = node.Visit(Factorize()) node = node.Visit(ApplyOptionalArguments()) node = node.Visit(CombineContainers()) if lossy: hierarchy = node.Visit(visitors.ExtractSuperClassesByName()) node = node.Visit(FindCommonSuperClasses(hierarchy, use_abcs)) if max_union: node = node.Visit(CollapseLongParameterUnions(max_union)) node = node.Visit(CollapseLongReturnUnions(max_union)) node = node.Visit(CollapseLongConstantUnions(max_union)) if remove_mutable: node = node.Visit(AbsorbMutableParameters()) node = node.Visit(CombineContainers()) node = node.Visit(MergeTypeParameters()) node = node.Visit(visitors.AdjustSelf(force=True)) node = visitors.LookupClasses(node, builtins.GetBuiltinsPyTD()) node = node.Visit(RemoveInheritedMethods()) return node
def VisitClass(self, cls): """Add superclass methods and constants to this Class.""" if any(base for base in cls.parents if isinstance(base, pytd.NamedType)): raise AssertionError("AddInheritedMethods needs a resolved AST") # Filter out only the types we can reason about. # TODO(kramm): Do we want handle UnionTypes and GenericTypes at some point? bases = [ base.cls for base in cls.parents if isinstance(base, pytd.ClassType) ] # Don't pull in methods that are named the same as existing methods in # this class, local methods override parent class methods. names = {m.name for m in cls.methods} | {c.name for c in cls.constants} # TODO(kramm): This should do full-blown MRO. new_methods = cls.methods + tuple( m for base in bases for m in base.methods if m.name not in names) new_constants = cls.constants + tuple( c for base in bases for c in base.constants if c.name not in names) cls = cls.Replace(methods=new_methods, constants=new_constants) return cls.Visit(visitors.AdjustSelf(force=True))
def p_classdef(self, p): """classdef : CLASS NAME class_parents COLON maybe_class_funcs end_class""" _, _, name, (template, parents), _, class_funcs, _ = p methoddefs = [x for x in class_funcs if isinstance(x, NameAndSig)] constants = [x for x in class_funcs if isinstance(x, pytd.Constant)] if (set(f.name for f in methoddefs) | set(c.name for c in constants) != set(d.name for d in class_funcs)): # TODO(kramm): raise a syntax error right when the identifier is defined. raise make_syntax_error(self, 'Duplicate identifier(s)', p) template_names = {t.name for t in template} for _, sig, _ in methoddefs: for t in sig.template: if t.name in template_names: raise make_syntax_error( self, 'Duplicate template parameter %s' % t.name, p) cls = pytd.Class(name=name, parents=parents, methods=tuple(self.MergeSignatures(methoddefs)), constants=tuple(constants), template=template) p[0] = cls.Visit(visitors.AdjustSelf())