def assertTypesMatchPytd(self, ty, pytd_src, version=None): """Parses pytd_src and compares with ty.""" # TODO(pludemann): This is a copy of pytd.parse.parser_test_base.Parse() # TODO(pludemann): Consider using the pytd_tree to call # assertHasOnlySignatures (or similar) to guard against the # inferencer adding additional but harmless calls. pytd_tree = parser.TypeDeclParser(version=version).Parse( textwrap.dedent(pytd_src)) pytd_tree = pytd_tree.Visit( visitors.LookupBuiltins(builtins.GetBuiltinsAndTyping()[0])) pytd_tree = pytd_tree.Visit( visitors.ClassTypeToNamedType()) pytd_tree = pytd_tree.Visit( visitors.CanonicalOrderingVisitor(sort_signatures=True)) pytd_tree.Visit(visitors.VerifyVisitor()) ty = ty.Visit(visitors.ClassTypeToNamedType()) ty = ty.Visit(visitors.AdjustSelf(force=True)) ty = ty.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True)) ty.Visit(visitors.VerifyVisitor()) ty_src = pytd.Print(ty) + "\n" pytd_tree_src = pytd.Print(pytd_tree) + "\n" log.info("========== result ==========") _LogLines(log.info, ty_src) log.info("========== expected ==========") _LogLines(log.info, pytd_tree_src) log.info("==============================") # In the diff output, mark expected with "-" and actual with "+". # (In other words, display a change from "working" to "broken") self.assertMultiLineEqual(pytd_tree_src, ty_src)
def _postprocess_pyi(self, ast): """Apply all the PYI transformations we need.""" ast = ast.Visit( visitors.LookupBuiltins(self.builtins, full_names=False)) ast = ast.Visit(visitors.ExpandCompatibleBuiltins(self.builtins)) ast = ast.Visit(visitors.LookupLocalTypes()) return ast
def assertTypesMatchPytd(self, ty, pytd_src, version=None): """Parses pytd_src and compares with ty.""" pytd_tree = parser.parse_string(textwrap.dedent(pytd_src), python_version=version) pytd_tree = pytd_tree.Visit( visitors.LookupBuiltins(builtins.GetBuiltinsAndTyping()[0], full_names=False)) pytd_tree = pytd_tree.Visit(visitors.LookupLocalTypes()) pytd_tree = pytd_tree.Visit(visitors.ClassTypeToNamedType()) pytd_tree = pytd_tree.Visit( visitors.CanonicalOrderingVisitor(sort_signatures=True)) pytd_tree.Visit(visitors.VerifyVisitor()) ty = ty.Visit(visitors.ClassTypeToNamedType()) ty = ty.Visit(visitors.AdjustSelf(force=True)) ty = ty.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True)) ty.Visit(visitors.VerifyVisitor()) ty_src = pytd.Print(ty) + "\n" pytd_tree_src = pytd.Print(pytd_tree) + "\n" log.info("========== result ==========") _LogLines(log.info, ty_src) log.info("========== expected ==========") _LogLines(log.info, pytd_tree_src) log.info("==============================") # In the diff output, mark expected with "-" and actual with "+". # (In other words, display a change from "working" to "broken") self.assertMultiLineEqual(pytd_tree_src, ty_src)
def _postprocess_pyi(self, ast, package_name): """Apply all the PYI transformations we need.""" if package_name is not None: ast = ast.Visit(visitors.QualifyRelativeNames(package_name)) ast = ast.Visit( visitors.LookupBuiltins(self.builtins, full_names=False)) ast = ast.Visit(visitors.ExpandCompatibleBuiltins(self.builtins)) ast = ast.Visit(visitors.LookupLocalTypes()) return ast
def _postprocess_pyi(self, ast): """Apply all the PYI transformations we need.""" package_name = utils.get_pyi_package_name(ast.name, ast.is_package) if package_name: ast = ast.Visit(visitors.QualifyRelativeNames(package_name)) ast = ast.Visit(visitors.LookupBuiltins(self.builtins, full_names=False)) ast = ast.Visit(visitors.ExpandCompatibleBuiltins(self.builtins)) dependencies = self._collect_ast_dependencies(ast) if dependencies: self._load_ast_dependencies(dependencies, ast) ast = self._resolve_external_types(ast) ast = ast.Visit(visitors.LookupLocalTypes()) return ast
def GetBuiltinsAndTyping(python_version): """Get __builtin__.pytd and typing.pytd.""" assert python_version global _cached_builtins_pytd if _cached_builtins_pytd.cache: assert _cached_builtins_pytd.version == python_version else: t = parser.parse_string(_FindBuiltinFile("typing", python_version), name="typing", python_version=python_version) b = parser.parse_string(_FindBuiltinFile("__builtin__", python_version), name="__builtin__", python_version=python_version) b = b.Visit( visitors.LookupExternalTypes({"typing": t}, full_names=True, self_name="__builtin__")) t = t.Visit(visitors.LookupBuiltins(b)) b = b.Visit(visitors.NamedTypeToClassType()) t = t.Visit(visitors.NamedTypeToClassType()) b = b.Visit(visitors.AdjustTypeParameters()) t = t.Visit(visitors.AdjustTypeParameters()) b = b.Visit(visitors.CanonicalOrderingVisitor()) t = t.Visit(visitors.CanonicalOrderingVisitor()) b.Visit( visitors.FillInLocalPointers({ "": b, "typing": t, "__builtin__": b })) t.Visit( visitors.FillInLocalPointers({ "": t, "typing": t, "__builtin__": b })) b.Visit(visitors.VerifyLookup()) t.Visit(visitors.VerifyLookup()) b.Visit(visitors.VerifyContainers()) t.Visit(visitors.VerifyContainers()) _cached_builtins_pytd = Cache(python_version, (b, t)) return _cached_builtins_pytd.cache
def PrepareForExport(module_name, python_version, ast, loader): """Prepare an ast as if it was parsed and loaded. External dependencies will not be resolved, as the ast generated by this method is supposed to be exported. Args: module_name: The module_name as a string for the returned ast. python_version: A tuple of (major, minor) python version as string (see config.python_version). ast: pytd.TypeDeclUnit, is only used if src is None. loader: A load_pytd.Loader instance. Returns: A pytd.TypeDeclUnit representing the supplied AST as it would look after being written to a file and parsed. """ # This is a workaround for functionality which crept into places it doesn't # belong. Ideally this would call some transformation Visitors on ast to # transform it into the same ast we get after parsing and loading (compare # load_pytd.Loader.load_file). Unfortunately parsing has some special cases, # e.g. '__init__' return type and '__new__' being a 'staticmethod', which # need to be moved to visitors before we can do this. Printing an ast also # applies transformations, # e.g. visitors.PrintVisitor._FormatContainerContents, which need to move to # their own visitors so they can be applied without printing. src = pytd_utils.Print(ast) ast = pytd_builtins.ParsePyTD(src=src, module=module_name, python_version=python_version) ast = ast.Visit(visitors.LookupBuiltins(loader.builtins, full_names=False)) ast = ast.Visit(visitors.ExpandCompatibleBuiltins(loader.builtins)) ast = ast.Visit(visitors.LookupLocalTypes()) ast = ast.Visit(visitors.AdjustTypeParameters()) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.FillInLocalPointers({"": ast, module_name: ast})) ast = ast.Visit(visitors.CanonicalOrderingVisitor()) ast = ast.Visit( visitors.ClassTypeToLateType( ignore=[module_name + ".", "__builtin__.", "typing."])) return ast
def testExpandCompatibleBuiltins(self): b, _ = parser_builtins.GetBuiltinsAndTyping() src = textwrap.dedent(""" from typing import Tuple, Union def f1(a: float) -> None: ... def f2() -> float: ... def f3(a: bool) -> None: ... def f4() -> bool: ... def f5(a: unicode) -> None: ... def f6() -> unicode: ... def f7(a: Union[unicode, int]) -> None: ... def f8(a: Tuple[unicode, int]) -> None: ... """) expected = textwrap.dedent(""" from typing import Tuple, Union def f1(a: Union[float, int]) -> None: ... def f2() -> float: ... def f3(a: Union[bool, None]) -> None: ... def f4() -> bool: ... def f5(a: Union[unicode, str, bytes]) -> None: ... def f6() -> unicode: ... def f7(a: Union[unicode, bytes, str, int]) -> None: ... def f8(a: Tuple[Union[unicode, bytes, str], int]) -> None: ... """) src_tree, expected_tree = (self.Parse(s) .Visit(visitors.LookupBuiltins(b)) for s in (src, expected)) new_tree = src_tree.Visit(visitors.ExpandCompatibleBuiltins(b)) self.AssertSourceEquals(new_tree, expected_tree)
def GetBuiltinsAndTyping(): """Get __builtin__.pytd and typing.pytd.""" global _cached_builtins_pytd if not _cached_builtins_pytd: t = parser.TypeDeclParser().Parse(_FindBuiltinFile("typing"), name="typing") t = t.Visit(visitors.AddNamePrefix()) b = parser.TypeDeclParser().Parse(_FindBuiltinFile("__builtin__"), name="__builtin__") b = b.Visit(visitors.AddNamePrefix()) b = b.Visit(visitors.NamedTypeToClassType()) b = b.Visit( visitors.LookupExternalTypes({"typing": t}, full_names=True, self_name="__builtin__")) t = t.Visit(visitors.LookupBuiltins(b)) t = t.Visit(visitors.NamedTypeToClassType()) b = visitors.AdjustTypeParameters(b) t = visitors.AdjustTypeParameters(t) b.Visit( visitors.FillInModuleClasses({ "": b, "typing": t, "__builtin__": b })) t.Visit( visitors.FillInModuleClasses({ "": t, "typing": t, "__builtin__": b })) b.Visit(visitors.VerifyLookup()) t.Visit(visitors.VerifyLookup()) b.Visit(visitors.VerifyContainers()) t.Visit(visitors.VerifyContainers()) _cached_builtins_pytd = b, t return _cached_builtins_pytd
def GetBuiltinsAndTyping(): """Get __builtin__.pytd and typing.pytd.""" global _cached_builtins_pytd if not _cached_builtins_pytd: t = parser.parse_string(_FindBuiltinFile("typing"), name="typing") b = parser.parse_string(_FindBuiltinFile("__builtin__"), name="__builtin__") b = b.Visit( visitors.LookupExternalTypes({"typing": t}, full_names=True, self_name="__builtin__")) t = t.Visit(visitors.LookupBuiltins(b)) b = b.Visit(visitors.NamedTypeToClassType()) t = t.Visit(visitors.NamedTypeToClassType()) b = b.Visit(visitors.AdjustTypeParameters()) t = t.Visit(visitors.AdjustTypeParameters()) b = b.Visit(visitors.CanonicalOrderingVisitor()) t = t.Visit(visitors.CanonicalOrderingVisitor()) b.Visit( visitors.FillInLocalPointers({ "": b, "typing": t, "__builtin__": b })) t.Visit( visitors.FillInLocalPointers({ "": t, "typing": t, "__builtin__": b })) b.Visit(visitors.VerifyLookup()) t.Visit(visitors.VerifyLookup()) b.Visit(visitors.VerifyContainers()) t.Visit(visitors.VerifyContainers()) _cached_builtins_pytd = b, t return _cached_builtins_pytd
def _postprocess_pyi(self, ast): """Apply all the PYI transformations we need.""" ast = ast.Visit(visitors.LookupBuiltins(self.builtins)) ast = ast.Visit(visitors.ExpandCompatibleBuiltins(self.builtins)) ast = ast.Visit(visitors.NamedTypeToClassType()) return ast
def parse(self, src): ast = parser.parse_string(textwrap.dedent(src)) ast = ast.Visit(visitors.LookupBuiltins( builtins.GetBuiltinsAndTyping(self.PYTHON_VERSION)[0])) return ast
def parse(self, src): ast = parser.parse_string(textwrap.dedent(src)) ast = ast.Visit(visitors.LookupBuiltins(self.builtins_pytd)) return ast
def ParseAndResolve(self, src): ast = self.Parse(src) return ast.Visit( visitors.LookupBuiltins(builtins.GetBuiltinsAndTyping()[0]))
def ParseAndResolve(self, src): ast = self.Parse(src) return ast.Visit(visitors.LookupBuiltins(self.builtins))
def PicklePyi(self, src, module_name): src = textwrap.dedent(src) ast = parser.parse_string(src, python_version=self.PYTHON_VERSION) ast = ast.Visit( visitors.LookupBuiltins(self.loader.builtins, full_names=False)) return self._Pickle(ast, module_name)