def assertTypesMatchPytd(self, ty, pytd_src): """Parses pytd_src and compares with ty.""" pytd_tree = parser.parse_string( textwrap.dedent(pytd_src), options=parser.PyiOptions(python_version=self.python_version)) pytd_tree = pytd_tree.Visit( visitors.LookupBuiltins(self.loader.builtins, full_names=False)) pytd_tree = pytd_tree.Visit(visitors.LookupLocalTypes()) pytd_tree = pytd_tree.Visit(visitors.ClassTypeToNamedType()) pytd_tree = pytd_tree.Visit( visitors.CanonicalOrderingVisitor(sort_signatures=True)) pytd_tree.Visit(visitors.VerifyVisitor()) ty = ty.Visit(visitors.ClassTypeToNamedType()) ty = ty.Visit(visitors.AdjustSelf()) ty = ty.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True)) ty.Visit(visitors.VerifyVisitor()) ty_src = pytd_utils.Print(ty) + "\n" pytd_tree_src = pytd_utils.Print(pytd_tree) + "\n" log.info("========== result ==========") _LogLines(log.info, ty_src) log.info("========== expected ==========") _LogLines(log.info, pytd_tree_src) log.info("==============================") # In the diff output, mark expected with "-" and actual with "+". # (In other words, display a change from "working" to "broken") self.assertMultiLineEqual(pytd_tree_src, ty_src)
def test_asteq(self): # This creates two ASts that are equivalent but whose sources are slightly # different. The union types are different (int,str) vs (str,int) but the # ordering is ignored when testing for equality (which ASTeq uses). src1 = textwrap.dedent(""" from typing import Union def foo(a: Union[int, str]) -> C: ... T = TypeVar('T') class C(typing.Generic[T], object): def bar(x: T) -> NoneType: ... CONSTANT = ... # type: C[float] """) src2 = textwrap.dedent(""" from typing import Union CONSTANT = ... # type: C[float] T = TypeVar('T') class C(typing.Generic[T], object): def bar(x: T) -> NoneType: ... def foo(a: Union[str, int]) -> C: ... """) tree1 = parser.parse_string(src1, python_version=self.python_version) tree2 = parser.parse_string(src2, python_version=self.python_version) tree1.Visit(visitors.VerifyVisitor()) tree2.Visit(visitors.VerifyVisitor()) self.assertTrue(tree1.constants) self.assertTrue(tree1.classes) self.assertTrue(tree1.functions) self.assertTrue(tree2.constants) self.assertTrue(tree2.classes) self.assertTrue(tree2.functions) self.assertIsInstance(tree1, pytd.TypeDeclUnit) self.assertIsInstance(tree2, pytd.TypeDeclUnit) # For the ==, != tests, TypeDeclUnit uses identity # pylint: disable=g-generic-assert # pylint: disable=comparison-with-itself self.assertTrue(tree1 == tree1) self.assertTrue(tree2 == tree2) self.assertFalse(tree1 == tree2) self.assertFalse(tree2 == tree1) self.assertFalse(tree1 != tree1) self.assertFalse(tree2 != tree2) self.assertTrue(tree1 != tree2) self.assertTrue(tree2 != tree1) # pylint: enable=g-generic-assert # pylint: enable=comparison-with-itself self.assertEqual(tree1, tree1) self.assertEqual(tree2, tree2) self.assertNotEqual(tree1, tree2) self.assertTrue(pytd_utils.ASTeq(tree1, tree2)) self.assertTrue(pytd_utils.ASTeq(tree1, tree1)) self.assertTrue(pytd_utils.ASTeq(tree2, tree1)) self.assertTrue(pytd_utils.ASTeq(tree2, tree2))
def testDefaceUnresolved2(self): builtins = self.Parse(textwrap.dedent(""" from typing import Generic, TypeVar class int(object): pass T = TypeVar("T") class list(Generic[T]): pass """)) src = textwrap.dedent(""" from typing import Union class A(X): def a(self, a: A, b: X, c: int) -> X: raise X() def c(self) -> Union[list[X], int] """) expected = textwrap.dedent(""" from typing import Union class A(?): def a(self, a: A, b: ?, c: int) -> ?: raise ? def c(self) -> Union[list[?], int] """) tree = self.Parse(src) new_tree = tree.Visit(visitors.DefaceUnresolved([tree, builtins])) new_tree.Visit(visitors.VerifyVisitor()) self.AssertSourceEquals(new_tree, expected)
def canonical_pyi(pyi, multiline_args=False, options=None): """Rewrite a pyi in canonical form.""" ast = parse_string(pyi, options=options) ast = ast.Visit(visitors.ClassTypeToNamedType()) ast = ast.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True)) ast.Visit(visitors.VerifyVisitor()) return pytd_utils.Print(ast, multiline_args)
def InferWithErrors(self, code, deep=True, pythonpath=(), module_name=None, analyze_annotated=True, quick=False, imports_map=None, **kwargs): """Runs inference on code expected to have type errors.""" kwargs.update( self._SetUpErrorHandling(code, pythonpath, analyze_annotated, quick, imports_map)) self.ConfigureOptions(module_name=module_name) unit, builtins_pytd = analyze.infer_types(deep=deep, **kwargs) unit.Visit(visitors.VerifyVisitor()) unit = optimize.Optimize(unit, builtins_pytd, lossy=False, use_abcs=False, max_union=7, remove_mutable=False) errorlog = kwargs["errorlog"] errorlog.assert_errors_match_expected() return pytd_utils.CanonicalOrdering(unit), errorlog
def _InferAndVerify( self, src, pythonpath, module_name, report_errors, analyze_annotated, imports_map=None, quick=False, **kwargs): """Infer types for the source code treating it as a module. Used by Infer(). Args: src: The source code of a module. Treat it as "__main__". pythonpath: --pythonpath as list/tuple of string module_name: Name of the module we're analyzing. E.g. "foo.bar.mymodule". report_errors: Whether to fail if the type inferencer reports any errors in the program. analyze_annotated: Whether to analyze functions with return annotations. imports_map: --imports_info data quick: Try to run faster, by avoiding costly computations. **kwargs: Keyword parameters to pass through to the type inferencer. Raises: AssertionError: If report_errors is True and we found errors. Returns: A pytd.TypeDeclUnit """ self.ConfigureOptions( module_name=module_name, quick=quick, use_pickled_files=True, pythonpath=[""] if (not pythonpath and imports_map) else pythonpath, imports_map=imports_map, analyze_annotated=analyze_annotated) errorlog = errors.ErrorLog() unit, builtins_pytd = analyze.infer_types( src, errorlog, self.options, loader=self.loader, **kwargs) unit.Visit(visitors.VerifyVisitor()) if report_errors and errorlog: errorlog.print_to_stderr() self.fail("Inferencer found %d errors" % len(errorlog)) return unit, builtins_pytd
def ToAST(self, src_or_tree): if isinstance(src_or_tree, str): # Put into a canonical form (removes comments, standard indents): return self.Parse(src_or_tree + "\n") else: # isinstance(src_or_tree, tuple): src_or_tree.Visit(visitors.VerifyVisitor()) return src_or_tree
def InferWithErrors(self, code, deep=True, pythonpath=(), analyze_annotated=True, quick=False, **kwargs): kwargs.update( self._SetUpErrorHandling(code, pythonpath, analyze_annotated, quick)) unit, builtins_pytd = analyze.infer_types(deep=deep, **kwargs) unit.Visit(visitors.VerifyVisitor()) unit = optimize.Optimize(unit, builtins_pytd, lossy=False, use_abcs=False, max_union=7, remove_mutable=False) return pytd_utils.CanonicalOrdering(unit), kwargs["errorlog"]
def ParseWithBuiltins(self, src): ast = parser.parse_string(textwrap.dedent(src), options=self.options) ast = ast.Visit(visitors.LookupExternalTypes( {"builtins": self.loader.builtins, "typing": self.loader.typing})) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.AdjustTypeParameters()) ast.Visit(visitors.FillInLocalPointers({ "": ast, "builtins": self.loader.builtins})) ast.Visit(visitors.VerifyVisitor()) return ast
def ParseWithBuiltins(self, src): ast = parser.parse_string(textwrap.dedent(src), python_version=self.PYTHON_VERSION) ast = ast.Visit(visitors.LookupExternalTypes( {"__builtin__": self.loader.builtins, "typing": self.loader.typing})) ast = ast.Visit(visitors.NamedTypeToClassType()) ast = ast.Visit(visitors.AdjustTypeParameters()) ast.Visit(visitors.FillInLocalPointers({ "": ast, "__builtin__": self.loader.builtins})) ast.Visit(visitors.VerifyVisitor()) return ast
def Parse(self, src, name=None, version=None, platform=None): version = version or self.PYTHON_VERSION tree = parser.parse_string( textwrap.dedent(src), name=name, python_version=version, platform=platform) tree = tree.Visit(visitors.NamedTypeToClassType()) tree = tree.Visit(visitors.AdjustTypeParameters()) # Convert back to named types for easier testing tree = tree.Visit(visitors.ClassTypeToNamedType()) tree.Visit(visitors.VerifyVisitor()) return tree
def InferFromFile(self, filename, pythonpath): with open(filename, "r") as fi: code = fi.read() errorlog = errors.ErrorLog() self.ConfigureOptions( module_name=load_pytd.get_module_name(filename, pythonpath), pythonpath=pythonpath) unit, _ = analyze.infer_types(code, errorlog, self.options, loader=self.loader, filename=filename) unit.Visit(visitors.VerifyVisitor()) return pytd_utils.CanonicalOrdering(unit)
def Parse(self, src, name=None, version=None, platform=None): if version: self.options.python_version = version if platform: self.options.platform = platform tree = parser.parse_string( textwrap.dedent(src), name=name, options=self.options) tree = tree.Visit(visitors.NamedTypeToClassType()) tree = tree.Visit(visitors.AdjustTypeParameters()) # Convert back to named types for easier testing tree = tree.Visit(visitors.ClassTypeToNamedType()) tree.Visit(visitors.VerifyVisitor()) return tree
def InferFromFile(self, filename, pythonpath): with open(filename, "r") as fi: code = fi.read() errorlog = test_utils.TestErrorLog(code) if errorlog.expected: self.fail( "Cannot assert errors with InferFromFile(); use InferWithErrors()") self.ConfigureOptions( module_name=load_pytd.get_module_name(filename, pythonpath), pythonpath=pythonpath) unit, _ = analyze.infer_types(code, errorlog, self.options, loader=self.loader, filename=filename) unit.Visit(visitors.VerifyVisitor()) return pytd_utils.CanonicalOrdering(unit)
def InferFromFile(self, filename, pythonpath, python_version=None): with open(filename, "rb") as fi: code = fi.read() errorlog = errors.ErrorLog() self.options.tweak(module_name=load_pytd.get_module_name( filename, pythonpath), pythonpath=pythonpath, python_version=python_version) self._CreateLoader() unit, _ = analyze.infer_types(code, errorlog, self.options, loader=self.loader, filename=filename) unit.Visit(visitors.VerifyVisitor()) return pytd_utils.CanonicalOrdering(unit)
def _InferAndVerify(self, src, pythonpath=(), module_name=None, imports_map=None, report_errors=False, quick=False, **kwargs): """Infer types for the source code treating it as a module. Used by Infer(). Args: src: The source code of a module. Treat it as "__main__". pythonpath: --pythonpath as list/tuple of string module_name: Name of the module we're analyzing. E.g. "foo.bar.mymodule". imports_map: --imports_info data report_errors: Whether to fail if the type inferencer reports any errors in the program. quick: Try to run faster, by avoiding costly computations. **kwargs: Keyword parameters to pass through to the type inferencer. Raises: AssertionError: If report_errors is True and we found errors. Returns: A pytd.TypeDeclUnit """ self.options.tweak(module_name=module_name, quick=quick) errorlog = errors.ErrorLog() self.loader = load_pytd.PickledPyiLoader( base_module=module_name, python_version=self.PYTHON_VERSION, pythonpath=[""] if (not pythonpath and imports_map) else pythonpath, imports_map=imports_map) unit, builtins_pytd = analyze.infer_types(src, errorlog, self.options, loader=self.loader, **kwargs) unit.Visit(visitors.VerifyVisitor()) unit = pytd_utils.CanonicalOrdering(unit) if report_errors and len(errorlog): errorlog.print_to_stderr() self.fail("Inferencer found %d errors" % len(errorlog)) return unit, builtins_pytd
def generate_pyi(input_filename, errorlog, options, loader): """Run the inferencer on one file, producing output. Args: input_filename: name of the file to process errorlog: Where error messages go. Instance of errors.ErrorLog. options: config.Options object. loader: A load_pytd.Loader instance. Returns: A tuple, (PYI Ast as string, TypeDeclUnit). Raises: CompileError: If we couldn't parse the input file. UsageError: If the input filepath is invalid. """ mod, builtins = _call(analyze.infer_types, input_filename, errorlog, options, loader) mod.Visit(visitors.VerifyVisitor()) mod = optimize.Optimize( mod, builtins, # TODO(kramm): Add FLAGs for these lossy=False, use_abcs=False, max_union=7, remove_mutable=False) mod = pytd_utils.CanonicalOrdering(mod, sort_signatures=True) result = pytd.Print(mod) log.info("=========== pyi optimized =============") log.info("\n%s", result) log.info("========================================") if not result.endswith("\n"): result += "\n" result_prefix = "" if options.quick: result_prefix += "# (generated with --quick)\n" if result_prefix: result = result_prefix + "\n" + result return result, mod
def testDefaceUnresolved(self): builtins = self.Parse(textwrap.dedent(""" class int(object): pass """)) src = textwrap.dedent(""" class A(X): def a(self, a: A, b: X, c: int) -> X: raise X() def b(self) -> X[int] """) expected = textwrap.dedent(""" class A(?): def a(self, a: A, b: ?, c: int) -> ?: raise ? def b(self) -> ? """) tree = self.Parse(src) new_tree = tree.Visit(visitors.DefaceUnresolved([tree, builtins])) new_tree.Visit(visitors.VerifyVisitor()) self.AssertSourceEquals(new_tree, expected)
def test_deface_unresolved(self): builtins = self.Parse(textwrap.dedent(""" class int: pass """)) src = textwrap.dedent(""" class A(X): def a(self, a: A, b: X, c: int) -> X: raise X() def b(self) -> X[int]: ... """) expected = textwrap.dedent(""" from typing import Any class A(Any): def a(self, a: A, b: Any, c: int) -> Any: raise Any def b(self) -> Any: ... """) tree = self.Parse(src) new_tree = tree.Visit(visitors.DefaceUnresolved([tree, builtins])) new_tree.Visit(visitors.VerifyVisitor()) self.AssertSourceEquals(new_tree, expected)
def generate_pyi(src, options=None, loader=None): """Run the inferencer on a string of source code, producing output. Args: src: The source code. options: config.Options object. loader: A load_pytd.Loader instance. Returns: A tuple, (errors.ErrorLog, PYI Ast as string, TypeDeclUnit). Raises: CompileError: If we couldn't parse the input file. UsageError: If the input filepath is invalid. """ options = options or config.Options.create() with config.verbosity_from(options): errorlog, (mod, builtins) = _call(analyze.infer_types, src, options, loader) mod.Visit(visitors.VerifyVisitor()) mod = optimize.Optimize( mod, builtins, # TODO(b/159038508): Add FLAGs for these lossy=False, use_abcs=False, max_union=7, remove_mutable=False) mod = pytd_utils.CanonicalOrdering(mod, sort_signatures=True) result = pytd_utils.Print(mod) log.info("=========== pyi optimized =============") log.info("\n%s", result) log.info("========================================") result += "\n" if options.quick: result = "# (generated with --quick)\n\n" + result return errorlog, result, mod
def canonical_pyi(pyi, python_version, multiline_args=False): ast = parser.parse_string(pyi, python_version=python_version) ast = ast.Visit(visitors.ClassTypeToNamedType()) ast = ast.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True)) ast.Visit(visitors.VerifyVisitor()) return Print(ast, multiline_args)