def assertTypesMatchPytd(self, ty, pytd_src):
"""Parses pytd_src and compares with ty."""
pytd_tree = parser.parse_string(
textwrap.dedent(pytd_src),
options=parser.PyiOptions(python_version=self.python_version))
pytd_tree = pytd_tree.Visit(
visitors.LookupBuiltins(self.loader.builtins, full_names=False))
pytd_tree = pytd_tree.Visit(visitors.LookupLocalTypes())
pytd_tree = pytd_tree.Visit(visitors.ClassTypeToNamedType())
pytd_tree = pytd_tree.Visit(
visitors.CanonicalOrderingVisitor(sort_signatures=True))
pytd_tree.Visit(visitors.VerifyVisitor())
ty = ty.Visit(visitors.ClassTypeToNamedType())
ty = ty.Visit(visitors.AdjustSelf())
ty = ty.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True))
ty.Visit(visitors.VerifyVisitor())
ty_src = pytd_utils.Print(ty) + "\n"
pytd_tree_src = pytd_utils.Print(pytd_tree) + "\n"
log.info("========== result ==========")
_LogLines(log.info, ty_src)
log.info("========== expected ==========")
_LogLines(log.info, pytd_tree_src)
log.info("==============================")
# In the diff output, mark expected with "-" and actual with "+".
# (In other words, display a change from "working" to "broken")
self.assertMultiLineEqual(pytd_tree_src, ty_src)
def test_asteq(self):
# This creates two ASts that are equivalent but whose sources are slightly
# different. The union types are different (int,str) vs (str,int) but the
# ordering is ignored when testing for equality (which ASTeq uses).
src1 = textwrap.dedent("""
from typing import Union
def foo(a: Union[int, str]) -> C: ...
T = TypeVar('T')
class C(typing.Generic[T], object):
def bar(x: T) -> NoneType: ...
CONSTANT = ... # type: C[float]
""")
src2 = textwrap.dedent("""
from typing import Union
CONSTANT = ... # type: C[float]
T = TypeVar('T')
class C(typing.Generic[T], object):
def bar(x: T) -> NoneType: ...
def foo(a: Union[str, int]) -> C: ...
""")
tree1 = parser.parse_string(src1, python_version=self.python_version)
tree2 = parser.parse_string(src2, python_version=self.python_version)
tree1.Visit(visitors.VerifyVisitor())
tree2.Visit(visitors.VerifyVisitor())
self.assertTrue(tree1.constants)
self.assertTrue(tree1.classes)
self.assertTrue(tree1.functions)
self.assertTrue(tree2.constants)
self.assertTrue(tree2.classes)
self.assertTrue(tree2.functions)
self.assertIsInstance(tree1, pytd.TypeDeclUnit)
self.assertIsInstance(tree2, pytd.TypeDeclUnit)
# For the ==, != tests, TypeDeclUnit uses identity
# pylint: disable=g-generic-assert
# pylint: disable=comparison-with-itself
self.assertTrue(tree1 == tree1)
self.assertTrue(tree2 == tree2)
self.assertFalse(tree1 == tree2)
self.assertFalse(tree2 == tree1)
self.assertFalse(tree1 != tree1)
self.assertFalse(tree2 != tree2)
self.assertTrue(tree1 != tree2)
self.assertTrue(tree2 != tree1)
# pylint: enable=g-generic-assert
# pylint: enable=comparison-with-itself
self.assertEqual(tree1, tree1)
self.assertEqual(tree2, tree2)
self.assertNotEqual(tree1, tree2)
self.assertTrue(pytd_utils.ASTeq(tree1, tree2))
self.assertTrue(pytd_utils.ASTeq(tree1, tree1))
self.assertTrue(pytd_utils.ASTeq(tree2, tree1))
self.assertTrue(pytd_utils.ASTeq(tree2, tree2))
def testDefaceUnresolved2(self):
builtins = self.Parse(textwrap.dedent("""
from typing import Generic, TypeVar
class int(object):
pass
T = TypeVar("T")
class list(Generic[T]):
pass
"""))
src = textwrap.dedent("""
from typing import Union
class A(X):
def a(self, a: A, b: X, c: int) -> X:
raise X()
def c(self) -> Union[list[X], int]
""")
expected = textwrap.dedent("""
from typing import Union
class A(?):
def a(self, a: A, b: ?, c: int) -> ?:
raise ?
def c(self) -> Union[list[?], int]
""")
tree = self.Parse(src)
new_tree = tree.Visit(visitors.DefaceUnresolved([tree, builtins]))
new_tree.Visit(visitors.VerifyVisitor())
self.AssertSourceEquals(new_tree, expected)
def canonical_pyi(pyi, multiline_args=False, options=None): """Rewrite a pyi in canonical form.""" ast = parse_string(pyi, options=options) ast = ast.Visit(visitors.ClassTypeToNamedType()) ast = ast.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True)) ast.Visit(visitors.VerifyVisitor()) return pytd_utils.Print(ast, multiline_args)
def InferWithErrors(self,
code,
deep=True,
pythonpath=(),
module_name=None,
analyze_annotated=True,
quick=False,
imports_map=None,
**kwargs):
"""Runs inference on code expected to have type errors."""
kwargs.update(
self._SetUpErrorHandling(code, pythonpath, analyze_annotated,
quick, imports_map))
self.ConfigureOptions(module_name=module_name)
unit, builtins_pytd = analyze.infer_types(deep=deep, **kwargs)
unit.Visit(visitors.VerifyVisitor())
unit = optimize.Optimize(unit,
builtins_pytd,
lossy=False,
use_abcs=False,
max_union=7,
remove_mutable=False)
errorlog = kwargs["errorlog"]
errorlog.assert_errors_match_expected()
return pytd_utils.CanonicalOrdering(unit), errorlog
def _InferAndVerify(
self, src, pythonpath, module_name, report_errors, analyze_annotated,
imports_map=None, quick=False, **kwargs):
"""Infer types for the source code treating it as a module.
Used by Infer().
Args:
src: The source code of a module. Treat it as "__main__".
pythonpath: --pythonpath as list/tuple of string
module_name: Name of the module we're analyzing. E.g. "foo.bar.mymodule".
report_errors: Whether to fail if the type inferencer reports any errors
in the program.
analyze_annotated: Whether to analyze functions with return annotations.
imports_map: --imports_info data
quick: Try to run faster, by avoiding costly computations.
**kwargs: Keyword parameters to pass through to the type inferencer.
Raises:
AssertionError: If report_errors is True and we found errors.
Returns:
A pytd.TypeDeclUnit
"""
self.ConfigureOptions(
module_name=module_name, quick=quick, use_pickled_files=True,
pythonpath=[""] if (not pythonpath and imports_map) else pythonpath,
imports_map=imports_map, analyze_annotated=analyze_annotated)
errorlog = errors.ErrorLog()
unit, builtins_pytd = analyze.infer_types(
src, errorlog, self.options, loader=self.loader, **kwargs)
unit.Visit(visitors.VerifyVisitor())
if report_errors and errorlog:
errorlog.print_to_stderr()
self.fail("Inferencer found %d errors" % len(errorlog))
return unit, builtins_pytd
def ToAST(self, src_or_tree):
if isinstance(src_or_tree, str):
# Put into a canonical form (removes comments, standard indents):
return self.Parse(src_or_tree + "\n")
else: # isinstance(src_or_tree, tuple):
src_or_tree.Visit(visitors.VerifyVisitor())
return src_or_tree
def InferWithErrors(self, code, deep=True, pythonpath=(),
analyze_annotated=True, quick=False, **kwargs):
kwargs.update(
self._SetUpErrorHandling(code, pythonpath, analyze_annotated, quick))
unit, builtins_pytd = analyze.infer_types(deep=deep, **kwargs)
unit.Visit(visitors.VerifyVisitor())
unit = optimize.Optimize(unit, builtins_pytd, lossy=False, use_abcs=False,
max_union=7, remove_mutable=False)
return pytd_utils.CanonicalOrdering(unit), kwargs["errorlog"]
def ParseWithBuiltins(self, src):
ast = parser.parse_string(textwrap.dedent(src), options=self.options)
ast = ast.Visit(visitors.LookupExternalTypes(
{"builtins": self.loader.builtins, "typing": self.loader.typing}))
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = ast.Visit(visitors.AdjustTypeParameters())
ast.Visit(visitors.FillInLocalPointers({
"": ast, "builtins": self.loader.builtins}))
ast.Visit(visitors.VerifyVisitor())
return ast
def ParseWithBuiltins(self, src):
ast = parser.parse_string(textwrap.dedent(src),
python_version=self.PYTHON_VERSION)
ast = ast.Visit(visitors.LookupExternalTypes(
{"__builtin__": self.loader.builtins, "typing": self.loader.typing}))
ast = ast.Visit(visitors.NamedTypeToClassType())
ast = ast.Visit(visitors.AdjustTypeParameters())
ast.Visit(visitors.FillInLocalPointers({
"": ast, "__builtin__": self.loader.builtins}))
ast.Visit(visitors.VerifyVisitor())
return ast
def Parse(self, src, name=None, version=None, platform=None):
version = version or self.PYTHON_VERSION
tree = parser.parse_string(
textwrap.dedent(src), name=name, python_version=version,
platform=platform)
tree = tree.Visit(visitors.NamedTypeToClassType())
tree = tree.Visit(visitors.AdjustTypeParameters())
# Convert back to named types for easier testing
tree = tree.Visit(visitors.ClassTypeToNamedType())
tree.Visit(visitors.VerifyVisitor())
return tree
def InferFromFile(self, filename, pythonpath):
with open(filename, "r") as fi:
code = fi.read()
errorlog = errors.ErrorLog()
self.ConfigureOptions(
module_name=load_pytd.get_module_name(filename, pythonpath),
pythonpath=pythonpath)
unit, _ = analyze.infer_types(code, errorlog, self.options,
loader=self.loader, filename=filename)
unit.Visit(visitors.VerifyVisitor())
return pytd_utils.CanonicalOrdering(unit)
def Parse(self, src, name=None, version=None, platform=None):
if version:
self.options.python_version = version
if platform:
self.options.platform = platform
tree = parser.parse_string(
textwrap.dedent(src), name=name, options=self.options)
tree = tree.Visit(visitors.NamedTypeToClassType())
tree = tree.Visit(visitors.AdjustTypeParameters())
# Convert back to named types for easier testing
tree = tree.Visit(visitors.ClassTypeToNamedType())
tree.Visit(visitors.VerifyVisitor())
return tree
def InferFromFile(self, filename, pythonpath):
with open(filename, "r") as fi:
code = fi.read()
errorlog = test_utils.TestErrorLog(code)
if errorlog.expected:
self.fail(
"Cannot assert errors with InferFromFile(); use InferWithErrors()")
self.ConfigureOptions(
module_name=load_pytd.get_module_name(filename, pythonpath),
pythonpath=pythonpath)
unit, _ = analyze.infer_types(code, errorlog, self.options,
loader=self.loader, filename=filename)
unit.Visit(visitors.VerifyVisitor())
return pytd_utils.CanonicalOrdering(unit)
def InferFromFile(self, filename, pythonpath, python_version=None):
with open(filename, "rb") as fi:
code = fi.read()
errorlog = errors.ErrorLog()
self.options.tweak(module_name=load_pytd.get_module_name(
filename, pythonpath),
pythonpath=pythonpath,
python_version=python_version)
self._CreateLoader()
unit, _ = analyze.infer_types(code,
errorlog,
self.options,
loader=self.loader,
filename=filename)
unit.Visit(visitors.VerifyVisitor())
return pytd_utils.CanonicalOrdering(unit)
def _InferAndVerify(self,
src,
pythonpath=(),
module_name=None,
imports_map=None,
report_errors=False,
quick=False,
**kwargs):
"""Infer types for the source code treating it as a module.
Used by Infer().
Args:
src: The source code of a module. Treat it as "__main__".
pythonpath: --pythonpath as list/tuple of string
module_name: Name of the module we're analyzing. E.g. "foo.bar.mymodule".
imports_map: --imports_info data
report_errors: Whether to fail if the type inferencer reports any errors
in the program.
quick: Try to run faster, by avoiding costly computations.
**kwargs: Keyword parameters to pass through to the type inferencer.
Raises:
AssertionError: If report_errors is True and we found errors.
Returns:
A pytd.TypeDeclUnit
"""
self.options.tweak(module_name=module_name, quick=quick)
errorlog = errors.ErrorLog()
self.loader = load_pytd.PickledPyiLoader(
base_module=module_name,
python_version=self.PYTHON_VERSION,
pythonpath=[""] if
(not pythonpath and imports_map) else pythonpath,
imports_map=imports_map)
unit, builtins_pytd = analyze.infer_types(src,
errorlog,
self.options,
loader=self.loader,
**kwargs)
unit.Visit(visitors.VerifyVisitor())
unit = pytd_utils.CanonicalOrdering(unit)
if report_errors and len(errorlog):
errorlog.print_to_stderr()
self.fail("Inferencer found %d errors" % len(errorlog))
return unit, builtins_pytd
def generate_pyi(input_filename, errorlog, options, loader):
"""Run the inferencer on one file, producing output.
Args:
input_filename: name of the file to process
errorlog: Where error messages go. Instance of errors.ErrorLog.
options: config.Options object.
loader: A load_pytd.Loader instance.
Returns:
A tuple, (PYI Ast as string, TypeDeclUnit).
Raises:
CompileError: If we couldn't parse the input file.
UsageError: If the input filepath is invalid.
"""
mod, builtins = _call(analyze.infer_types, input_filename, errorlog,
options, loader)
mod.Visit(visitors.VerifyVisitor())
mod = optimize.Optimize(
mod,
builtins,
# TODO(kramm): Add FLAGs for these
lossy=False,
use_abcs=False,
max_union=7,
remove_mutable=False)
mod = pytd_utils.CanonicalOrdering(mod, sort_signatures=True)
result = pytd.Print(mod)
log.info("=========== pyi optimized =============")
log.info("\n%s", result)
log.info("========================================")
if not result.endswith("\n"):
result += "\n"
result_prefix = ""
if options.quick:
result_prefix += "# (generated with --quick)\n"
if result_prefix:
result = result_prefix + "\n" + result
return result, mod
def testDefaceUnresolved(self):
builtins = self.Parse(textwrap.dedent("""
class int(object):
pass
"""))
src = textwrap.dedent("""
class A(X):
def a(self, a: A, b: X, c: int) -> X:
raise X()
def b(self) -> X[int]
""")
expected = textwrap.dedent("""
class A(?):
def a(self, a: A, b: ?, c: int) -> ?:
raise ?
def b(self) -> ?
""")
tree = self.Parse(src)
new_tree = tree.Visit(visitors.DefaceUnresolved([tree, builtins]))
new_tree.Visit(visitors.VerifyVisitor())
self.AssertSourceEquals(new_tree, expected)
def test_deface_unresolved(self):
builtins = self.Parse(textwrap.dedent("""
class int:
pass
"""))
src = textwrap.dedent("""
class A(X):
def a(self, a: A, b: X, c: int) -> X:
raise X()
def b(self) -> X[int]: ...
""")
expected = textwrap.dedent("""
from typing import Any
class A(Any):
def a(self, a: A, b: Any, c: int) -> Any:
raise Any
def b(self) -> Any: ...
""")
tree = self.Parse(src)
new_tree = tree.Visit(visitors.DefaceUnresolved([tree, builtins]))
new_tree.Visit(visitors.VerifyVisitor())
self.AssertSourceEquals(new_tree, expected)
def generate_pyi(src, options=None, loader=None):
"""Run the inferencer on a string of source code, producing output.
Args:
src: The source code.
options: config.Options object.
loader: A load_pytd.Loader instance.
Returns:
A tuple, (errors.ErrorLog, PYI Ast as string, TypeDeclUnit).
Raises:
CompileError: If we couldn't parse the input file.
UsageError: If the input filepath is invalid.
"""
options = options or config.Options.create()
with config.verbosity_from(options):
errorlog, (mod, builtins) = _call(analyze.infer_types, src, options,
loader)
mod.Visit(visitors.VerifyVisitor())
mod = optimize.Optimize(
mod,
builtins,
# TODO(b/159038508): Add FLAGs for these
lossy=False,
use_abcs=False,
max_union=7,
remove_mutable=False)
mod = pytd_utils.CanonicalOrdering(mod, sort_signatures=True)
result = pytd_utils.Print(mod)
log.info("=========== pyi optimized =============")
log.info("\n%s", result)
log.info("========================================")
result += "\n"
if options.quick:
result = "# (generated with --quick)\n\n" + result
return errorlog, result, mod
def canonical_pyi(pyi, python_version, multiline_args=False):
ast = parser.parse_string(pyi, python_version=python_version)
ast = ast.Visit(visitors.ClassTypeToNamedType())
ast = ast.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True))
ast.Visit(visitors.VerifyVisitor())
return Print(ast, multiline_args)
