def check_error(self, src, expected_line, message): """Check that parsing the src raises the expected error.""" with self.assertRaises(parser.ParseError) as e: parser.parse_string(textwrap.dedent(src).lstrip(), options=self.options) self.assertRegex(utils.message(e.exception), re.escape(message)) self.assertEqual(expected_line, e.exception.line)
def parse(): try: parser.parse_string(src) except parser.ParseError: # It is essential to clear the error, otherwise the system exc_info # will hold references to lots of stuff hanging off the exception. sys.exc_clear()
def setUp(self): builtins = parser.parse_string(textwrap.dedent(_BUILTINS), name="__builtin__", python_version=self.PYTHON_VERSION) typing = parser.parse_string("class Generic: ...", name="typing", python_version=self.PYTHON_VERSION) self.mini_builtins = pytd_utils.Concat(builtins, typing)
def assertInferredPyiEquals(self, expected_pyi=None, filename=None): assert bool(expected_pyi) != bool(filename) if filename: with open(self._DataPath(filename), "r") as f: expected_pyi = f.read() self.assertTrue( parser.parse_string(self.stdout).ASTeq( parser.parse_string(expected_pyi)))
def check_error(self, src, expected_line, message): """Check that parsing the src raises the expected error.""" try: parser.parse_string(textwrap.dedent(src)) self.fail("ParseError expected") except parser.ParseError as e: self.assertRegexpMatches(e.message, re.escape(message)) self.assertEquals(expected_line, e.line)
def setUp(self): super().setUp() builtins = parser.parse_string(textwrap.dedent(_BUILTINS), name="builtins", options=self.options) typing = parser.parse_string("class Generic: ...", name="typing", options=self.options) self.mini_builtins = pytd_utils.Concat(builtins, typing)
def parse(): try: parser.parse_string(src, python_version=self.PYTHON_VERSION) except parser.ParseError: if six.PY2: # It is essential to clear the error, otherwise the system exc_info # will hold references to lots of stuff hanging off the exception. # This happens only in Python2. sys.exc_clear()
def setUp(self): super(TestTypeMatch, self).setUp() builtins = parser.parse_string(textwrap.dedent(_BUILTINS), name="__builtin__", python_version=self.python_version) typing = parser.parse_string("class Generic: ...", name="typing", python_version=self.python_version) self.mini_builtins = pytd_utils.Concat(builtins, typing)
def _test_parse(pyi_file): python_version = (2, 7) module = os.path.splitext(os.path.basename(pyi_file))[0] if "__init__" == module: module = os.path.basename(os.path.dirname(pyi_file)) with open(pyi_file) as f: src = f.read() parser.parse_string(src, filename=pyi_file, name=module, python_version=python_version)
def assertInferredPyiEquals(self, expected_pyi=None, filename=None): assert bool(expected_pyi) != bool(filename) if filename: with open(self._DataPath(filename), "r") as f: expected_pyi = f.read() message = ("\n==Expected pyi==\n" + expected_pyi + "\n==Actual pyi==\n" + self.stdout) self.assertTrue( parser.parse_string(self.stdout).ASTeq( parser.parse_string(expected_pyi)), message)
def test_asteq(self): # This creates two ASts that are equivalent but whose sources are slightly # different. The union types are different (int,str) vs (str,int) but the # ordering is ignored when testing for equality (which ASTeq uses). src1 = textwrap.dedent(""" from typing import Union def foo(a: Union[int, str]) -> C: ... T = TypeVar('T') class C(typing.Generic[T], object): def bar(x: T) -> NoneType: ... CONSTANT = ... # type: C[float] """) src2 = textwrap.dedent(""" from typing import Union CONSTANT = ... # type: C[float] T = TypeVar('T') class C(typing.Generic[T], object): def bar(x: T) -> NoneType: ... def foo(a: Union[str, int]) -> C: ... """) tree1 = parser.parse_string(src1, python_version=self.python_version) tree2 = parser.parse_string(src2, python_version=self.python_version) tree1.Visit(visitors.VerifyVisitor()) tree2.Visit(visitors.VerifyVisitor()) self.assertTrue(tree1.constants) self.assertTrue(tree1.classes) self.assertTrue(tree1.functions) self.assertTrue(tree2.constants) self.assertTrue(tree2.classes) self.assertTrue(tree2.functions) self.assertIsInstance(tree1, pytd.TypeDeclUnit) self.assertIsInstance(tree2, pytd.TypeDeclUnit) # For the ==, != tests, TypeDeclUnit uses identity # pylint: disable=g-generic-assert # pylint: disable=comparison-with-itself self.assertTrue(tree1 == tree1) self.assertTrue(tree2 == tree2) self.assertFalse(tree1 == tree2) self.assertFalse(tree2 == tree1) self.assertFalse(tree1 != tree1) self.assertFalse(tree2 != tree2) self.assertTrue(tree1 != tree2) self.assertTrue(tree2 != tree1) # pylint: enable=g-generic-assert # pylint: enable=comparison-with-itself self.assertEqual(tree1, tree1) self.assertEqual(tree2, tree2) self.assertNotEqual(tree1, tree2) self.assertTrue(pytd_utils.ASTeq(tree1, tree2)) self.assertTrue(pytd_utils.ASTeq(tree1, tree1)) self.assertTrue(pytd_utils.ASTeq(tree2, tree1)) self.assertTrue(pytd_utils.ASTeq(tree2, tree2))
def testInferToFile(self): self.pytype_args[self._DataPath("simple.py")] = self.INCLUDE pyi_file = self._TmpPath("simple.pyi") self.pytype_args["--output"] = pyi_file self._RunPytype(self.pytype_args) self.assertOutputStateMatches(stdout=False, stderr=False, returncode=False) with open(pyi_file, "r") as f: pyi = f.read() with open(self._DataPath("simple.pyi"), "r") as f: expected_pyi = f.read() self.assertTrue( parser.parse_string(pyi).ASTeq(parser.parse_string(expected_pyi)))
def test_error_formatting(self): src = """\ class Foo: this is not valid""" try: parser.parse_string(textwrap.dedent(src), filename="foo.py") self.fail("ParseError expected") except parser.ParseError as e: self.assertMultiLineEqual(textwrap.dedent("""\ File: "foo.py", line 2 this is not valid ^ ParseError: syntax error, unexpected NAME, expecting ':' or '='""" ), str(e))
def testASTdiff(self): src1 = textwrap.dedent("""\ a = ... # type: int b = ... # type: str""") src2 = textwrap.dedent("""\ a = ... # type: int b = ... # type: float""") tree1 = parser.parse_string(src1, python_version=self.PYTHON_VERSION) tree2 = parser.parse_string(src2, python_version=self.PYTHON_VERSION) normalize = lambda diff: textwrap.dedent("\n".join(diff)) self.assertEqual(normalize(tree1.ASTdiff(tree1)), src1) self.assertEqual(normalize(tree2.ASTdiff(tree2)), src2) diff_pattern = r"(?s)- b.*\+ b" self.assertRegexpMatches(normalize(tree1.ASTdiff(tree2)), diff_pattern) self.assertRegexpMatches(normalize(tree2.ASTdiff(tree1)), diff_pattern)
def test_strict(self): ast = parser.parse_string(pytd_src(""" import typing T = TypeVar('T') class list(typing.Generic[T], object): pass class A(): pass class B(A): pass class `~unknown0`(): pass a = ... # type: A def left() -> `~unknown0`: ... def right() -> list[A]: ... """), options=self.options) ast = self.LinkAgainstSimpleBuiltins(ast) m = type_match.TypeMatch(type_match.get_all_subclasses([ast])) left, right = ast.Lookup("left"), ast.Lookup("right") unknown0 = escape.unknown(0) self.assertEqual( m.match(left, right, {}), booleq.And((booleq.Eq(unknown0, "list"), booleq.Eq(f"{unknown0}.list.T", "A"))))
def _TestTypeParameters(self, reverse=False): ast = parser.parse_string(pytd_src(""" from typing import Any, Generic class `~unknown0`(): def next(self) -> Any: ... T = TypeVar('T') class A(Generic[T], object): def next(self) -> Any: ... class B(): pass def left(x: `~unknown0`) -> Any: ... def right(x: A[B]) -> Any: ... """), options=self.options) ast = self.LinkAgainstSimpleBuiltins(ast) m = type_match.TypeMatch() left, right = ast.Lookup("left"), ast.Lookup("right") match = m.match(right, left, {}) if reverse else m.match( left, right, {}) unknown0 = escape.unknown(0) self.assertEqual( match, booleq.And((booleq.Eq(unknown0, "A"), booleq.Eq(f"{unknown0}.A.T", "B")))) self.assertIn(f"{unknown0}.A.T", m.solver.variables)
def PicklePyi(self, src, module_name): src = textwrap.dedent(src) ast = parser.parse_string( src, options=parser.PyiOptions.from_toplevel_options(self.options)) ast = ast.Visit( visitors.LookupBuiltins(self.loader.builtins, full_names=False)) return self._Pickle(ast, module_name)
def parse_type_definition(pyi_subdir, module, python_version): """Load and parse a *.pyi from typeshed. Args: pyi_subdir: the directory where the module should be found. module: the module name (without any file extension) python_version: sys.version_info[:2] Returns: None if the module doesn't have a definition. Else a tuple of the filename and the AST of the module. """ assert python_version typeshed = _get_typeshed() try: filename, src = typeshed.get_module_file(pyi_subdir, module, python_version) except IOError: return None ast = parser.parse_string(src, filename=filename, name=module, python_version=python_version) return filename, ast
def main(): argument_parser = make_parser() opts = argument_parser.parse_args() python_version = utils.version_from_string(opts.python_version) try: utils.validate_version(python_version) except utils.UsageError as e: sys.stderr.write("Usage error: %s\n" % utils.message(e)) sys.exit(1) with open(opts.input) as fi: sourcecode = fi.read() try: parsed = parser.parse_string(sourcecode, filename=opts.input, python_version=python_version) except parser.ParseError as e: sys.stderr.write(str(e)) sys.exit(1) if opts.optimize: parsed = optimize.Optimize(parsed, builtins.GetBuiltinsPyTD(python_version), lossy=opts.lossy, use_abcs=opts.use_abcs, max_union=opts.max_union, remove_mutable=opts.remove_mutable, can_do_lookup=False) if opts.output is not None: out_text = pytd_utils.Print(parsed, opts.multiline_args) if opts.output == "-": sys.stdout.write(out_text) else: with open(opts.output, "w") as out: out.write(out_text)
def ParsePyTD(src=None, filename=None, python_version=None, module=None, lookup_classes=False): """Parse pytd sourcecode and do name lookup for builtins. This loads a pytd and also makes sure that all names are resolved (i.e., that all primitive types in the AST are ClassType, and not NameType). Args: src: PyTD source code. filename: The filename the source code is from. python_version: The Python version to parse the pytd for. module: The name of the module we're parsing. lookup_classes: If we should also lookup the class of every ClassType. Returns: A pytd.TypeDeclUnit. """ assert python_version if src is None: with open(filename, "rb") as fi: src = fi.read() ast = parser.parse_string(src, filename=filename, name=module, python_version=python_version) if lookup_classes: ast = visitors.LookupClasses(ast, GetBuiltinsPyTD(python_version)) return ast
def check(self, src, expected=None, prologue=None, name=None, version=None, platform=None): """Check the parsing of src. This checks that parsing the source and then printing the resulting AST results in the expected text. Args: src: A source string. expected: Optional expected result string. If not provided, src is used instead. The special value IGNORE can be used to skip checking the parsed results against expected text. prologue: An optional prologue to be prepended to the expected text before comparisson. Useful for imports that are introduced during printing the AST. name: The name of the module. version: A python version tuple (None for default value). platform: A platform string (None for default value). Returns: The parsed pytd.TypeDeclUnit. """ src = textwrap.dedent(src) ast = parser.parse_string(src, name=name, python_version=version, platform=platform) actual = pytd.Print(ast) if expected != IGNORE: expected = src if expected is None else textwrap.dedent(expected) if prologue: expected = "%s\n\n%s" % (textwrap.dedent(prologue), expected) self.assertMultiLineEqual(expected, actual) return ast
def assertTypesMatchPytd(self, ty, pytd_src, version=None): """Parses pytd_src and compares with ty.""" # TODO(pludemann): This is a copy of pytd.parse.parser_test_base.Parse() # TODO(pludemann): Consider using the pytd_tree to call # assertHasOnlySignatures (or similar) to guard against the # inferencer adding additional but harmless calls. pytd_tree = parser.parse_string(textwrap.dedent(pytd_src), python_version=version) pytd_tree = pytd_tree.Visit( visitors.LookupBuiltins(builtins.GetBuiltinsAndTyping()[0], full_names=False)) pytd_tree = pytd_tree.Visit(visitors.LookupLocalTypes()) pytd_tree = pytd_tree.Visit(visitors.ClassTypeToNamedType()) pytd_tree = pytd_tree.Visit( visitors.CanonicalOrderingVisitor(sort_signatures=True)) pytd_tree.Visit(visitors.VerifyVisitor()) ty = ty.Visit(visitors.ClassTypeToNamedType()) ty = ty.Visit(visitors.AdjustSelf(force=True)) ty = ty.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True)) ty.Visit(visitors.VerifyVisitor()) ty_src = pytd.Print(ty) + "\n" pytd_tree_src = pytd.Print(pytd_tree) + "\n" log.info("========== result ==========") _LogLines(log.info, ty_src) log.info("========== expected ==========") _LogLines(log.info, pytd_tree_src) log.info("==============================") # In the diff output, mark expected with "-" and actual with "+". # (In other words, display a change from "working" to "broken") self.assertMultiLineEqual(pytd_tree_src, ty_src)
def test_strict(self): ast = parser.parse_string(textwrap.dedent(""" import typing T = TypeVar('T') class list(typing.Generic[T], object): pass class A(): pass class B(A): pass class `~unknown0`(): pass a = ... # type: A def left() -> `~unknown0` def right() -> list[A] """), python_version=self.python_version) ast = self.LinkAgainstSimpleBuiltins(ast) m = type_match.TypeMatch(type_match.get_all_subclasses([ast])) left, right = ast.Lookup("left"), ast.Lookup("right") self.assertEqual( m.match(left, right, {}), booleq.And((booleq.Eq("~unknown0", "list"), booleq.Eq("~unknown0.list.T", "A"))))
def assertTypesMatchPytd(self, ty, pytd_src): """Parses pytd_src and compares with ty.""" pytd_tree = parser.parse_string( textwrap.dedent(pytd_src), options=parser.PyiOptions(python_version=self.python_version)) pytd_tree = pytd_tree.Visit( visitors.LookupBuiltins(self.loader.builtins, full_names=False)) pytd_tree = pytd_tree.Visit(visitors.LookupLocalTypes()) pytd_tree = pytd_tree.Visit(visitors.ClassTypeToNamedType()) pytd_tree = pytd_tree.Visit( visitors.CanonicalOrderingVisitor(sort_signatures=True)) pytd_tree.Visit(visitors.VerifyVisitor()) ty = ty.Visit(visitors.ClassTypeToNamedType()) ty = ty.Visit(visitors.AdjustSelf()) ty = ty.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True)) ty.Visit(visitors.VerifyVisitor()) ty_src = pytd_utils.Print(ty) + "\n" pytd_tree_src = pytd_utils.Print(pytd_tree) + "\n" log.info("========== result ==========") _LogLines(log.info, ty_src) log.info("========== expected ==========") _LogLines(log.info, pytd_tree_src) log.info("==============================") # In the diff output, mark expected with "-" and actual with "+". # (In other words, display a change from "working" to "broken") self.assertMultiLineEqual(pytd_tree_src, ty_src)
def test_optional(self): ast = parser.parse_string(textwrap.dedent(""" def left(a: int) -> int: ... def right(a: int, *args) -> int: ... """), python_version=self.python_version) m = type_match.TypeMatch() self.assertEqual(m.match(ast.Lookup("left"), ast.Lookup("right"), {}), booleq.TRUE)
def testOptional(self): ast = parser.parse_string(textwrap.dedent(""" def left(a: int) -> int def right(a: int, ...) -> int """), python_version=self.PYTHON_VERSION) m = type_match.TypeMatch() self.assertEqual(m.match(ast.Lookup("left"), ast.Lookup("right"), {}), booleq.TRUE)
def test_astdiff(self): src1 = textwrap.dedent(""" a: int b: str""").lstrip() src2 = textwrap.dedent(""" a: int b: float""").lstrip() tree1 = parser.parse_string(src1, python_version=self.python_version) tree2 = parser.parse_string(src2, python_version=self.python_version) normalize = lambda diff: textwrap.dedent("\n".join(diff)) self.assertEqual(normalize(pytd_utils.ASTdiff(tree1, tree1)), src1) self.assertEqual(normalize(pytd_utils.ASTdiff(tree2, tree2)), src2) diff_pattern = r"(?s)- b.*\+ b" six.assertRegex(self, normalize(pytd_utils.ASTdiff(tree1, tree2)), diff_pattern) six.assertRegex(self, normalize(pytd_utils.ASTdiff(tree2, tree1)), diff_pattern)
def testReturn(self): ast = parser.parse_string(textwrap.dedent(""" def left(a: int) -> float def right(a: int) -> int """)) m = type_match.TypeMatch() self.assertNotEqual(m.match(ast.Lookup("left"), ast.Lookup("right"), {}), booleq.TRUE)
def testASTeq(self): # This creates two ASts that are equivalent but whose sources are slightly # different. The union types are different (int,str) vs (str,int) but the # ordering is ignored when testing for equality (which ASTeq uses). src1 = textwrap.dedent(""" def foo(a: int or str) -> C T = TypeVar('T') class C(typing.Generic[T], object): def bar(x: T) -> NoneType CONSTANT = ... # type: C[float] """) src2 = textwrap.dedent(""" CONSTANT = ... # type: C[float] T = TypeVar('T') class C(typing.Generic[T], object): def bar(x: T) -> NoneType def foo(a: str or int) -> C """) tree1 = parser.parse_string(src1) tree2 = parser.parse_string(src2) tree1.Visit(visitors.VerifyVisitor()) tree2.Visit(visitors.VerifyVisitor()) self.assertTrue(tree1.constants) self.assertTrue(tree1.classes) self.assertTrue(tree1.functions) self.assertTrue(tree2.constants) self.assertTrue(tree2.classes) self.assertTrue(tree2.functions) self.assertIsInstance(tree1, pytd.TypeDeclUnit) self.assertIsInstance(tree2, pytd.TypeDeclUnit) # For the ==, != tests, TypeDeclUnit uses identity self.assertTrue(tree1 == tree1) self.assertTrue(tree2 == tree2) self.assertFalse(tree1 == tree2) self.assertFalse(tree2 == tree1) self.assertFalse(tree1 != tree1) self.assertFalse(tree2 != tree2) self.assertTrue(tree1 != tree2) self.assertTrue(tree2 != tree1) self.assertEqual(tree1, tree1) self.assertEqual(tree2, tree2) self.assertNotEqual(tree1, tree2) self.assertTrue(tree1.ASTeq(tree2)) self.assertTrue(tree1.ASTeq(tree1)) self.assertTrue(tree2.ASTeq(tree1)) self.assertTrue(tree2.ASTeq(tree2))
def namedtuple_ast(name, fields, defaults, options): """Make an AST with a namedtuple definition for the given name and fields. Args: name: The namedtuple name. fields: The namedtuple fields. defaults: Sequence of booleans, whether each field has a default. options: A config.Options object. Returns: A pytd.TypeDeclUnit with the namedtuple definition in its classes. """ typevar = visitors.CreateTypeParametersForSignatures.PREFIX + name num_fields = len(fields) field_defs = "\n ".join( "%s = ... # type: typing.Any" % field for field in fields) fields_as_parameters = "".join( ", " + field + (" = ..." if default else "") for field, default in zip(fields, defaults)) field_names_as_strings = ", ".join(repr(field) for field in fields) if options.strict_namedtuple_checks: tuple_superclass_type = "typing.Tuple[{}]".format( _repeat_type("typing.Any", num_fields)) else: tuple_superclass_type = "tuple" nt = textwrap.dedent(""" {typevar} = TypeVar("{typevar}", bound={name}) class {name}({tuple_superclass_type}): __dict__ = ... # type: collections.OrderedDict[str, typing.Any] __slots__ = [{field_names_as_strings}] _fields = ... # type: typing.Tuple[{repeat_str}] {field_defs} def __getnewargs__(self) -> typing.Tuple[{repeat_any}]: ... def __getstate__(self) -> None: ... def __init__(self, *args, **kwargs) -> None: ... def __new__( cls: typing.Type[{typevar}]{fields_as_parameters}) -> {typevar}: ... def _asdict(self) -> collections.OrderedDict[str, typing.Any]: ... @classmethod def _make(cls: typing.Type[{typevar}], iterable: typing.Iterable, new = ..., len: typing.Callable[[typing.Sized], int] = ... ) -> {typevar}: ... def _replace(self: {typevar}, **kwds) -> {typevar}: ... """).format( typevar=typevar, name=name, repeat_str=_repeat_type("str", num_fields), tuple_superclass_type=tuple_superclass_type, field_defs=field_defs, repeat_any=_repeat_type("typing.Any", num_fields), fields_as_parameters=fields_as_parameters, field_names_as_strings=field_names_as_strings) return parser.parse_string( nt, options=parser.PyiOptions.from_toplevel_options(options))