def testLookupTwoStarAliasesWithDefaultPyi(self): src1 = "def __getattr__(name) -> ?" src2 = "def __getattr__(name) -> ?" src3 = textwrap.dedent(""" from foo import * from bar import * """) ast1 = self.Parse(src1).Replace(name="foo").Visit( visitors.AddNamePrefix()) ast2 = self.Parse(src2).Replace(name="bar").Visit( visitors.AddNamePrefix()) ast3 = self.Parse(src3).Replace(name="baz").Visit( visitors.AddNamePrefix()) ast3 = ast3.Visit( visitors.LookupExternalTypes( { "foo": ast1, "bar": ast2, "baz": ast3 }, self_name="baz")) self.assertMultiLineEqual( pytd_utils.Print(ast3), textwrap.dedent(""" from typing import Any def baz.__getattr__(name) -> Any: ... """).strip())
def testLookupTwoStarAliasesWithSameClass(self): src1 = "class A(object): ..." src2 = "class A(object): ..." src3 = textwrap.dedent(""" from foo import * from bar import * """) ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) ast3 = self.Parse(src3).Replace(name="baz").Visit(visitors.AddNamePrefix()) self.assertRaises(KeyError, ast3.Visit, visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2, "baz": ast3}, self_name="baz"))
def testLookupTwoStarAliasesWithDifferentGetAttrs(self): src1 = "def __getattr__(name) -> int" src2 = "def __getattr__(name) -> str" src3 = textwrap.dedent(""" from foo import * from bar import * """) ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) ast3 = self.Parse(src3).Replace(name="baz").Visit(visitors.AddNamePrefix()) self.assertRaises(KeyError, ast3.Visit, visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2, "baz": ast3}, self_name="baz"))
def test_lookup_two_star_aliases_with_same_class(self): src1 = "class A: ..." src2 = "class A: ..." src3 = textwrap.dedent(""" from foo import * from bar import * """) ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) ast3 = self.Parse(src3).Replace(name="baz").Visit(visitors.AddNamePrefix()) self.assertRaises(KeyError, ast3.Visit, visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2, "baz": ast3}, self_name="baz"))
def testLookupStarAliasWithDifferentGetAttr(self): src1 = "def __getattr__(name) -> int" src2 = textwrap.dedent(""" from foo import * def __getattr__(name) -> str """) ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) ast2 = ast2.Visit(visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2}, self_name="bar")) self.assertMultiLineEqual(pytd.Print(ast2), textwrap.dedent("""\ def bar.__getattr__(name) -> str: ..."""))
def testLookupTwoStarAliases(self): src1 = "class A(object): ..." src2 = "class B(object): ..." src3 = textwrap.dedent(""" from foo import * from bar import * """) ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) ast3 = self.Parse(src3).Replace(name="baz").Visit(visitors.AddNamePrefix()) ast3 = ast3.Visit(visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2, "baz": ast3}, self_name="baz")) self.assertSetEqual({a.name for a in ast3.aliases}, {"baz.A", "baz.B"})
def test_lookup_star_alias_with_different_getattr(self): src1 = "def __getattr__(name) -> int: ..." src2 = textwrap.dedent(""" from foo import * def __getattr__(name) -> str: ... """) ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) ast2 = ast2.Visit(visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2}, self_name="bar")) self.assertMultiLineEqual(pytd_utils.Print(ast2), textwrap.dedent(""" def bar.__getattr__(name) -> str: ... """).strip())
def testAddNamePrefixTwice(self): src = textwrap.dedent(""" from typing import TypeVar x = ... # type: ? T = TypeVar("T") class X(Generic[T]): ... """) tree = self.Parse(src) tree = tree.Replace(name="foo").Visit(visitors.AddNamePrefix()) tree = tree.Replace(name="foo").Visit(visitors.AddNamePrefix()) self.assertIsNotNone(tree.Lookup("foo.foo.x")) self.assertEqual("foo.foo", tree.Lookup("foo.foo.T").scope) self.assertEqual("foo.foo.X", tree.Lookup("foo.foo.X").template[0].type_param.scope)
def test_lookup_star_alias_with_duplicate_class(self): src1 = "class A: ..." src2 = textwrap.dedent(""" from foo import * class A: x = ... # type: int """) ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) ast2 = ast2.Visit(visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2}, self_name="bar")) self.assertMultiLineEqual(pytd_utils.Print(ast2), textwrap.dedent(""" class bar.A: x: int """).strip())
def testLookupStarAliasWithDuplicateClass(self): src1 = "class A(object): ..." src2 = textwrap.dedent(""" from foo import * class A(object): x = ... # type: int """) ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) ast2 = ast2.Visit(visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2}, self_name="bar")) self.assertMultiLineEqual(pytd.Print(ast2), textwrap.dedent(""" class bar.A(object): x: int """).strip())
def post_process_ast(ast, src, name=None): """Post-process the parsed AST.""" ast = definitions.finalize_ast(ast) ast = ast.Visit(pep484.ConvertTypingToNative(name)) if name: ast = ast.Replace(name=name) ast = ast.Visit(visitors.AddNamePrefix()) else: # If there's no unique name, hash the sourcecode. ast = ast.Replace(name=hashlib.md5(src.encode("utf-8")).hexdigest()) ast = ast.Visit(visitors.StripExternalNamePrefix()) # Now that we have resolved external names, process any class decorators that # do code generation. try: ast = ast.Visit(decorate.DecorateClassVisitor()) except TypeError as e: # Convert errors into ParseError. Unfortunately we no longer have location # information if an error is raised during transformation of a class node. raise ParseError.from_exc(e) # Typeshed files that explicitly import and refer to "__builtin__" need to # have that rewritten to builtins ast = ast.Visit(visitors.RenameBuiltinsPrefix()) return ast
def test_add_name_prefix_on_nested_class_outside_ref(self): src = textwrap.dedent(""" class A: class B: ... b: A.B C = A.B def f(x: A.B) -> A.B: ... class D: b: A.B def f(self, x: A.B) -> A.B: ... """) expected = textwrap.dedent(""" from typing import Type foo.b: foo.A.B foo.C: Type[foo.A.B] class foo.A: class foo.A.B: ... class foo.D: b: foo.A.B def f(self, x: foo.A.B) -> foo.A.B: ... def foo.f(x: foo.A.B) -> foo.A.B: ... """).strip() self.assertMultiLineEqual( expected, pytd_utils.Print( self.Parse(src).Replace(name="foo").Visit( visitors.AddNamePrefix())))
def testAddNamePrefixOnNestedClassOutsideRef(self): src = textwrap.dedent(""" class A: class B: ... b: A.B C = A.B def f(x: A.B) -> A.B: ... class D: b: A.B def f(self, x: A.B) -> A.B: ... """) expected = textwrap.dedent("""\ from typing import Type foo.b: foo.A.B foo.C: Type[foo.A.B] class foo.A: class foo.A.B: ... class foo.D: b: foo.A.B def f(self, x: foo.A.B) -> foo.A.B: ... def foo.f(x: foo.A.B) -> foo.A.B: ...""") self.assertMultiLineEqual(expected, pytd.Print( self.Parse(src).Replace(name="foo").Visit(visitors.AddNamePrefix())))
def testLookupStarAlias(self): src1 = textwrap.dedent(""" x = ... # type: int T = TypeVar("T") class A(object): ... def f(x: T) -> T: ... B = A """) src2 = "from foo import *" ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix()) ast2 = ast2.Visit(visitors.LookupExternalTypes( {"foo": ast1, "bar": ast2}, self_name="bar")) self.assertEqual("bar", ast2.name) self.assertSetEqual({a.name for a in ast2.aliases}, {"bar.x", "bar.T", "bar.A", "bar.f", "bar.B"})
def parse(self, src, name, filename): """Parse a PYI file and return the corresponding AST. Note that parse() should be called exactly once per _Parser instance. It holds aggregated state during parsing and is not designed to be reused. Args: src: The source text to parse. name: The name of the module to be created. filename: The name of the source file. Returns: A pytd.TypeDeclUnit() representing the parsed pyi. Raises: ParseError: If the PYI source could not be parsed. """ # Ensure instances do not get reused. assert not self._used self._used = True self._filename = filename self._ast_name = name self._type_map = {} try: defs = parser_ext.parse(self, src) ast = self._build_type_decl_unit(defs) except ParseError as e: if self._error_location: line = e.line or self._error_location[0] try: text = src.splitlines()[line-1] except IndexError: text = None raise ParseError(utils.message(e), line=line, filename=self._filename, column=self._error_location[1], text=text) else: raise e ast = ast.Visit(_PropertyToConstant()) ast = ast.Visit(_InsertTypeParameters()) # TODO(kramm): This is in the wrong place- it should happen after resolving # local names, in load_pytd. ast = ast.Visit(pep484.ConvertTypingToNative(name)) if name: ast = ast.Replace(name=name) ast = ast.Visit(visitors.AddNamePrefix()) else: # If there's no unique name, hash the sourcecode. ast = ast.Replace(name=hashlib.md5(src.encode("utf-8")).hexdigest()) ast = ast.Visit(visitors.StripExternalNamePrefix()) # Typeshed files that explicitly import and refer to "builtins" need to have # that rewritten to __builtin__ ast = ast.Visit(visitors.RenameBuiltinsPrefix()) return ast.Replace(is_package=utils.is_pyi_directory_init(filename))
def testAddNamePrefixOnClassType(self): src = textwrap.dedent(""" x = ... # type: y class Y: ... """) tree = self.Parse(src) x = tree.Lookup("x") x = x.Replace(type=pytd.ClassType("Y")) tree = tree.Replace(constants=(x,), name="foo") tree = tree.Visit(visitors.AddNamePrefix()) self.assertEqual("foo.Y", tree.Lookup("foo.x").type.name)
def test_lookup_star_alias_in_unnamed_module(self): src1 = textwrap.dedent(""" class A: ... """) src2 = "from foo import *" ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) ast2 = self.Parse(src2) name = ast2.name ast2 = ast2.Visit(visitors.LookupExternalTypes( {"foo": ast1}, self_name=None)) self.assertEqual(name, ast2.name) self.assertEqual(pytd_utils.Print(ast2), "from foo import A")
def testAddNamePrefixOnNestedClassMethod(self): src = textwrap.dedent(""" class A: class B: def copy(self) -> A.B: ... """) expected = textwrap.dedent("""\ class foo.A: class foo.A.B: def copy(self) -> foo.A.B: ...""") self.assertMultiLineEqual(expected, pytd.Print( self.Parse(src).Replace(name="foo").Visit(visitors.AddNamePrefix())))
def test_add_name_prefix_on_nested_class_method(self): src = textwrap.dedent(""" class A: class B: def copy(self) -> A.B: ... """) expected = textwrap.dedent(""" class foo.A: class foo.A.B: def copy(self) -> foo.A.B: ... """).strip() self.assertMultiLineEqual(expected, pytd_utils.Print( self.Parse(src).Replace(name="foo").Visit(visitors.AddNamePrefix())))
def testLookupStarAliasWithDuplicateGetAttr(self): src1 = "def __getattr__(name) -> ?" src2 = textwrap.dedent(""" from foo import * def __getattr__(name) -> ? """) ast1 = self.Parse(src1).Replace(name="foo").Visit( visitors.AddNamePrefix()) ast2 = self.Parse(src2).Replace(name="bar").Visit( visitors.AddNamePrefix()) ast2 = ast2.Visit( visitors.LookupExternalTypes({ "foo": ast1, "bar": ast2 }, self_name="bar")) self.assertMultiLineEqual( pytd_utils.Print(ast2), textwrap.dedent(""" from typing import Any def bar.__getattr__(name) -> Any: ... """).strip())
def testLookupStarAliasInUnnamedModule(self): src1 = textwrap.dedent(""" class A(object): ... """) src2 = "from foo import *" ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix()) ast2 = self.Parse(src2) name = ast2.name ast2 = ast2.Visit(visitors.LookupExternalTypes( {"foo": ast1}, self_name=None)) self.assertEqual(name, ast2.name) self.assertMultiLineEqual(pytd.Print(ast2), textwrap.dedent("""\ import foo A = foo.A"""))
def post_process_ast(ast, src, name=None): """Post-process the parsed AST.""" ast = definitions.finalize_ast(ast) ast = ast.Visit(pep484.ConvertTypingToNative(name)) if name: ast = ast.Replace(name=name) ast = ast.Visit(visitors.AddNamePrefix()) else: # If there's no unique name, hash the sourcecode. ast = ast.Replace(name=hashlib.md5(src.encode("utf-8")).hexdigest()) ast = ast.Visit(visitors.StripExternalNamePrefix()) # Typeshed files that explicitly import and refer to "__builtin__" need to # have that rewritten to builtins return ast.Visit(visitors.RenameBuiltinsPrefix())
def testAddNamePrefixOnNestedClassAlias(self): src = textwrap.dedent(""" class A: class B: class C: ... D = A.B.C """) expected = textwrap.dedent(""" from typing import Type class foo.A: class foo.A.B: class foo.A.B.C: ... D: Type[foo.A.B.C] """).strip() self.assertMultiLineEqual(expected, pytd.Print( self.Parse(src).Replace(name="foo").Visit(visitors.AddNamePrefix())))
def test_add_name_prefix(self): src = textwrap.dedent(""" from typing import TypeVar def f(a: T) -> T: ... T = TypeVar("T") class X(Generic[T]): pass """) tree = self.Parse(src) self.assertIsNone(tree.Lookup("T").scope) self.assertEqual("X", tree.Lookup("X").template[0].type_param.scope) tree = tree.Replace(name="foo").Visit(visitors.AddNamePrefix()) self.assertIsNotNone(tree.Lookup("foo.f")) self.assertIsNotNone(tree.Lookup("foo.X")) self.assertEqual("foo", tree.Lookup("foo.T").scope) self.assertEqual("foo.X", tree.Lookup("foo.X").template[0].type_param.scope)
def post_process_ast(ast, src, name=None): """Post-process the parsed AST.""" ast = definitions.finalize_ast(ast) ast = ast.Visit(pep484.ConvertTypingToNative(name)) if name: ast = ast.Replace(name=name) ast = ast.Visit(visitors.AddNamePrefix()) else: # If there's no unique name, hash the sourcecode. ast = ast.Replace(name=hashlib.md5(src.encode("utf-8")).hexdigest()) ast = ast.Visit(visitors.StripExternalNamePrefix()) # Now that we have resolved external names, validate any class decorators that # do code generation. (We will generate the class lazily, but we should check # for errors at parse time so they can be reported early.) try: ast = ast.Visit(decorate.ValidateDecoratedClassVisitor()) except TypeError as e: # Convert errors into ParseError. Unfortunately we no longer have location # information if an error is raised during transformation of a class node. raise ParseError.from_exc(e) return ast