def testLookupTwoStarAliasesWithDefaultPyi(self):
src1 = "def __getattr__(name) -> ?"
src2 = "def __getattr__(name) -> ?"
src3 = textwrap.dedent("""
from foo import *
from bar import *
""")
ast1 = self.Parse(src1).Replace(name="foo").Visit(
visitors.AddNamePrefix())
ast2 = self.Parse(src2).Replace(name="bar").Visit(
visitors.AddNamePrefix())
ast3 = self.Parse(src3).Replace(name="baz").Visit(
visitors.AddNamePrefix())
ast3 = ast3.Visit(
visitors.LookupExternalTypes(
{
"foo": ast1,
"bar": ast2,
"baz": ast3
}, self_name="baz"))
self.assertMultiLineEqual(
pytd_utils.Print(ast3),
textwrap.dedent("""
from typing import Any
def baz.__getattr__(name) -> Any: ...
""").strip())
def testLookupTwoStarAliasesWithSameClass(self):
src1 = "class A(object): ..."
src2 = "class A(object): ..."
src3 = textwrap.dedent("""
from foo import *
from bar import *
""")
ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix())
ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix())
ast3 = self.Parse(src3).Replace(name="baz").Visit(visitors.AddNamePrefix())
self.assertRaises(KeyError, ast3.Visit, visitors.LookupExternalTypes(
{"foo": ast1, "bar": ast2, "baz": ast3}, self_name="baz"))
def testLookupTwoStarAliasesWithDifferentGetAttrs(self):
src1 = "def __getattr__(name) -> int"
src2 = "def __getattr__(name) -> str"
src3 = textwrap.dedent("""
from foo import *
from bar import *
""")
ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix())
ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix())
ast3 = self.Parse(src3).Replace(name="baz").Visit(visitors.AddNamePrefix())
self.assertRaises(KeyError, ast3.Visit, visitors.LookupExternalTypes(
{"foo": ast1, "bar": ast2, "baz": ast3}, self_name="baz"))
def test_lookup_two_star_aliases_with_same_class(self):
src1 = "class A: ..."
src2 = "class A: ..."
src3 = textwrap.dedent("""
from foo import *
from bar import *
""")
ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix())
ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix())
ast3 = self.Parse(src3).Replace(name="baz").Visit(visitors.AddNamePrefix())
self.assertRaises(KeyError, ast3.Visit, visitors.LookupExternalTypes(
{"foo": ast1, "bar": ast2, "baz": ast3}, self_name="baz"))
def testLookupStarAliasWithDifferentGetAttr(self):
src1 = "def __getattr__(name) -> int"
src2 = textwrap.dedent("""
from foo import *
def __getattr__(name) -> str
""")
ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix())
ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix())
ast2 = ast2.Visit(visitors.LookupExternalTypes(
{"foo": ast1, "bar": ast2}, self_name="bar"))
self.assertMultiLineEqual(pytd.Print(ast2), textwrap.dedent("""\
def bar.__getattr__(name) -> str: ..."""))
def testLookupTwoStarAliases(self):
src1 = "class A(object): ..."
src2 = "class B(object): ..."
src3 = textwrap.dedent("""
from foo import *
from bar import *
""")
ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix())
ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix())
ast3 = self.Parse(src3).Replace(name="baz").Visit(visitors.AddNamePrefix())
ast3 = ast3.Visit(visitors.LookupExternalTypes(
{"foo": ast1, "bar": ast2, "baz": ast3}, self_name="baz"))
self.assertSetEqual({a.name for a in ast3.aliases}, {"baz.A", "baz.B"})
def test_lookup_star_alias_with_different_getattr(self):
src1 = "def __getattr__(name) -> int: ..."
src2 = textwrap.dedent("""
from foo import *
def __getattr__(name) -> str: ...
""")
ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix())
ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix())
ast2 = ast2.Visit(visitors.LookupExternalTypes(
{"foo": ast1, "bar": ast2}, self_name="bar"))
self.assertMultiLineEqual(pytd_utils.Print(ast2), textwrap.dedent("""
def bar.__getattr__(name) -> str: ...
""").strip())
def testAddNamePrefixTwice(self):
src = textwrap.dedent("""
from typing import TypeVar
x = ... # type: ?
T = TypeVar("T")
class X(Generic[T]): ...
""")
tree = self.Parse(src)
tree = tree.Replace(name="foo").Visit(visitors.AddNamePrefix())
tree = tree.Replace(name="foo").Visit(visitors.AddNamePrefix())
self.assertIsNotNone(tree.Lookup("foo.foo.x"))
self.assertEqual("foo.foo", tree.Lookup("foo.foo.T").scope)
self.assertEqual("foo.foo.X",
tree.Lookup("foo.foo.X").template[0].type_param.scope)
def test_lookup_star_alias_with_duplicate_class(self):
src1 = "class A: ..."
src2 = textwrap.dedent("""
from foo import *
class A:
x = ... # type: int
""")
ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix())
ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix())
ast2 = ast2.Visit(visitors.LookupExternalTypes(
{"foo": ast1, "bar": ast2}, self_name="bar"))
self.assertMultiLineEqual(pytd_utils.Print(ast2), textwrap.dedent("""
class bar.A:
x: int
""").strip())
def testLookupStarAliasWithDuplicateClass(self):
src1 = "class A(object): ..."
src2 = textwrap.dedent("""
from foo import *
class A(object):
x = ... # type: int
""")
ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix())
ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix())
ast2 = ast2.Visit(visitors.LookupExternalTypes(
{"foo": ast1, "bar": ast2}, self_name="bar"))
self.assertMultiLineEqual(pytd.Print(ast2), textwrap.dedent("""
class bar.A(object):
x: int
""").strip())
def post_process_ast(ast, src, name=None):
"""Post-process the parsed AST."""
ast = definitions.finalize_ast(ast)
ast = ast.Visit(pep484.ConvertTypingToNative(name))
if name:
ast = ast.Replace(name=name)
ast = ast.Visit(visitors.AddNamePrefix())
else:
# If there's no unique name, hash the sourcecode.
ast = ast.Replace(name=hashlib.md5(src.encode("utf-8")).hexdigest())
ast = ast.Visit(visitors.StripExternalNamePrefix())
# Now that we have resolved external names, process any class decorators that
# do code generation.
try:
ast = ast.Visit(decorate.DecorateClassVisitor())
except TypeError as e:
# Convert errors into ParseError. Unfortunately we no longer have location
# information if an error is raised during transformation of a class node.
raise ParseError.from_exc(e)
# Typeshed files that explicitly import and refer to "__builtin__" need to
# have that rewritten to builtins
ast = ast.Visit(visitors.RenameBuiltinsPrefix())
return ast
def test_add_name_prefix_on_nested_class_outside_ref(self):
src = textwrap.dedent("""
class A:
class B: ...
b: A.B
C = A.B
def f(x: A.B) -> A.B: ...
class D:
b: A.B
def f(self, x: A.B) -> A.B: ...
""")
expected = textwrap.dedent("""
from typing import Type
foo.b: foo.A.B
foo.C: Type[foo.A.B]
class foo.A:
class foo.A.B: ...
class foo.D:
b: foo.A.B
def f(self, x: foo.A.B) -> foo.A.B: ...
def foo.f(x: foo.A.B) -> foo.A.B: ...
""").strip()
self.assertMultiLineEqual(
expected,
pytd_utils.Print(
self.Parse(src).Replace(name="foo").Visit(
visitors.AddNamePrefix())))
def testAddNamePrefixOnNestedClassOutsideRef(self):
src = textwrap.dedent("""
class A:
class B: ...
b: A.B
C = A.B
def f(x: A.B) -> A.B: ...
class D:
b: A.B
def f(self, x: A.B) -> A.B: ...
""")
expected = textwrap.dedent("""\
from typing import Type
foo.b: foo.A.B
foo.C: Type[foo.A.B]
class foo.A:
class foo.A.B: ...
class foo.D:
b: foo.A.B
def f(self, x: foo.A.B) -> foo.A.B: ...
def foo.f(x: foo.A.B) -> foo.A.B: ...""")
self.assertMultiLineEqual(expected, pytd.Print(
self.Parse(src).Replace(name="foo").Visit(visitors.AddNamePrefix())))
def testLookupStarAlias(self):
src1 = textwrap.dedent("""
x = ... # type: int
T = TypeVar("T")
class A(object): ...
def f(x: T) -> T: ...
B = A
""")
src2 = "from foo import *"
ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix())
ast2 = self.Parse(src2).Replace(name="bar").Visit(visitors.AddNamePrefix())
ast2 = ast2.Visit(visitors.LookupExternalTypes(
{"foo": ast1, "bar": ast2}, self_name="bar"))
self.assertEqual("bar", ast2.name)
self.assertSetEqual({a.name for a in ast2.aliases},
{"bar.x", "bar.T", "bar.A", "bar.f", "bar.B"})
def parse(self, src, name, filename):
"""Parse a PYI file and return the corresponding AST.
Note that parse() should be called exactly once per _Parser instance. It
holds aggregated state during parsing and is not designed to be reused.
Args:
src: The source text to parse.
name: The name of the module to be created.
filename: The name of the source file.
Returns:
A pytd.TypeDeclUnit() representing the parsed pyi.
Raises:
ParseError: If the PYI source could not be parsed.
"""
# Ensure instances do not get reused.
assert not self._used
self._used = True
self._filename = filename
self._ast_name = name
self._type_map = {}
try:
defs = parser_ext.parse(self, src)
ast = self._build_type_decl_unit(defs)
except ParseError as e:
if self._error_location:
line = e.line or self._error_location[0]
try:
text = src.splitlines()[line-1]
except IndexError:
text = None
raise ParseError(utils.message(e), line=line, filename=self._filename,
column=self._error_location[1], text=text)
else:
raise e
ast = ast.Visit(_PropertyToConstant())
ast = ast.Visit(_InsertTypeParameters())
# TODO(kramm): This is in the wrong place- it should happen after resolving
# local names, in load_pytd.
ast = ast.Visit(pep484.ConvertTypingToNative(name))
if name:
ast = ast.Replace(name=name)
ast = ast.Visit(visitors.AddNamePrefix())
else:
# If there's no unique name, hash the sourcecode.
ast = ast.Replace(name=hashlib.md5(src.encode("utf-8")).hexdigest())
ast = ast.Visit(visitors.StripExternalNamePrefix())
# Typeshed files that explicitly import and refer to "builtins" need to have
# that rewritten to __builtin__
ast = ast.Visit(visitors.RenameBuiltinsPrefix())
return ast.Replace(is_package=utils.is_pyi_directory_init(filename))
def testAddNamePrefixOnClassType(self):
src = textwrap.dedent("""
x = ... # type: y
class Y: ...
""")
tree = self.Parse(src)
x = tree.Lookup("x")
x = x.Replace(type=pytd.ClassType("Y"))
tree = tree.Replace(constants=(x,), name="foo")
tree = tree.Visit(visitors.AddNamePrefix())
self.assertEqual("foo.Y", tree.Lookup("foo.x").type.name)
def test_lookup_star_alias_in_unnamed_module(self):
src1 = textwrap.dedent("""
class A: ...
""")
src2 = "from foo import *"
ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix())
ast2 = self.Parse(src2)
name = ast2.name
ast2 = ast2.Visit(visitors.LookupExternalTypes(
{"foo": ast1}, self_name=None))
self.assertEqual(name, ast2.name)
self.assertEqual(pytd_utils.Print(ast2), "from foo import A")
def testAddNamePrefixOnNestedClassMethod(self):
src = textwrap.dedent("""
class A:
class B:
def copy(self) -> A.B: ...
""")
expected = textwrap.dedent("""\
class foo.A:
class foo.A.B:
def copy(self) -> foo.A.B: ...""")
self.assertMultiLineEqual(expected, pytd.Print(
self.Parse(src).Replace(name="foo").Visit(visitors.AddNamePrefix())))
def test_add_name_prefix_on_nested_class_method(self):
src = textwrap.dedent("""
class A:
class B:
def copy(self) -> A.B: ...
""")
expected = textwrap.dedent("""
class foo.A:
class foo.A.B:
def copy(self) -> foo.A.B: ...
""").strip()
self.assertMultiLineEqual(expected, pytd_utils.Print(
self.Parse(src).Replace(name="foo").Visit(visitors.AddNamePrefix())))
def testLookupStarAliasWithDuplicateGetAttr(self):
src1 = "def __getattr__(name) -> ?"
src2 = textwrap.dedent("""
from foo import *
def __getattr__(name) -> ?
""")
ast1 = self.Parse(src1).Replace(name="foo").Visit(
visitors.AddNamePrefix())
ast2 = self.Parse(src2).Replace(name="bar").Visit(
visitors.AddNamePrefix())
ast2 = ast2.Visit(
visitors.LookupExternalTypes({
"foo": ast1,
"bar": ast2
},
self_name="bar"))
self.assertMultiLineEqual(
pytd_utils.Print(ast2),
textwrap.dedent("""
from typing import Any
def bar.__getattr__(name) -> Any: ...
""").strip())
def testLookupStarAliasInUnnamedModule(self):
src1 = textwrap.dedent("""
class A(object): ...
""")
src2 = "from foo import *"
ast1 = self.Parse(src1).Replace(name="foo").Visit(visitors.AddNamePrefix())
ast2 = self.Parse(src2)
name = ast2.name
ast2 = ast2.Visit(visitors.LookupExternalTypes(
{"foo": ast1}, self_name=None))
self.assertEqual(name, ast2.name)
self.assertMultiLineEqual(pytd.Print(ast2), textwrap.dedent("""\
import foo
A = foo.A"""))
def post_process_ast(ast, src, name=None):
"""Post-process the parsed AST."""
ast = definitions.finalize_ast(ast)
ast = ast.Visit(pep484.ConvertTypingToNative(name))
if name:
ast = ast.Replace(name=name)
ast = ast.Visit(visitors.AddNamePrefix())
else:
# If there's no unique name, hash the sourcecode.
ast = ast.Replace(name=hashlib.md5(src.encode("utf-8")).hexdigest())
ast = ast.Visit(visitors.StripExternalNamePrefix())
# Typeshed files that explicitly import and refer to "__builtin__" need to
# have that rewritten to builtins
return ast.Visit(visitors.RenameBuiltinsPrefix())
def testAddNamePrefixOnNestedClassAlias(self):
src = textwrap.dedent("""
class A:
class B:
class C: ...
D = A.B.C
""")
expected = textwrap.dedent("""
from typing import Type
class foo.A:
class foo.A.B:
class foo.A.B.C: ...
D: Type[foo.A.B.C]
""").strip()
self.assertMultiLineEqual(expected, pytd.Print(
self.Parse(src).Replace(name="foo").Visit(visitors.AddNamePrefix())))
def test_add_name_prefix(self):
src = textwrap.dedent("""
from typing import TypeVar
def f(a: T) -> T: ...
T = TypeVar("T")
class X(Generic[T]):
pass
""")
tree = self.Parse(src)
self.assertIsNone(tree.Lookup("T").scope)
self.assertEqual("X", tree.Lookup("X").template[0].type_param.scope)
tree = tree.Replace(name="foo").Visit(visitors.AddNamePrefix())
self.assertIsNotNone(tree.Lookup("foo.f"))
self.assertIsNotNone(tree.Lookup("foo.X"))
self.assertEqual("foo", tree.Lookup("foo.T").scope)
self.assertEqual("foo.X",
tree.Lookup("foo.X").template[0].type_param.scope)
def post_process_ast(ast, src, name=None):
"""Post-process the parsed AST."""
ast = definitions.finalize_ast(ast)
ast = ast.Visit(pep484.ConvertTypingToNative(name))
if name:
ast = ast.Replace(name=name)
ast = ast.Visit(visitors.AddNamePrefix())
else:
# If there's no unique name, hash the sourcecode.
ast = ast.Replace(name=hashlib.md5(src.encode("utf-8")).hexdigest())
ast = ast.Visit(visitors.StripExternalNamePrefix())
# Now that we have resolved external names, validate any class decorators that
# do code generation. (We will generate the class lazily, but we should check
# for errors at parse time so they can be reported early.)
try:
ast = ast.Visit(decorate.ValidateDecoratedClassVisitor())
except TypeError as e:
# Convert errors into ParseError. Unfortunately we no longer have location
# information if an error is raised during transformation of a class node.
raise ParseError.from_exc(e)
return ast
