def check_error(self, src, expected_line, message):
"""Check that parsing the src raises the expected error."""
with self.assertRaises(parser.ParseError) as e:
parser.parse_string(textwrap.dedent(src).lstrip(),
options=self.options)
self.assertRegex(utils.message(e.exception), re.escape(message))
self.assertEqual(expected_line, e.exception.line)
def parse():
try:
parser.parse_string(src)
except parser.ParseError:
# It is essential to clear the error, otherwise the system exc_info
# will hold references to lots of stuff hanging off the exception.
sys.exc_clear()
def setUp(self):
builtins = parser.parse_string(textwrap.dedent(_BUILTINS),
name="__builtin__",
python_version=self.PYTHON_VERSION)
typing = parser.parse_string("class Generic: ...", name="typing",
python_version=self.PYTHON_VERSION)
self.mini_builtins = pytd_utils.Concat(builtins, typing)
def assertInferredPyiEquals(self, expected_pyi=None, filename=None):
assert bool(expected_pyi) != bool(filename)
if filename:
with open(self._DataPath(filename), "r") as f:
expected_pyi = f.read()
self.assertTrue(
parser.parse_string(self.stdout).ASTeq(
parser.parse_string(expected_pyi)))
def check_error(self, src, expected_line, message):
"""Check that parsing the src raises the expected error."""
try:
parser.parse_string(textwrap.dedent(src))
self.fail("ParseError expected")
except parser.ParseError as e:
self.assertRegexpMatches(e.message, re.escape(message))
self.assertEquals(expected_line, e.line)
def setUp(self):
super().setUp()
builtins = parser.parse_string(textwrap.dedent(_BUILTINS),
name="builtins",
options=self.options)
typing = parser.parse_string("class Generic: ...",
name="typing",
options=self.options)
self.mini_builtins = pytd_utils.Concat(builtins, typing)
def parse():
try:
parser.parse_string(src, python_version=self.PYTHON_VERSION)
except parser.ParseError:
if six.PY2:
# It is essential to clear the error, otherwise the system exc_info
# will hold references to lots of stuff hanging off the exception.
# This happens only in Python2.
sys.exc_clear()
def setUp(self):
super(TestTypeMatch, self).setUp()
builtins = parser.parse_string(textwrap.dedent(_BUILTINS),
name="__builtin__",
python_version=self.python_version)
typing = parser.parse_string("class Generic: ...",
name="typing",
python_version=self.python_version)
self.mini_builtins = pytd_utils.Concat(builtins, typing)
def _test_parse(pyi_file):
python_version = (2, 7)
module = os.path.splitext(os.path.basename(pyi_file))[0]
if "__init__" == module:
module = os.path.basename(os.path.dirname(pyi_file))
with open(pyi_file) as f:
src = f.read()
parser.parse_string(src, filename=pyi_file, name=module,
python_version=python_version)
def assertInferredPyiEquals(self, expected_pyi=None, filename=None):
assert bool(expected_pyi) != bool(filename)
if filename:
with open(self._DataPath(filename), "r") as f:
expected_pyi = f.read()
message = ("\n==Expected pyi==\n" + expected_pyi +
"\n==Actual pyi==\n" + self.stdout)
self.assertTrue(
parser.parse_string(self.stdout).ASTeq(
parser.parse_string(expected_pyi)), message)
def test_asteq(self):
# This creates two ASts that are equivalent but whose sources are slightly
# different. The union types are different (int,str) vs (str,int) but the
# ordering is ignored when testing for equality (which ASTeq uses).
src1 = textwrap.dedent("""
from typing import Union
def foo(a: Union[int, str]) -> C: ...
T = TypeVar('T')
class C(typing.Generic[T], object):
def bar(x: T) -> NoneType: ...
CONSTANT = ... # type: C[float]
""")
src2 = textwrap.dedent("""
from typing import Union
CONSTANT = ... # type: C[float]
T = TypeVar('T')
class C(typing.Generic[T], object):
def bar(x: T) -> NoneType: ...
def foo(a: Union[str, int]) -> C: ...
""")
tree1 = parser.parse_string(src1, python_version=self.python_version)
tree2 = parser.parse_string(src2, python_version=self.python_version)
tree1.Visit(visitors.VerifyVisitor())
tree2.Visit(visitors.VerifyVisitor())
self.assertTrue(tree1.constants)
self.assertTrue(tree1.classes)
self.assertTrue(tree1.functions)
self.assertTrue(tree2.constants)
self.assertTrue(tree2.classes)
self.assertTrue(tree2.functions)
self.assertIsInstance(tree1, pytd.TypeDeclUnit)
self.assertIsInstance(tree2, pytd.TypeDeclUnit)
# For the ==, != tests, TypeDeclUnit uses identity
# pylint: disable=g-generic-assert
# pylint: disable=comparison-with-itself
self.assertTrue(tree1 == tree1)
self.assertTrue(tree2 == tree2)
self.assertFalse(tree1 == tree2)
self.assertFalse(tree2 == tree1)
self.assertFalse(tree1 != tree1)
self.assertFalse(tree2 != tree2)
self.assertTrue(tree1 != tree2)
self.assertTrue(tree2 != tree1)
# pylint: enable=g-generic-assert
# pylint: enable=comparison-with-itself
self.assertEqual(tree1, tree1)
self.assertEqual(tree2, tree2)
self.assertNotEqual(tree1, tree2)
self.assertTrue(pytd_utils.ASTeq(tree1, tree2))
self.assertTrue(pytd_utils.ASTeq(tree1, tree1))
self.assertTrue(pytd_utils.ASTeq(tree2, tree1))
self.assertTrue(pytd_utils.ASTeq(tree2, tree2))
def testInferToFile(self):
self.pytype_args[self._DataPath("simple.py")] = self.INCLUDE
pyi_file = self._TmpPath("simple.pyi")
self.pytype_args["--output"] = pyi_file
self._RunPytype(self.pytype_args)
self.assertOutputStateMatches(stdout=False,
stderr=False,
returncode=False)
with open(pyi_file, "r") as f:
pyi = f.read()
with open(self._DataPath("simple.pyi"), "r") as f:
expected_pyi = f.read()
self.assertTrue(
parser.parse_string(pyi).ASTeq(parser.parse_string(expected_pyi)))
def test_error_formatting(self):
src = """\
class Foo:
this is not valid"""
try:
parser.parse_string(textwrap.dedent(src), filename="foo.py")
self.fail("ParseError expected")
except parser.ParseError as e:
self.assertMultiLineEqual(textwrap.dedent("""\
File: "foo.py", line 2
this is not valid
^
ParseError: syntax error, unexpected NAME, expecting ':' or '='"""
), str(e))
def testASTdiff(self):
src1 = textwrap.dedent("""\
a = ... # type: int
b = ... # type: str""")
src2 = textwrap.dedent("""\
a = ... # type: int
b = ... # type: float""")
tree1 = parser.parse_string(src1, python_version=self.PYTHON_VERSION)
tree2 = parser.parse_string(src2, python_version=self.PYTHON_VERSION)
normalize = lambda diff: textwrap.dedent("\n".join(diff))
self.assertEqual(normalize(tree1.ASTdiff(tree1)), src1)
self.assertEqual(normalize(tree2.ASTdiff(tree2)), src2)
diff_pattern = r"(?s)- b.*\+ b"
self.assertRegexpMatches(normalize(tree1.ASTdiff(tree2)), diff_pattern)
self.assertRegexpMatches(normalize(tree2.ASTdiff(tree1)), diff_pattern)
def test_strict(self):
ast = parser.parse_string(pytd_src("""
import typing
T = TypeVar('T')
class list(typing.Generic[T], object):
pass
class A():
pass
class B(A):
pass
class `~unknown0`():
pass
a = ... # type: A
def left() -> `~unknown0`: ...
def right() -> list[A]: ...
"""),
options=self.options)
ast = self.LinkAgainstSimpleBuiltins(ast)
m = type_match.TypeMatch(type_match.get_all_subclasses([ast]))
left, right = ast.Lookup("left"), ast.Lookup("right")
unknown0 = escape.unknown(0)
self.assertEqual(
m.match(left, right, {}),
booleq.And((booleq.Eq(unknown0,
"list"), booleq.Eq(f"{unknown0}.list.T",
"A"))))
def _TestTypeParameters(self, reverse=False):
ast = parser.parse_string(pytd_src("""
from typing import Any, Generic
class `~unknown0`():
def next(self) -> Any: ...
T = TypeVar('T')
class A(Generic[T], object):
def next(self) -> Any: ...
class B():
pass
def left(x: `~unknown0`) -> Any: ...
def right(x: A[B]) -> Any: ...
"""),
options=self.options)
ast = self.LinkAgainstSimpleBuiltins(ast)
m = type_match.TypeMatch()
left, right = ast.Lookup("left"), ast.Lookup("right")
match = m.match(right, left, {}) if reverse else m.match(
left, right, {})
unknown0 = escape.unknown(0)
self.assertEqual(
match,
booleq.And((booleq.Eq(unknown0,
"A"), booleq.Eq(f"{unknown0}.A.T", "B"))))
self.assertIn(f"{unknown0}.A.T", m.solver.variables)
def PicklePyi(self, src, module_name):
src = textwrap.dedent(src)
ast = parser.parse_string(
src, options=parser.PyiOptions.from_toplevel_options(self.options))
ast = ast.Visit(
visitors.LookupBuiltins(self.loader.builtins, full_names=False))
return self._Pickle(ast, module_name)
def parse_type_definition(pyi_subdir, module, python_version):
"""Load and parse a *.pyi from typeshed.
Args:
pyi_subdir: the directory where the module should be found.
module: the module name (without any file extension)
python_version: sys.version_info[:2]
Returns:
None if the module doesn't have a definition.
Else a tuple of the filename and the AST of the module.
"""
assert python_version
typeshed = _get_typeshed()
try:
filename, src = typeshed.get_module_file(pyi_subdir, module,
python_version)
except IOError:
return None
ast = parser.parse_string(src,
filename=filename,
name=module,
python_version=python_version)
return filename, ast
def main():
argument_parser = make_parser()
opts = argument_parser.parse_args()
python_version = utils.version_from_string(opts.python_version)
try:
utils.validate_version(python_version)
except utils.UsageError as e:
sys.stderr.write("Usage error: %s\n" % utils.message(e))
sys.exit(1)
with open(opts.input) as fi:
sourcecode = fi.read()
try:
parsed = parser.parse_string(sourcecode, filename=opts.input,
python_version=python_version)
except parser.ParseError as e:
sys.stderr.write(str(e))
sys.exit(1)
if opts.optimize:
parsed = optimize.Optimize(parsed,
builtins.GetBuiltinsPyTD(python_version),
lossy=opts.lossy,
use_abcs=opts.use_abcs,
max_union=opts.max_union,
remove_mutable=opts.remove_mutable,
can_do_lookup=False)
if opts.output is not None:
out_text = pytd_utils.Print(parsed, opts.multiline_args)
if opts.output == "-":
sys.stdout.write(out_text)
else:
with open(opts.output, "w") as out:
out.write(out_text)
def ParsePyTD(src=None,
filename=None,
python_version=None,
module=None,
lookup_classes=False):
"""Parse pytd sourcecode and do name lookup for builtins.
This loads a pytd and also makes sure that all names are resolved (i.e.,
that all primitive types in the AST are ClassType, and not NameType).
Args:
src: PyTD source code.
filename: The filename the source code is from.
python_version: The Python version to parse the pytd for.
module: The name of the module we're parsing.
lookup_classes: If we should also lookup the class of every ClassType.
Returns:
A pytd.TypeDeclUnit.
"""
assert python_version
if src is None:
with open(filename, "rb") as fi:
src = fi.read()
ast = parser.parse_string(src,
filename=filename,
name=module,
python_version=python_version)
if lookup_classes:
ast = visitors.LookupClasses(ast, GetBuiltinsPyTD(python_version))
return ast
def check(self, src, expected=None, prologue=None, name=None,
version=None, platform=None):
"""Check the parsing of src.
This checks that parsing the source and then printing the resulting
AST results in the expected text.
Args:
src: A source string.
expected: Optional expected result string. If not provided, src is
used instead. The special value IGNORE can be used to skip
checking the parsed results against expected text.
prologue: An optional prologue to be prepended to the expected text
before comparisson. Useful for imports that are introduced during
printing the AST.
name: The name of the module.
version: A python version tuple (None for default value).
platform: A platform string (None for default value).
Returns:
The parsed pytd.TypeDeclUnit.
"""
src = textwrap.dedent(src)
ast = parser.parse_string(src, name=name, python_version=version,
platform=platform)
actual = pytd.Print(ast)
if expected != IGNORE:
expected = src if expected is None else textwrap.dedent(expected)
if prologue:
expected = "%s\n\n%s" % (textwrap.dedent(prologue), expected)
self.assertMultiLineEqual(expected, actual)
return ast
def assertTypesMatchPytd(self, ty, pytd_src, version=None):
"""Parses pytd_src and compares with ty."""
# TODO(pludemann): This is a copy of pytd.parse.parser_test_base.Parse()
# TODO(pludemann): Consider using the pytd_tree to call
# assertHasOnlySignatures (or similar) to guard against the
# inferencer adding additional but harmless calls.
pytd_tree = parser.parse_string(textwrap.dedent(pytd_src),
python_version=version)
pytd_tree = pytd_tree.Visit(
visitors.LookupBuiltins(builtins.GetBuiltinsAndTyping()[0],
full_names=False))
pytd_tree = pytd_tree.Visit(visitors.LookupLocalTypes())
pytd_tree = pytd_tree.Visit(visitors.ClassTypeToNamedType())
pytd_tree = pytd_tree.Visit(
visitors.CanonicalOrderingVisitor(sort_signatures=True))
pytd_tree.Visit(visitors.VerifyVisitor())
ty = ty.Visit(visitors.ClassTypeToNamedType())
ty = ty.Visit(visitors.AdjustSelf(force=True))
ty = ty.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True))
ty.Visit(visitors.VerifyVisitor())
ty_src = pytd.Print(ty) + "\n"
pytd_tree_src = pytd.Print(pytd_tree) + "\n"
log.info("========== result ==========")
_LogLines(log.info, ty_src)
log.info("========== expected ==========")
_LogLines(log.info, pytd_tree_src)
log.info("==============================")
# In the diff output, mark expected with "-" and actual with "+".
# (In other words, display a change from "working" to "broken")
self.assertMultiLineEqual(pytd_tree_src, ty_src)
def test_strict(self):
ast = parser.parse_string(textwrap.dedent("""
import typing
T = TypeVar('T')
class list(typing.Generic[T], object):
pass
class A():
pass
class B(A):
pass
class `~unknown0`():
pass
a = ... # type: A
def left() -> `~unknown0`
def right() -> list[A]
"""),
python_version=self.python_version)
ast = self.LinkAgainstSimpleBuiltins(ast)
m = type_match.TypeMatch(type_match.get_all_subclasses([ast]))
left, right = ast.Lookup("left"), ast.Lookup("right")
self.assertEqual(
m.match(left, right, {}),
booleq.And((booleq.Eq("~unknown0",
"list"), booleq.Eq("~unknown0.list.T",
"A"))))
def assertTypesMatchPytd(self, ty, pytd_src):
"""Parses pytd_src and compares with ty."""
pytd_tree = parser.parse_string(
textwrap.dedent(pytd_src),
options=parser.PyiOptions(python_version=self.python_version))
pytd_tree = pytd_tree.Visit(
visitors.LookupBuiltins(self.loader.builtins, full_names=False))
pytd_tree = pytd_tree.Visit(visitors.LookupLocalTypes())
pytd_tree = pytd_tree.Visit(visitors.ClassTypeToNamedType())
pytd_tree = pytd_tree.Visit(
visitors.CanonicalOrderingVisitor(sort_signatures=True))
pytd_tree.Visit(visitors.VerifyVisitor())
ty = ty.Visit(visitors.ClassTypeToNamedType())
ty = ty.Visit(visitors.AdjustSelf())
ty = ty.Visit(visitors.CanonicalOrderingVisitor(sort_signatures=True))
ty.Visit(visitors.VerifyVisitor())
ty_src = pytd_utils.Print(ty) + "\n"
pytd_tree_src = pytd_utils.Print(pytd_tree) + "\n"
log.info("========== result ==========")
_LogLines(log.info, ty_src)
log.info("========== expected ==========")
_LogLines(log.info, pytd_tree_src)
log.info("==============================")
# In the diff output, mark expected with "-" and actual with "+".
# (In other words, display a change from "working" to "broken")
self.assertMultiLineEqual(pytd_tree_src, ty_src)
def test_optional(self):
ast = parser.parse_string(textwrap.dedent("""
def left(a: int) -> int: ...
def right(a: int, *args) -> int: ...
"""), python_version=self.python_version)
m = type_match.TypeMatch()
self.assertEqual(m.match(ast.Lookup("left"), ast.Lookup("right"), {}),
booleq.TRUE)
def testOptional(self):
ast = parser.parse_string(textwrap.dedent("""
def left(a: int) -> int
def right(a: int, ...) -> int
"""), python_version=self.PYTHON_VERSION)
m = type_match.TypeMatch()
self.assertEqual(m.match(ast.Lookup("left"), ast.Lookup("right"), {}),
booleq.TRUE)
def test_astdiff(self):
src1 = textwrap.dedent("""
a: int
b: str""").lstrip()
src2 = textwrap.dedent("""
a: int
b: float""").lstrip()
tree1 = parser.parse_string(src1, python_version=self.python_version)
tree2 = parser.parse_string(src2, python_version=self.python_version)
normalize = lambda diff: textwrap.dedent("\n".join(diff))
self.assertEqual(normalize(pytd_utils.ASTdiff(tree1, tree1)), src1)
self.assertEqual(normalize(pytd_utils.ASTdiff(tree2, tree2)), src2)
diff_pattern = r"(?s)- b.*\+ b"
six.assertRegex(self, normalize(pytd_utils.ASTdiff(tree1, tree2)),
diff_pattern)
six.assertRegex(self, normalize(pytd_utils.ASTdiff(tree2, tree1)),
diff_pattern)
def testReturn(self):
ast = parser.parse_string(textwrap.dedent("""
def left(a: int) -> float
def right(a: int) -> int
"""))
m = type_match.TypeMatch()
self.assertNotEqual(m.match(ast.Lookup("left"), ast.Lookup("right"), {}),
booleq.TRUE)
def testASTeq(self):
# This creates two ASts that are equivalent but whose sources are slightly
# different. The union types are different (int,str) vs (str,int) but the
# ordering is ignored when testing for equality (which ASTeq uses).
src1 = textwrap.dedent("""
def foo(a: int or str) -> C
T = TypeVar('T')
class C(typing.Generic[T], object):
def bar(x: T) -> NoneType
CONSTANT = ... # type: C[float]
""")
src2 = textwrap.dedent("""
CONSTANT = ... # type: C[float]
T = TypeVar('T')
class C(typing.Generic[T], object):
def bar(x: T) -> NoneType
def foo(a: str or int) -> C
""")
tree1 = parser.parse_string(src1)
tree2 = parser.parse_string(src2)
tree1.Visit(visitors.VerifyVisitor())
tree2.Visit(visitors.VerifyVisitor())
self.assertTrue(tree1.constants)
self.assertTrue(tree1.classes)
self.assertTrue(tree1.functions)
self.assertTrue(tree2.constants)
self.assertTrue(tree2.classes)
self.assertTrue(tree2.functions)
self.assertIsInstance(tree1, pytd.TypeDeclUnit)
self.assertIsInstance(tree2, pytd.TypeDeclUnit)
# For the ==, != tests, TypeDeclUnit uses identity
self.assertTrue(tree1 == tree1)
self.assertTrue(tree2 == tree2)
self.assertFalse(tree1 == tree2)
self.assertFalse(tree2 == tree1)
self.assertFalse(tree1 != tree1)
self.assertFalse(tree2 != tree2)
self.assertTrue(tree1 != tree2)
self.assertTrue(tree2 != tree1)
self.assertEqual(tree1, tree1)
self.assertEqual(tree2, tree2)
self.assertNotEqual(tree1, tree2)
self.assertTrue(tree1.ASTeq(tree2))
self.assertTrue(tree1.ASTeq(tree1))
self.assertTrue(tree2.ASTeq(tree1))
self.assertTrue(tree2.ASTeq(tree2))
def namedtuple_ast(name, fields, defaults, options):
"""Make an AST with a namedtuple definition for the given name and fields.
Args:
name: The namedtuple name.
fields: The namedtuple fields.
defaults: Sequence of booleans, whether each field has a default.
options: A config.Options object.
Returns:
A pytd.TypeDeclUnit with the namedtuple definition in its classes.
"""
typevar = visitors.CreateTypeParametersForSignatures.PREFIX + name
num_fields = len(fields)
field_defs = "\n ".join(
"%s = ... # type: typing.Any" % field for field in fields)
fields_as_parameters = "".join(
", " + field + (" = ..." if default else "")
for field, default in zip(fields, defaults))
field_names_as_strings = ", ".join(repr(field) for field in fields)
if options.strict_namedtuple_checks:
tuple_superclass_type = "typing.Tuple[{}]".format(
_repeat_type("typing.Any", num_fields))
else:
tuple_superclass_type = "tuple"
nt = textwrap.dedent("""
{typevar} = TypeVar("{typevar}", bound={name})
class {name}({tuple_superclass_type}):
__dict__ = ... # type: collections.OrderedDict[str, typing.Any]
__slots__ = [{field_names_as_strings}]
_fields = ... # type: typing.Tuple[{repeat_str}]
{field_defs}
def __getnewargs__(self) -> typing.Tuple[{repeat_any}]: ...
def __getstate__(self) -> None: ...
def __init__(self, *args, **kwargs) -> None: ...
def __new__(
cls: typing.Type[{typevar}]{fields_as_parameters}) -> {typevar}: ...
def _asdict(self) -> collections.OrderedDict[str, typing.Any]: ...
@classmethod
def _make(cls: typing.Type[{typevar}],
iterable: typing.Iterable,
new = ...,
len: typing.Callable[[typing.Sized], int] = ...
) -> {typevar}: ...
def _replace(self: {typevar}, **kwds) -> {typevar}: ...
""").format(
typevar=typevar,
name=name,
repeat_str=_repeat_type("str", num_fields),
tuple_superclass_type=tuple_superclass_type,
field_defs=field_defs,
repeat_any=_repeat_type("typing.Any", num_fields),
fields_as_parameters=fields_as_parameters,
field_names_as_strings=field_names_as_strings)
return parser.parse_string(
nt, options=parser.PyiOptions.from_toplevel_options(options))
