def main() -> None:
args = parser.parse_args()
new_parser = args.parser == "new"
if args.multiline:
sep = "\n"
else:
sep = " "
program = sep.join(args.program)
if new_parser:
tree = _peg_parser.parse_string(program)
if args.diff:
a = _peg_parser.parse_string(program, oldparser=True)
b = tree
diff = diff_trees(a, b, args.verbose)
if diff:
for line in diff:
print(line)
else:
print("# Trees are the same")
else:
print("# Parsed using the new parser")
print(format_tree(tree, args.verbose))
else:
tree = _peg_parser.parse_string(program, oldparser=True)
print("# Parsed using the old parser")
print(format_tree(tree, args.verbose))
def main() -> None:
args = parser.parse_args()
if args.diff and not args.grammar_file:
parser.error("-d/--diff requires -g/--grammar-file")
if args.multiline:
sep = "\n"
else:
sep = " "
program = sep.join(args.program)
if args.grammar_file:
tree = _peg_parser.parse_string(program)
if args.diff:
a = tree
b = _peg_parser.parse_string(program, oldparser=True)
diff = diff_trees(a, b, args.verbose)
if diff:
for line in diff:
print(line)
else:
print("# Trees are the same")
else:
print(f"# Parsed using {args.grammar_file}")
print(format_tree(tree, args.verbose))
else:
tree = _peg_parser.parse_string(program, oldparser=True)
print("# Parse using the old parser")
print(format_tree(tree, args.verbose))
def test_incorrect_ast_generation_with_specialized_errors(self) -> None:
for source, error_text in FAIL_SPECIALIZED_MESSAGE_CASES:
exc = IndentationError if "indent" in error_text else SyntaxError
with self.assertRaises(exc) as se:
peg_parser.parse_string(source)
self.assertTrue(
error_text in se.exception.msg,
f"Actual error message does not match expexted for {source}")
def test_correct_ast_generation_without_pos_info(self) -> None:
for source in GOOD_BUT_FAIL_SOURCES:
actual_ast = peg_parser.parse_string(source)
expected_ast = peg_parser.parse_string(source, oldparser=True)
self.assertEqual(
ast.dump(actual_ast),
ast.dump(expected_ast),
f"Wrong AST generation for source: {source}",
)
def test_correct_ast_generatrion_eval(self) -> None:
for source in EXPRESSIONS_TEST_SOURCES:
actual_ast = peg_parser.parse_string(source, mode='eval')
expected_ast = peg_parser.parse_string(source, mode='eval', oldparser=True)
self.assertEqual(
ast.dump(actual_ast, include_attributes=True),
ast.dump(expected_ast, include_attributes=True),
f"Wrong AST generation for source: {source}",
)
def test_correct_but_known_to_fail_ast_generation_on_source_files(self) -> None:
for source in GOOD_BUT_FAIL_SOURCES:
actual_ast = peg_parser.parse_string(source)
expected_ast = peg_parser.parse_string(source, oldparser=True)
self.assertEqual(
ast.dump(actual_ast, include_attributes=True),
ast.dump(expected_ast, include_attributes=True),
f"Wrong AST generation for source: {source}",
)
def test_correct_ast_generation_on_source_files(self) -> None:
self.maxDiff = None
for source in TEST_SOURCES:
actual_ast = peg_parser.parse_string(source)
expected_ast = peg_parser.parse_string(source, oldparser=True)
self.assertEqual(
ast.dump(actual_ast, include_attributes=True),
ast.dump(expected_ast, include_attributes=True),
f"Wrong AST generation for source: {source}",
)
def check_nested_expr(nesting_depth: int) -> bool:
expr = f"{'(' * nesting_depth}0{')' * nesting_depth}"
try:
parse_string(expr)
print(f"Nesting depth of {nesting_depth} is successful")
return True
except Exception as err:
print(f"{FAIL}(Failed with nesting depth of {nesting_depth}{ENDC}")
print(f"{FAIL}\t{err}{ENDC}")
return False
def compare_trees(
actual_tree: ast.AST, file: str, verbose: bool, include_attributes: bool = False,
) -> int:
with open(file) as f:
expected_tree = _peg_parser.parse_string(f.read(), oldparser=True)
expected_text = ast_dump(expected_tree, include_attributes=include_attributes)
actual_text = ast_dump(actual_tree, include_attributes=include_attributes)
if actual_text == expected_text:
if verbose:
print("Tree for {file}:")
print(show_parse.format_tree(actual_tree, include_attributes))
return 0
print(f"Diffing ASTs for {file} ...")
expected = show_parse.format_tree(expected_tree, include_attributes)
actual = show_parse.format_tree(actual_tree, include_attributes)
if verbose:
print("Expected for {file}:")
print(expected)
print("Actual for {file}:")
print(actual)
print(f"Diff for {file}:")
diff = show_parse.diff_trees(expected_tree, actual_tree, include_attributes)
for line in diff:
print(line)
return 1
def parse_file(source: str, file: str, mode: int, oldparser: bool) -> Tuple[Any, float]:
t0 = time.time()
if mode == COMPILE:
result = _peg_parser.compile_string(
source,
filename=file,
oldparser=oldparser,
)
else:
result = _peg_parser.parse_string(
source,
filename=file,
oldparser=oldparser,
ast=(mode == PARSE),
)
t1 = time.time()
return result, t1 - t0
def test_incorrect_ast_generation_on_source_files(self) -> None:
for source in FAIL_SOURCES:
with self.assertRaises(
SyntaxError,
msg=f"Parsing {source} did not raise an exception"):
peg_parser.parse_string(source)
def is_parsing_failure(source: str) -> bool:
try:
_peg_parser.parse_string(source, mode="exec", oldparser=True)
except SyntaxError:
return False
return True
def parse_directory(
directory: str,
grammar_file: str,
tokens_file: str,
verbose: bool,
excluded_files: List[str],
skip_actions: bool,
tree_arg: int,
short: bool,
mode: int,
parser: str,
) -> int:
if parser == "cpython" and (tree_arg or mode == 0):
print("Cannot specify tree argument or mode=0 with the cpython parser.", file=sys.stderr)
return 1
if not directory:
print("You must specify a directory of files to test.", file=sys.stderr)
return 1
if grammar_file and tokens_file:
if not os.path.exists(grammar_file):
print(f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr)
return 1
else:
print(
"A grammar file or a tokens file was not provided - attempting to use existing parser from stdlib...\n"
)
if tree_arg:
assert mode == 1, "Mode should be 1 (parse), when comparing the generated trees"
# For a given directory, traverse files and attempt to parse each one
# - Output success/failure for each file
errors = 0
files = []
trees = {} # Trees to compare (after everything else is done)
total_seconds = 0
for file in sorted(glob(f"{directory}/**/*.py", recursive=True)):
# Only attempt to parse Python files and files that are not excluded
should_exclude_file = False
for pattern in excluded_files:
if PurePath(file).match(pattern):
should_exclude_file = True
break
if not should_exclude_file:
with tokenize.open(file) as f:
source = f.read()
try:
t0 = time.time()
if mode == 2:
result = _peg_parser.compile_string(
source,
filename=file,
oldparser=parser == "cpython",
)
else:
result = _peg_parser.parse_string(
source,
filename=file,
oldparser=parser == "cpython"
)
t1 = time.time()
total_seconds += (t1 - t0)
if tree_arg:
trees[file] = result
if not short:
report_status(succeeded=True, file=file, verbose=verbose)
except Exception as error:
try:
_peg_parser.parse_string(source, mode="exec", oldparser=True)
except Exception:
if not short:
print(f"File {file} cannot be parsed by either pegen or the ast module.")
else:
report_status(
succeeded=False, file=file, verbose=verbose, error=error, short=short
)
errors += 1
files.append(file)
t1 = time.time()
total_files = len(files)
total_bytes = 0
total_lines = 0
for file in files:
# Count lines and bytes separately
with open(file, "rb") as f:
total_lines += sum(1 for _ in f)
total_bytes += f.tell()
print(
f"Checked {total_files:,} files, {total_lines:,} lines,",
f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.",
)
if total_seconds > 0:
print(
f"That's {total_lines / total_seconds :,.0f} lines/sec,",
f"or {total_bytes / total_seconds :,.0f} bytes/sec.",
)
if short:
print_memstats()
if errors:
print(f"Encountered {errors} failures.", file=sys.stderr)
# Compare trees (the dict is empty unless -t is given)
compare_trees_errors = 0
for file, tree in trees.items():
if not short:
print("Comparing ASTs for", file)
if compare_trees(tree, file, verbose, tree_arg >= 2) == 1:
compare_trees_errors += 1
if errors or compare_trees_errors:
return 1
return 0
def time_notree(source, parser):
if parser == "old":
return _peg_parser.parse_string(source, oldparser=True, ast=False)
else:
return _peg_parser.parse_string(source, ast=False)
def time_parse(source, parser):
if parser == "old":
return _peg_parser.parse_string(source, oldparser=True)
else:
return _peg_parser.parse_string(source)
def test_fstring_parse_error_tracebacks(self) -> None:
for source, error_text in FSTRINGS_TRACEBACKS.values():
with self.assertRaises(SyntaxError) as se:
peg_parser.parse_string(dedent(source))
self.assertEqual(error_text, se.exception.text)
def test_tokenizer_errors_are_propagated(self) -> None:
n = 201
with self.assertRaisesRegex(SyntaxError,
"too many nested parentheses"):
peg_parser.parse_string(n * '(' + ')' * n)
def show_parse(source: str, verbose: bool = False) -> str:
tree = _peg_parser.parse_string(source, oldparser=True)
return format_tree(tree, verbose).rstrip("\n")