def compare_trees(
actual_tree: ast.AST, file: str, verbose: bool, include_attributes: bool = False,
) -> int:
with open(file) as f:
expected_tree = _peg_parser.parse_string(f.read(), oldparser=True)
expected_text = ast_dump(expected_tree, include_attributes=include_attributes)
actual_text = ast_dump(actual_tree, include_attributes=include_attributes)
if actual_text == expected_text:
if verbose:
print("Tree for {file}:")
print(show_parse.format_tree(actual_tree, include_attributes))
return 0
print(f"Diffing ASTs for {file} ...")
expected = show_parse.format_tree(expected_tree, include_attributes)
actual = show_parse.format_tree(actual_tree, include_attributes)
if verbose:
print("Expected for {file}:")
print(expected)
print("Actual for {file}:")
print(actual)
print(f"Diff for {file}:")
diff = show_parse.diff_trees(expected_tree, actual_tree, include_attributes)
for line in diff:
print(line)
return 1
def verify_ast_generation(source: str, stmt: str, tmp_path: PurePath) -> None:
grammar = parse_string(source, GrammarParser)
extension = generate_parser_c_extension(grammar, tmp_path)
expected_ast = ast.parse(stmt)
actual_ast = extension.parse_string(stmt, mode=1)
assert ast_dump(expected_ast) == ast_dump(actual_ast)
def test_correct_but_known_to_fail_ast_generation_on_source_files(
parser_extension: Any, source: str) -> None:
actual_ast = parser_extension.parse_string(source, mode=1)
expected_ast = ast.parse(source)
assert ast_dump(actual_ast, include_attributes=True) == ast_dump(
expected_ast,
include_attributes=True), f"Wrong AST generation for source: {source}"
def format_tree(tree: ast.AST, verbose: bool = False) -> str:
with tempfile.NamedTemporaryFile("w+") as tf:
tf.write(ast_dump(tree, include_attributes=verbose))
tf.write("\n")
tf.flush()
cmd = f"black -q {tf.name}"
sts = os.system(cmd)
if sts:
raise RuntimeError(f"Command {cmd!r} failed with status 0x{sts:x}")
tf.seek(0)
return tf.read()
def test_c_parser(tmp_path: PurePath) -> None:
grammar_source = """
start[mod_ty]: a=stmt* $ { Module(a, NULL, p->arena) }
stmt[stmt_ty]: a=expr_stmt { a }
expr_stmt[stmt_ty]: a=expression NEWLINE { _Py_Expr(a, EXTRA) }
expression[expr_ty]: ( l=expression '+' r=term { _Py_BinOp(l, Add, r, EXTRA) }
| l=expression '-' r=term { _Py_BinOp(l, Sub, r, EXTRA) }
| t=term { t }
)
term[expr_ty]: ( l=term '*' r=factor { _Py_BinOp(l, Mult, r, EXTRA) }
| l=term '/' r=factor { _Py_BinOp(l, Div, r, EXTRA) }
| f=factor { f }
)
factor[expr_ty]: ('(' e=expression ')' { e }
| a=atom { a }
)
atom[expr_ty]: ( n=NAME { n }
| n=NUMBER { n }
| s=STRING { s }
)
"""
grammar = parse_string(grammar_source, GrammarParser)
extension = generate_parser_c_extension(grammar, tmp_path)
expressions = [
"4+5",
"4-5",
"4*5",
"1+4*5",
"1+4/5",
"(1+1) + (1+1)",
"(1+1) - (1+1)",
"(1+1) * (1+1)",
"(1+1) / (1+1)",
]
for expr in expressions:
the_ast = extension.parse_string(expr, mode=1)
expected_ast = ast.parse(expr)
assert ast_dump(the_ast) == ast_dump(expected_ast)
def test_with_stmt_with_paren(tmp_path: PurePath) -> None:
grammar_source = """
start[mod_ty]: a=[statements] ENDMARKER { Module(a, NULL, p->arena) }
statements[asdl_seq*]: a=statement+ { _PyPegen_seq_flatten(p, a) }
statement[asdl_seq*]: a=compound_stmt { _PyPegen_singleton_seq(p, a) }
compound_stmt[stmt_ty]: with_stmt
with_stmt[stmt_ty]: (
a='with' '(' b=','.with_item+ ')' ':' c=block {
_Py_With(b, _PyPegen_singleton_seq(p, c), NULL, EXTRA) }
)
with_item[withitem_ty]: (
e=NAME o=['as' t=NAME { t }] { _Py_withitem(e, _PyPegen_set_expr_context(p, o, Store), p->arena) }
)
block[stmt_ty]: a=pass_stmt NEWLINE { a } | NEWLINE INDENT a=pass_stmt DEDENT { a }
pass_stmt[stmt_ty]: a='pass' { _Py_Pass(EXTRA) }
"""
stmt = "with (\n a as b,\n c as d\n): pass"
grammar = parse_string(grammar_source, GrammarParser)
extension = generate_parser_c_extension(grammar, tmp_path)
the_ast = extension.parse_string(stmt, mode=1)
assert ast_dump(the_ast).startswith(
"Module(body=[With(items=[withitem(context_expr=Name(id='a', ctx=Load()), optional_vars=Name(id='b', ctx=Store())), "
"withitem(context_expr=Name(id='c', ctx=Load()), optional_vars=Name(id='d', ctx=Store()))]"
)
def test_correct_ast_generation_without_pos_info(parser_extension: Any,
source: str) -> None:
actual_ast = parser_extension.parse_string(source, mode=1)
expected_ast = ast.parse(source)
assert ast_dump(actual_ast) == ast_dump(
expected_ast), f"Wrong AST generation for source: {source}"