def test_start_leader() -> None:
grammar = """
start: attr | NAME
attr: start '.' NAME
"""
# Would assert False without a special case in compute_left_recursives().
make_parser(grammar)
def test_left_recursion_too_complex() -> None:
grammar = """
start: foo
foo: bar '+' | baz '+' | '+'
bar: baz '-' | foo '-' | '-'
baz: foo '*' | bar '*' | '*'
"""
with pytest.raises(ValueError) as errinfo:
make_parser(grammar)
assert "no leader" in str(errinfo.value)
def test_dangling_reference() -> None:
grammar = """
start: foo ENDMARKER
foo: bar NAME
"""
with pytest.raises(GrammarError):
parser_class = make_parser(grammar)
def test_repeat_with_sep_simple() -> None:
grammar = """
start: ','.thing+ NEWLINE
thing: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("1, 2, 3\n", parser_class)
assert node == [
[
TokenInfo(NUMBER,
string="1",
start=(1, 0),
end=(1, 1),
line="1, 2, 3\n"),
TokenInfo(NUMBER,
string="2",
start=(1, 3),
end=(1, 4),
line="1, 2, 3\n"),
TokenInfo(NUMBER,
string="3",
start=(1, 6),
end=(1, 7),
line="1, 2, 3\n"),
],
TokenInfo(NEWLINE,
string="\n",
start=(1, 7),
end=(1, 8),
line="1, 2, 3\n"),
]
def test_bad_token_reference() -> None:
grammar = """
start: foo
foo: NAMEE
"""
with pytest.raises(GrammarError):
parser_class = make_parser(grammar)
def test_repeat_1_simple() -> None:
grammar = """
start: thing thing+ NEWLINE
thing: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("1 2 3\n", parser_class)
assert node == [
TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1),
line="1 2 3\n"),
[
TokenInfo(NUMBER,
string="2",
start=(1, 2),
end=(1, 3),
line="1 2 3\n"),
TokenInfo(NUMBER,
string="3",
start=(1, 4),
end=(1, 5),
line="1 2 3\n"),
],
TokenInfo(NEWLINE,
string="\n",
start=(1, 5),
end=(1, 6),
line="1 2 3\n"),
]
with pytest.raises(SyntaxError):
parse_string("1\n", parser_class)
def test_locations_in_alt_action_and_group() -> None:
grammar = """
start: t=term NEWLINE? $ { ast.Expression(t, LOCATIONS) }
term:
| l=term '*' r=factor { ast.BinOp(l, ast.Mult(), r, LOCATIONS) }
| l=term '/' r=factor { ast.BinOp(l, ast.Div(), r, LOCATIONS) }
| factor
factor:
| (
n=NAME { ast.Name(id=n.string, ctx=ast.Load(), LOCATIONS) } |
n=NUMBER { ast.Constant(value=ast.literal_eval(n.string), LOCATIONS) }
)
"""
parser_class = make_parser(grammar)
source = "2*3\n"
o = ast.dump(parse_string(source, parser_class).body,
include_attributes=True)
p = ast.dump(ast.parse(source).body[0].value,
include_attributes=True).replace(" kind=None,", "")
diff = "\n".join(
difflib.unified_diff(o.split("\n"), p.split("\n"), "cpython",
"python-pegen"))
if diff:
print(diff)
assert not diff
def test_optional_literal() -> None:
grammar = """
start: sum NEWLINE
sum: term '+' ?
term: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("1+\n", parser_class)
assert node == [
[
TokenInfo(NUMBER,
string="1",
start=(1, 0),
end=(1, 1),
line="1+\n"),
TokenInfo(OP, string="+", start=(1, 1), end=(1, 2), line="1+\n"),
],
TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="1+\n"),
]
node = parse_string("1\n", parser_class)
assert node == [
[
TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1),
line="1\n"), None
],
TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"),
]
def test_keywords() -> None:
grammar = """
start: 'one' 'two' 'three' 'four' 'five' "six" "seven" "eight" "nine" "ten"
"""
parser_class = make_parser(grammar)
assert parser_class.KEYWORDS == ("five", "four", "one", "three", "two")
assert parser_class.SOFT_KEYWORDS == ("eight", "nine", "seven", "six",
"ten")
def test_cut_early_exit() -> None:
grammar = """
start: '(' ~ expr ')' | '(' name ')'
expr: NUMBER
name: NAME
"""
parser_class = make_parser(grammar)
with pytest.raises(SyntaxError):
parse_string("(a)", parser_class, verbose=True)
def test_forced() -> None:
grammar = """
start: NAME &&':' | NAME
"""
parser_class = make_parser(grammar)
assert parse_string("number :", parser_class, verbose=True)
with pytest.raises(SyntaxError) as e:
parse_string("a", parser_class, verbose=True)
assert "expected ':'" in str(e.exconly())
def test_forced_with_group() -> None:
grammar = """
start: NAME &&(':' | ';') | NAME
"""
parser_class = make_parser(grammar)
assert parse_string("number :", parser_class, verbose=True)
assert parse_string("number ;", parser_class, verbose=True)
with pytest.raises(SyntaxError) as e:
parse_string("a", parser_class, verbose=True)
assert "expected (':' | ';')" in e.value.args[0]
def test_cut() -> None:
grammar = """
start: '(' ~ expr ')'
expr: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("(1)", parser_class, verbose=True)
assert node == [
TokenInfo(OP, string="(", start=(1, 0), end=(1, 1), line="(1)"),
TokenInfo(NUMBER, string="1", start=(1, 1), end=(1, 2), line="(1)"),
TokenInfo(OP, string=")", start=(1, 2), end=(1, 3), line="(1)"),
]
def test_expr_grammar() -> None:
grammar = """
start: sum NEWLINE
sum: term '+' term | term
term: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("42\n", parser_class)
assert node == [
TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n"),
TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"),
]
def test_soft_keyword() -> None:
grammar = """
start:
| "number" n=NUMBER { eval(n.string) }
| "string" n=STRING { n.string }
| SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { f"{l.string} = {n.string}"}
"""
parser_class = make_parser(grammar)
assert parse_string("number 1", parser_class, verbose=True) == 1
assert parse_string("string 'b'", parser_class, verbose=True) == "'b'"
assert parse_string("number test 1", parser_class,
verbose=True) == "test = 1"
assert parse_string("string test 'b'", parser_class,
verbose=True) == "test = 'b'"
with pytest.raises(SyntaxError):
parse_string("test 1", parser_class, verbose=True)
def test_repeat_1_complex() -> None:
grammar = """
start: term ('+' term)+ NEWLINE
term: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("1 + 2 + 3\n", parser_class)
assert node == [
TokenInfo(NUMBER,
string="1",
start=(1, 0),
end=(1, 1),
line="1 + 2 + 3\n"),
[
[
TokenInfo(OP,
string="+",
start=(1, 2),
end=(1, 3),
line="1 + 2 + 3\n"),
TokenInfo(NUMBER,
string="2",
start=(1, 4),
end=(1, 5),
line="1 + 2 + 3\n"),
],
[
TokenInfo(OP,
string="+",
start=(1, 6),
end=(1, 7),
line="1 + 2 + 3\n"),
TokenInfo(NUMBER,
string="3",
start=(1, 8),
end=(1, 9),
line="1 + 2 + 3\n"),
],
],
TokenInfo(NEWLINE,
string="\n",
start=(1, 9),
end=(1, 10),
line="1 + 2 + 3\n"),
]
with pytest.raises(SyntaxError):
parse_string("1\n", parser_class)
def index() -> None:
# you must tell the variable 'form' what you named the class, above
# 'form' is the variable name used in this template: index.html
form = GrammarForm()
form.grammar(class_="form-control")
output_text = "\n"
if form.validate_on_submit():
grammar_source = form.grammar.data
program_source = form.source.data
output = io.StringIO()
try:
parser_class = make_parser(grammar_source)
result = parse_string(program_source, parser_class, verbose=False)
print(result, file=output)
except Exception as e:
traceback.print_exc(file=output)
output_text += output.getvalue()
return render_template("index.html", form=form, output=output_text)
def test_gather() -> None:
grammar = """
start: ','.thing+ NEWLINE
thing: NUMBER
"""
rules = parse_string(grammar, GrammarParser).rules
assert str(rules["start"]) == "start: ','.thing+ NEWLINE"
print(repr(rules["start"]))
assert repr(rules["start"]).startswith(
"Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'"
)
assert str(rules["thing"]) == "thing: NUMBER"
parser_class = make_parser(grammar)
node = parse_string("42\n", parser_class)
assert node == [
[
TokenInfo(NUMBER,
string="42",
start=(1, 0),
end=(1, 2),
line="42\n")
],
TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"),
]
node = parse_string("1, 2\n", parser_class)
assert node == [
[
TokenInfo(NUMBER,
string="1",
start=(1, 0),
end=(1, 1),
line="1, 2\n"),
TokenInfo(NUMBER,
string="2",
start=(1, 3),
end=(1, 4),
line="1, 2\n"),
],
TokenInfo(NEWLINE,
string="\n",
start=(1, 4),
end=(1, 5),
line="1, 2\n"),
]
def test_alt_optional_operator() -> None:
grammar = """
start: sum NEWLINE
sum: term ['+' term]
term: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("1 + 2\n", parser_class)
assert node == [
[
TokenInfo(NUMBER,
string="1",
start=(1, 0),
end=(1, 1),
line="1 + 2\n"),
[
TokenInfo(OP,
string="+",
start=(1, 2),
end=(1, 3),
line="1 + 2\n"),
TokenInfo(NUMBER,
string="2",
start=(1, 4),
end=(1, 5),
line="1 + 2\n"),
],
],
TokenInfo(NEWLINE,
string="\n",
start=(1, 5),
end=(1, 6),
line="1 + 2\n"),
]
node = parse_string("1\n", parser_class)
assert node == [
[
TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1),
line="1\n"), None
],
TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"),
]
def test_lookahead() -> None:
grammar = """
start: (expr_stmt | assign_stmt) &'.'
expr_stmt: !(target '=') expr
assign_stmt: target '=' expr
expr: term ('+' term)*
target: NAME
term: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("foo = 12 + 12 .", parser_class)
assert node == [
TokenInfo(NAME,
string="foo",
start=(1, 0),
end=(1, 3),
line="foo = 12 + 12 ."),
TokenInfo(OP,
string="=",
start=(1, 4),
end=(1, 5),
line="foo = 12 + 12 ."),
[
TokenInfo(NUMBER,
string="12",
start=(1, 6),
end=(1, 8),
line="foo = 12 + 12 ."),
[[
TokenInfo(OP,
string="+",
start=(1, 9),
end=(1, 10),
line="foo = 12 + 12 ."),
TokenInfo(NUMBER,
string="12",
start=(1, 11),
end=(1, 13),
line="foo = 12 + 12 ."),
]],
],
]
def test_python_expr() -> None:
grammar = """
start: expr NEWLINE? $ { ast.Expression(expr, LOCATIONS) }
expr: ( expr '+' term { ast.BinOp(expr, ast.Add(), term, LOCATIONS) }
| expr '-' term { ast.BinOp(expr, ast.Sub(), term, LOCATIONS) }
| term { term }
)
term: ( l=term '*' r=factor { ast.BinOp(l, ast.Mult(), r, LOCATIONS) }
| l=term '/' r=factor { ast.BinOp(l, ast.Div(), r, LOCATIONS) }
| factor { factor }
)
factor: ( '(' expr ')' { expr }
| atom { atom }
)
atom: ( n=NAME { ast.Name(id=n.string, ctx=ast.Load(), LOCATIONS) }
| n=NUMBER { ast.Constant(value=ast.literal_eval(n.string), LOCATIONS) }
)
"""
parser_class = make_parser(grammar)
node = parse_string("(1 + 2*3 + 5)/(6 - 2)\n", parser_class)
code = compile(node, "", "eval")
val = eval(code)
assert val == 3.0
def test_missing_start() -> None:
grammar = """
foo: NAME
"""
with pytest.raises(GrammarError):
parser_class = make_parser(grammar)
def test_named_lookahead_error() -> None:
grammar = """
start: foo=!'x' NAME
"""
with pytest.raises(SyntaxError):
make_parser(grammar)