def test_start_leader():
grammar = """
start: attr | NAME
attr: start '.' NAME
"""
# Would assert False without a special case in compute_left_recursives().
make_parser(grammar)
def test_opt_sequence(self) -> None:
grammar = """
start: [NAME*]
"""
# This case was failing because of a double trailing comma at the end
# of a line in the generated source. See bpo-41044
make_parser(grammar)
def test_left_recursion_too_complex(self) -> None:
grammar = """
start: foo
foo: bar '+' | baz '+' | '+'
bar: baz '-' | foo '-' | '-'
baz: foo '*' | bar '*' | '*'
"""
with self.assertRaises(ValueError) as errinfo:
make_parser(grammar)
self.assertTrue("no leader" in str(errinfo.exception.value))
def test_left_recursion_too_complex():
grammar = """
start: foo
foo: bar '+' | baz '+' | '+'
bar: baz '-' | foo '-' | '-'
baz: foo '*' | bar '*' | '*'
"""
with pytest.raises(ValueError) as errinfo:
make_parser(grammar)
assert "no leader" in str(errinfo.value)
def test_repeat_1_complex(self) -> None:
grammar = """
start: term ('+' term)+ NEWLINE
term: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("1 + 2 + 3\n", parser_class)
self.assertEqual(node, [
[TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 + 2 + 3\n")],
[
[
[
TokenInfo(OP, string="+", start=(1, 2), end=(1, 3), line="1 + 2 + 3\n"),
[TokenInfo(NUMBER, string="2", start=(1, 4), end=(1, 5), line="1 + 2 + 3\n")],
]
],
[
[
TokenInfo(OP, string="+", start=(1, 6), end=(1, 7), line="1 + 2 + 3\n"),
[TokenInfo(NUMBER, string="3", start=(1, 8), end=(1, 9), line="1 + 2 + 3\n")],
]
],
],
TokenInfo(NEWLINE, string="\n", start=(1, 9), end=(1, 10), line="1 + 2 + 3\n"),
])
with self.assertRaises(SyntaxError):
parse_string("1\n", parser_class)
def test_bad_token_reference():
grammar = """
start: foo
foo: NAMEE
"""
with pytest.raises(GrammarError):
parser_class = make_parser(grammar)
def test_dangling_reference(self) -> None:
grammar = """
start: foo ENDMARKER
foo: bar NAME
"""
with self.assertRaises(GrammarError):
parser_class = make_parser(grammar)
def test_repeat_1_simple():
grammar = """
start: thing thing+ NEWLINE
thing: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("1 2 3\n", parser_class)
assert node == [[
TokenInfo(NUMBER, string='1', start=(1, 0), end=(1, 1), line='1 2 3\n')
],
[[[
TokenInfo(NUMBER,
string='2',
start=(1, 2),
end=(1, 3),
line='1 2 3\n')
]],
[[
TokenInfo(NUMBER,
string='3',
start=(1, 4),
end=(1, 5),
line='1 2 3\n')
]]],
TokenInfo(NEWLINE,
string='\n',
start=(1, 5),
end=(1, 6),
line='1 2 3\n')]
with pytest.raises(SyntaxError):
parse_string("1\n", parser_class)
def test_optional_literal():
grammar = """
start: sum NEWLINE
sum: term '+' ?
term: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("1+\n", parser_class)
assert node == [
[
[
TokenInfo(NUMBER,
string="1",
start=(1, 0),
end=(1, 1),
line="1+\n")
],
TokenInfo(OP, string="+", start=(1, 1), end=(1, 2), line="1+\n"),
],
TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="1+\n"),
]
node = parse_string("1\n", parser_class)
assert node == [
[[TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n")],
None],
TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"),
]
def test_repeat_with_sep_simple(self) -> None:
grammar = """
start: ','.thing+ NEWLINE
thing: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("1, 2, 3\n", parser_class)
self.assertEqual(
node,
[
[
TokenInfo(NUMBER,
string="1",
start=(1, 0),
end=(1, 1),
line="1, 2, 3\n"),
TokenInfo(NUMBER,
string="2",
start=(1, 3),
end=(1, 4),
line="1, 2, 3\n"),
TokenInfo(NUMBER,
string="3",
start=(1, 6),
end=(1, 7),
line="1, 2, 3\n"),
],
TokenInfo(NEWLINE,
string="\n",
start=(1, 7),
end=(1, 8),
line="1, 2, 3\n"),
],
)
def test_dangling_reference():
grammar = """
start: foo ENDMARKER
foo: bar NAME
"""
with pytest.raises(GrammarError):
parser_class = make_parser(grammar)
def test_bad_token_reference(self) -> None:
grammar = """
start: foo
foo: NAMEE
"""
with self.assertRaises(GrammarError):
parser_class = make_parser(grammar)
def test_invalid_rule_name(self) -> None:
grammar = """
start: _a b
_a: 'a'
b: 'b'
"""
with self.assertRaisesRegex(GrammarError, "cannot start with underscore: '_a'"):
parser_class = make_parser(grammar)
def test_invalid_variable_name_in_temporal_rule(self) -> None:
grammar = """
start: a b
a: (_x='a' | 'b') | 'c'
b: 'b'
"""
with self.assertRaisesRegex(GrammarError, "cannot start with underscore: '_x'"):
parser_class = make_parser(grammar)
def test_forced_with_group(self) -> None:
grammar = """
start: NAME &&(':' | ';') | NAME
"""
parser_class = make_parser(grammar)
self.assertTrue(parse_string("number :", parser_class, verbose=True))
self.assertTrue(parse_string("number ;", parser_class, verbose=True))
with self.assertRaises(SyntaxError) as e:
parse_string("a", parser_class, verbose=True)
self.assertIn("expected (':' | ';')", e.exception.args[0])
def test_forced(self) -> None:
grammar = """
start: NAME &&':' | NAME
"""
parser_class = make_parser(grammar)
self.assertTrue(parse_string("number :", parser_class, verbose=True))
with self.assertRaises(SyntaxError) as e:
parse_string("a", parser_class, verbose=True)
self.assertIn("expected ':'", str(e.exception))
def test_alt_optional_operator(self) -> None:
grammar = """
start: sum NEWLINE
sum: term ['+' term]
term: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("1 + 2\n", parser_class)
self.assertEqual(
node,
[
[
TokenInfo(NUMBER,
string="1",
start=(1, 0),
end=(1, 1),
line="1 + 2\n"),
[
TokenInfo(OP,
string="+",
start=(1, 2),
end=(1, 3),
line="1 + 2\n"),
TokenInfo(NUMBER,
string="2",
start=(1, 4),
end=(1, 5),
line="1 + 2\n"),
],
],
TokenInfo(NEWLINE,
string="\n",
start=(1, 5),
end=(1, 6),
line="1 + 2\n"),
],
)
node = parse_string("1\n", parser_class)
self.assertEqual(
node,
[
[
TokenInfo(NUMBER,
string="1",
start=(1, 0),
end=(1, 1),
line="1\n"),
None,
],
TokenInfo(
NEWLINE, string="\n", start=(1, 1), end=(1, 2),
line="1\n"),
],
)
def test_cut():
grammar = """
start: '(' ~ expr ')'
expr: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("(1)", parser_class, verbose=True)
assert node == [
TokenInfo(OP, string="(", start=(1, 0), end=(1, 1), line="(1)"),
[TokenInfo(NUMBER, string="1", start=(1, 1), end=(1, 2), line="(1)")],
TokenInfo(OP, string=")", start=(1, 2), end=(1, 3), line="(1)"),
]
def test_expr_grammar(self) -> None:
grammar = """
start: sum NEWLINE
sum: term '+' term | term
term: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("42\n", parser_class)
self.assertEqual(node, [
[[TokenInfo(NUMBER, string="42", start=(1, 0), end=(1, 2), line="42\n")]],
TokenInfo(NEWLINE, string="\n", start=(1, 2), end=(1, 3), line="42\n"),
])
def test_cut(self) -> None:
grammar = """
start: '(' ~ expr ')'
expr: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("(1)", parser_class)
self.assertEqual(node, [
TokenInfo(OP, string="(", start=(1, 0), end=(1, 1), line="(1)"),
[TokenInfo(NUMBER, string="1", start=(1, 1), end=(1, 2), line="(1)")],
TokenInfo(OP, string=")", start=(1, 2), end=(1, 3), line="(1)"),
])
def test_gather(self) -> None:
grammar = """
start: ','.thing+ NEWLINE
thing: NUMBER
"""
rules = parse_string(grammar, GrammarParser).rules
self.assertEqual(str(rules["start"]), "start: ','.thing+ NEWLINE")
self.assertTrue(
repr(rules["start"]).startswith(
"Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'"
))
self.assertEqual(str(rules["thing"]), "thing: NUMBER")
parser_class = make_parser(grammar)
node = parse_string("42\n", parser_class)
assert node == [
[[
TokenInfo(NUMBER,
string="42",
start=(1, 0),
end=(1, 2),
line="42\n")
]],
TokenInfo(NEWLINE,
string="\n",
start=(1, 2),
end=(1, 3),
line="42\n"),
]
node = parse_string("1, 2\n", parser_class)
assert node == [
[
[
TokenInfo(NUMBER,
string="1",
start=(1, 0),
end=(1, 1),
line="1, 2\n")
],
[
TokenInfo(NUMBER,
string="2",
start=(1, 3),
end=(1, 4),
line="1, 2\n")
],
],
TokenInfo(NEWLINE,
string="\n",
start=(1, 4),
end=(1, 5),
line="1, 2\n"),
]
def test_lookahead():
grammar = """
start: (expr_stmt | assign_stmt) &'.'
expr_stmt: !(target '=') expr
assign_stmt: target '=' expr
expr: term ('+' term)*
target: NAME
term: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("foo = 12 + 12 .", parser_class)
assert node == [[[
[
TokenInfo(NAME,
string="foo",
start=(1, 0),
end=(1, 3),
line="foo = 12 + 12 .")
],
TokenInfo(OP,
string="=",
start=(1, 4),
end=(1, 5),
line="foo = 12 + 12 ."),
[
[
TokenInfo(NUMBER,
string="12",
start=(1, 6),
end=(1, 8),
line="foo = 12 + 12 .")
],
[[[
TokenInfo(
OP,
string="+",
start=(1, 9),
end=(1, 10),
line="foo = 12 + 12 .",
),
[
TokenInfo(
NUMBER,
string="12",
start=(1, 11),
end=(1, 13),
line="foo = 12 + 12 .",
)
],
]]],
],
]]]
def test_repeat_0_complex():
grammar = """
start: term ('+' term)* NEWLINE
term: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("1 + 2 + 3\n", parser_class)
assert node == [
[
TokenInfo(NUMBER,
string="1",
start=(1, 0),
end=(1, 1),
line="1 + 2 + 3\n")
],
[
[[
TokenInfo(OP,
string="+",
start=(1, 2),
end=(1, 3),
line="1 + 2 + 3\n"),
[
TokenInfo(NUMBER,
string="2",
start=(1, 4),
end=(1, 5),
line="1 + 2 + 3\n")
],
]],
[[
TokenInfo(OP,
string="+",
start=(1, 6),
end=(1, 7),
line="1 + 2 + 3\n"),
[
TokenInfo(NUMBER,
string="3",
start=(1, 8),
end=(1, 9),
line="1 + 2 + 3\n")
],
]],
],
TokenInfo(NEWLINE,
string="\n",
start=(1, 9),
end=(1, 10),
line="1 + 2 + 3\n"),
]
def test_repeat_1_complex():
grammar = """
start: term ('+' term)+ NEWLINE
term: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("1 + 2 + 3\n", parser_class)
assert node == [[
TokenInfo(NUMBER,
string='1',
start=(1, 0),
end=(1, 1),
line='1 + 2 + 3\n')
],
[[[
TokenInfo(OP,
string='+',
start=(1, 2),
end=(1, 3),
line='1 + 2 + 3\n'),
[
TokenInfo(NUMBER,
string='2',
start=(1, 4),
end=(1, 5),
line='1 + 2 + 3\n')
]
]],
[[
TokenInfo(OP,
string='+',
start=(1, 6),
end=(1, 7),
line='1 + 2 + 3\n'),
[
TokenInfo(NUMBER,
string='3',
start=(1, 8),
end=(1, 9),
line='1 + 2 + 3\n')
]
]]],
TokenInfo(NEWLINE,
string='\n',
start=(1, 9),
end=(1, 10),
line='1 + 2 + 3\n')]
with pytest.raises(SyntaxError):
parse_string("1\n", parser_class)
def test_soft_keyword(self) -> None:
grammar = """
start:
| "number" n=NUMBER { eval(n.string) }
| "string" n=STRING { n.string }
| SOFT_KEYWORD l=NAME n=(NUMBER | NAME | STRING) { f"{l.string} = {n.string}"}
"""
parser_class = make_parser(grammar)
self.assertEqual(parse_string("number 1", parser_class), 1)
self.assertEqual(parse_string("string 'b'", parser_class), "'b'")
self.assertEqual(parse_string("number test 1", parser_class),
"test = 1")
assert (parse_string("string test 'b'", parser_class) == "test = 'b'")
with self.assertRaises(SyntaxError):
parse_string("test 1", parser_class)
def test_expr_grammar():
grammar = """
start: sum NEWLINE
sum: term '+' term | term
term: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("42\n", parser_class)
assert node == [[[
TokenInfo(NUMBER, string='42', start=(1, 0), end=(1, 2), line='42\n')
]],
TokenInfo(NEWLINE,
string='\n',
start=(1, 2),
end=(1, 3),
line='42\n')]
def test_repeat_1_simple(self) -> None:
grammar = """
start: thing thing+ NEWLINE
thing: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("1 2 3\n", parser_class)
self.assertEqual(node, [
[TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1 2 3\n")],
[
[[TokenInfo(NUMBER, string="2", start=(1, 2), end=(1, 3), line="1 2 3\n")]],
[[TokenInfo(NUMBER, string="3", start=(1, 4), end=(1, 5), line="1 2 3\n")]],
],
TokenInfo(NEWLINE, string="\n", start=(1, 5), end=(1, 6), line="1 2 3\n"),
])
with self.assertRaises(SyntaxError):
parse_string("1\n", parser_class)
def test_repeat_0_simple(self) -> None:
grammar = """
start: thing thing* NEWLINE
thing: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("1 2 3\n", parser_class)
self.assertEqual(
node,
[
TokenInfo(NUMBER,
string="1",
start=(1, 0),
end=(1, 1),
line="1 2 3\n"),
[
TokenInfo(NUMBER,
string="2",
start=(1, 2),
end=(1, 3),
line="1 2 3\n"),
TokenInfo(NUMBER,
string="3",
start=(1, 4),
end=(1, 5),
line="1 2 3\n"),
],
TokenInfo(NEWLINE,
string="\n",
start=(1, 5),
end=(1, 6),
line="1 2 3\n"),
],
)
node = parse_string("1\n", parser_class)
self.assertEqual(
node,
[
TokenInfo(
NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n"),
[],
TokenInfo(
NEWLINE, string="\n", start=(1, 1), end=(1, 2),
line="1\n"),
],
)
def test_repeat_0_simple():
grammar = """
start: thing thing* NEWLINE
thing: NUMBER
"""
parser_class = make_parser(grammar)
node = parse_string("1 2 3\n", parser_class)
assert node == [
[
TokenInfo(NUMBER,
string="1",
start=(1, 0),
end=(1, 1),
line="1 2 3\n")
],
[
[[
TokenInfo(NUMBER,
string="2",
start=(1, 2),
end=(1, 3),
line="1 2 3\n")
]],
[[
TokenInfo(NUMBER,
string="3",
start=(1, 4),
end=(1, 5),
line="1 2 3\n")
]],
],
TokenInfo(NEWLINE,
string="\n",
start=(1, 5),
end=(1, 6),
line="1 2 3\n"),
]
node = parse_string("1\n", parser_class)
assert node == [
[TokenInfo(NUMBER, string="1", start=(1, 0), end=(1, 1), line="1\n")],
[],
TokenInfo(NEWLINE, string="\n", start=(1, 1), end=(1, 2), line="1\n"),
]
def part1(fname: str) -> int:
"""Part 1.
BAsically we'll just treat + and * the same. It will evaluate left-to-right.
>>> part1("./data/day18_test.txt")
585
"""
grammar = make_parser("""
start: expr NEWLINE $ { ast.Expression(expr, lineno=1, col_offset=0) }
expr: ( e=expr '+' t=term { ast.BinOp(e, ast.Add(), t, lineno=e.lineno, col_offset=e.col_offset, end_lineno=t.end_lineno, end_col_offset=t.end_col_offset) }
| e=expr '*' t=term { ast.BinOp(e, ast.Mult(), t, lineno=e.lineno, col_offset=e.col_offset, end_lineno=t.end_lineno, end_col_offset=t.end_col_offset) }
| term { term }
)
term: '(' expr ')' { expr } | atom { atom }
atom: n=NUMBER { ast.Constant(value=ast.literal_eval(n.string), lineno=n.start[0], col_offset=n.start[1], end_lineno=n.end[0], end_col_offset=n.end[1]) }"""
)
return sum(
eval(compile(parse_string(expr, grammar), "", "eval"))
for expr in get_data(fname))