def generate_parser(grammar: Grammar) -> Type[Parser]:
# Generate a parser.
out = io.StringIO()
genr = PythonParserGenerator(grammar, out)
genr.generate("<string>")
# Load the generated parser class.
ns: Dict[str, Any] = {}
exec(out.getvalue(), ns)
return ns["GeneratedParser"]
def generate_parser(rules):
# Generate a parser.
out = io.StringIO()
genr = PythonParserGenerator(rules, out)
genr.generate("<string>")
# Load the generated parser class.
ns = {}
exec(out.getvalue(), ns)
return ns['GeneratedParser']
def test_unreachable_explicit() -> None:
source = """
start: NAME { UNREACHABLE }
"""
grammar = parse_string(source, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar,
out,
unreachable_formatting="This is a test")
genr.generate("<string>")
assert "This is a test" in out.getvalue()
def test_unreachable_implicit3() -> None:
source = """
start: NAME | invalid_input { None }
invalid_input: NUMBER
"""
grammar = parse_string(source, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar,
out,
unreachable_formatting="This is a test")
genr.generate("<string>")
assert "This is a test" not in out.getvalue()
def test_unreachable_implicit2(self) -> None:
source = """
start: NAME | '(' invalid_input ')'
invalid_input: NUMBER { None }
"""
grammar = parse_string(source, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar,
out,
unreachable_formatting="This is a test")
genr.generate("<string>")
self.assertIn("This is a test", out.getvalue())
def generate_parser(grammar: Grammar,
parser_path: Optional[str] = None,
parser_name: str = "GeneratedParser") -> Type[Parser]:
# Generate a parser.
out = io.StringIO()
genr = PythonParserGenerator(grammar, out)
genr.generate("<string>")
# Load the generated parser class.
ns: Dict[str, Any] = {}
if parser_path:
with open(parser_path, "w") as f:
f.write(out.getvalue())
mod = import_file("py_parser", parser_path)
return getattr(mod, parser_name)
else:
exec(out.getvalue(), ns)
return ns[parser_name]
def test_mutually_left_recursive(self) -> None:
grammar_source = """
start: foo 'E'
foo: bar 'A' | 'B'
bar: foo 'C' | 'D'
"""
grammar: Grammar = parse_string(grammar_source, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar, out)
rules = grammar.rules
self.assertFalse(rules["start"].left_recursive)
self.assertTrue(rules["foo"].left_recursive)
self.assertTrue(rules["bar"].left_recursive)
genr.generate("<string>")
ns: Dict[str, Any] = {}
exec(out.getvalue(), ns)
parser_class: Type[Parser] = ns["GeneratedParser"]
node = parse_string("D A C A E", parser_class)
self.assertEqual(node, [
[
[
[
[TokenInfo(type=NAME, string="D", start=(1, 0), end=(1, 1), line="D A C A E")],
TokenInfo(type=NAME, string="A", start=(1, 2), end=(1, 3), line="D A C A E"),
],
TokenInfo(type=NAME, string="C", start=(1, 4), end=(1, 5), line="D A C A E"),
],
TokenInfo(type=NAME, string="A", start=(1, 6), end=(1, 7), line="D A C A E"),
],
TokenInfo(type=NAME, string="E", start=(1, 8), end=(1, 9), line="D A C A E"),
])
node = parse_string("B C A E", parser_class)
self.assertIsNotNone(node)
self.assertEqual(node, [
[
[
[TokenInfo(type=NAME, string="B", start=(1, 0), end=(1, 1), line="B C A E")],
TokenInfo(type=NAME, string="C", start=(1, 2), end=(1, 3), line="B C A E"),
],
TokenInfo(type=NAME, string="A", start=(1, 4), end=(1, 5), line="B C A E"),
],
TokenInfo(type=NAME, string="E", start=(1, 6), end=(1, 7), line="B C A E"),
])
def test_nasty_mutually_left_recursive():
# This grammar does not recognize 'x - + =', much to my chagrin.
# But that's the way PEG works.
# [Breathlessly]
# The problem is that the toplevel target call
# recurses into maybe, which recognizes 'x - +',
# and then the toplevel target looks for another '+',
# which fails, so it retreats to NAME,
# which succeeds, so we end up just recognizing 'x',
# and then start fails because there's no '=' after that.
grammar = """
start: target '='
target: maybe '+' | NAME
maybe: maybe '-' | target
"""
grammar = parse_string(grammar, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar, out)
genr.generate("<string>")
ns = {}
exec(out.getvalue(), ns)
parser_class = ns["GeneratedParser"]
with pytest.raises(SyntaxError):
parse_string("x - + =", parser_class)
def test_mutually_left_recursive():
grammar = """
start: foo 'E'
foo: bar 'A' | 'B'
bar: foo 'C' | 'D'
"""
grammar = parse_string(grammar, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar, out)
rules = grammar.rules
assert not rules["start"].left_recursive
assert rules["foo"].left_recursive
assert rules["bar"].left_recursive
genr.generate("<string>")
ns = {}
exec(out.getvalue(), ns)
parser_class = ns["GeneratedParser"]
node = parse_string("D A C A E", parser_class)
assert node == [
[
[
[
[
TokenInfo(type=NAME,
string="D",
start=(1, 0),
end=(1, 1),
line="D A C A E")
],
TokenInfo(type=NAME,
string="A",
start=(1, 2),
end=(1, 3),
line="D A C A E"),
],
TokenInfo(type=NAME,
string="C",
start=(1, 4),
end=(1, 5),
line="D A C A E"),
],
TokenInfo(type=NAME,
string="A",
start=(1, 6),
end=(1, 7),
line="D A C A E"),
],
TokenInfo(type=NAME,
string="E",
start=(1, 8),
end=(1, 9),
line="D A C A E"),
]
node = parse_string("B C A E", parser_class)
assert node != None
assert node == [
[
[
[
TokenInfo(type=NAME,
string="B",
start=(1, 0),
end=(1, 1),
line="B C A E")
],
TokenInfo(type=NAME,
string="C",
start=(1, 2),
end=(1, 3),
line="B C A E"),
],
TokenInfo(type=NAME,
string="A",
start=(1, 4),
end=(1, 5),
line="B C A E"),
],
TokenInfo(type=NAME,
string="E",
start=(1, 6),
end=(1, 7),
line="B C A E"),
]
def test_mutually_left_recursive():
grammar = """
start: foo 'E'
foo: bar 'A' | 'B'
bar: foo 'C' | 'D'
"""
rules = parse_string(grammar, GrammarParser).rules
out = io.StringIO()
genr = PythonParserGenerator(rules, out)
assert not rules['start'].left_recursive
assert rules['foo'].left_recursive
assert rules['bar'].left_recursive
genr.generate("<string>")
ns = {}
exec(out.getvalue(), ns)
parser_class = ns['GeneratedParser']
node = parse_string("D A C A E", parser_class)
assert node == [[[[[
TokenInfo(type=NAME,
string='D',
start=(1, 0),
end=(1, 1),
line='D A C A E')
],
TokenInfo(type=NAME,
string='A',
start=(1, 2),
end=(1, 3),
line='D A C A E')],
TokenInfo(type=NAME,
string='C',
start=(1, 4),
end=(1, 5),
line='D A C A E')],
TokenInfo(type=NAME,
string='A',
start=(1, 6),
end=(1, 7),
line='D A C A E')],
TokenInfo(type=NAME,
string='E',
start=(1, 8),
end=(1, 9),
line='D A C A E')]
node = parse_string("B C A E", parser_class)
assert node != None
assert node == [[[[
TokenInfo(type=NAME,
string='B',
start=(1, 0),
end=(1, 1),
line='B C A E')
],
TokenInfo(type=NAME,
string='C',
start=(1, 2),
end=(1, 3),
line='B C A E')],
TokenInfo(type=NAME,
string='A',
start=(1, 4),
end=(1, 5),
line='B C A E')],
TokenInfo(type=NAME,
string='E',
start=(1, 6),
end=(1, 7),
line='B C A E')]