def generate_parser(rules):
# Generate a parser.
out = io.StringIO()
genr = PythonParserGenerator(rules, out)
genr.generate("<string>")
# Load the generated parser class.
ns = {}
exec(out.getvalue(), ns)
return ns['GeneratedParser']
def generate_parser(grammar: Grammar) -> Type[Parser]:
# Generate a parser.
out = io.StringIO()
genr = PythonParserGenerator(grammar, out)
genr.generate("<string>")
# Load the generated parser class.
ns: Dict[str, Any] = {}
exec(out.getvalue(), ns)
return ns["GeneratedParser"]
def test_unreachable_explicit() -> None:
source = """
start: NAME { UNREACHABLE }
"""
grammar = parse_string(source, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar,
out,
unreachable_formatting="This is a test")
genr.generate("<string>")
assert "This is a test" in out.getvalue()
def test_unreachable_implicit3() -> None:
source = """
start: NAME | invalid_input { None }
invalid_input: NUMBER
"""
grammar = parse_string(source, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar,
out,
unreachable_formatting="This is a test")
genr.generate("<string>")
assert "This is a test" not in out.getvalue()
def test_unreachable_implicit2(self) -> None:
source = """
start: NAME | '(' invalid_input ')'
invalid_input: NUMBER { None }
"""
grammar = parse_string(source, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar,
out,
unreachable_formatting="This is a test")
genr.generate("<string>")
self.assertIn("This is a test", out.getvalue())
def build_python_generator(
grammar: Grammar, grammar_file: str, output_file: str, skip_actions: bool = False,
) -> ParserGenerator:
with open(output_file, "w") as file:
gen: ParserGenerator = PythonParserGenerator(grammar, file) # TODO: skip_actions
gen.generate(grammar_file)
return gen
def build_generator(
tokenizer: Tokenizer,
grammar: Grammar,
grammar_file: str,
output_file: str,
compile_extension: bool = False,
verbose_c_extension: bool = False,
keep_asserts_in_extension: bool = True,
skip_actions: bool = False,
) -> ParserGenerator:
with open(output_file, "w") as file:
gen: ParserGenerator
if output_file.endswith(".c"):
gen = CParserGenerator(grammar, file, skip_actions=skip_actions)
elif output_file.endswith(".py"):
gen = PythonParserGenerator(grammar, file) # TODO: skip_actions
else:
raise Exception("Your output file must either be a .c or .py file")
gen.generate(grammar_file)
if compile_extension and output_file.endswith(".c"):
compile_c_extension(
output_file, verbose=verbose_c_extension, keep_asserts=keep_asserts_in_extension
)
return gen
def build_generator(
tokenizer: Tokenizer,
grammar: Grammar,
grammar_file: str,
output_file: str,
compile_extension: bool = False,
verbose_c_extension: bool = False,
keep_asserts_in_extension: bool = True,
skip_actions: bool = False,
) -> ParserGenerator:
# TODO: Allow other extensions; pass the output type as an argument.
if not output_file.endswith((".c", ".py")):
raise RuntimeError("Your output file must either be a .c or .py file")
with open(output_file, "w") as file:
gen: ParserGenerator
if output_file.endswith(".c"):
gen = CParserGenerator(grammar, file, skip_actions=skip_actions)
elif output_file.endswith(".py"):
gen = PythonParserGenerator(grammar, file) # TODO: skip_actions
else:
assert False # Should have been checked above
gen.generate(grammar_file)
if compile_extension and output_file.endswith(".c"):
compile_c_extension(output_file,
verbose=verbose_c_extension,
keep_asserts=keep_asserts_in_extension)
return gen
def generate_parser(grammar: Grammar,
parser_path: Optional[str] = None,
parser_name: str = "GeneratedParser") -> Type[Parser]:
# Generate a parser.
out = io.StringIO()
genr = PythonParserGenerator(grammar, out)
genr.generate("<string>")
# Load the generated parser class.
ns: Dict[str, Any] = {}
if parser_path:
with open(parser_path, "w") as f:
f.write(out.getvalue())
mod = import_file("py_parser", parser_path)
return getattr(mod, parser_name)
else:
exec(out.getvalue(), ns)
return ns[parser_name]
def test_mutually_left_recursive(self) -> None:
grammar_source = """
start: foo 'E'
foo: bar 'A' | 'B'
bar: foo 'C' | 'D'
"""
grammar: Grammar = parse_string(grammar_source, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar, out)
rules = grammar.rules
self.assertFalse(rules["start"].left_recursive)
self.assertTrue(rules["foo"].left_recursive)
self.assertTrue(rules["bar"].left_recursive)
genr.generate("<string>")
ns: Dict[str, Any] = {}
exec(out.getvalue(), ns)
parser_class: Type[Parser] = ns["GeneratedParser"]
node = parse_string("D A C A E", parser_class)
self.assertEqual(node, [
[
[
[
[TokenInfo(type=NAME, string="D", start=(1, 0), end=(1, 1), line="D A C A E")],
TokenInfo(type=NAME, string="A", start=(1, 2), end=(1, 3), line="D A C A E"),
],
TokenInfo(type=NAME, string="C", start=(1, 4), end=(1, 5), line="D A C A E"),
],
TokenInfo(type=NAME, string="A", start=(1, 6), end=(1, 7), line="D A C A E"),
],
TokenInfo(type=NAME, string="E", start=(1, 8), end=(1, 9), line="D A C A E"),
])
node = parse_string("B C A E", parser_class)
self.assertIsNotNone(node)
self.assertEqual(node, [
[
[
[TokenInfo(type=NAME, string="B", start=(1, 0), end=(1, 1), line="B C A E")],
TokenInfo(type=NAME, string="C", start=(1, 2), end=(1, 3), line="B C A E"),
],
TokenInfo(type=NAME, string="A", start=(1, 4), end=(1, 5), line="B C A E"),
],
TokenInfo(type=NAME, string="E", start=(1, 6), end=(1, 7), line="B C A E"),
])
def test_nullable():
grammar = """
start: sign NUMBER
sign: ['-' | '+']
"""
rules = parse_string(grammar, GrammarParser).rules
out = io.StringIO()
genr = PythonParserGenerator(rules, out)
assert rules['start'].nullable is False # Not None!
assert rules['sign'].nullable
def test_nullable():
grammar = """
start: sign NUMBER
sign: ['-' | '+']
"""
grammar = parse_string(grammar, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar, out)
rules = grammar.rules
assert rules["start"].nullable is False # Not None!
assert rules["sign"].nullable
def test_nullable(self) -> None:
grammar_source = """
start: sign NUMBER
sign: ['-' | '+']
"""
grammar: Grammar = parse_string(grammar_source, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar, out)
rules = grammar.rules
self.assertFalse(rules["start"].nullable) # Not None!
self.assertTrue(rules["sign"].nullable)
def test_advanced_left_recursive():
grammar = """
start: NUMBER | sign start
sign: ['-']
"""
rules = parse_string(grammar, GrammarParser).rules
out = io.StringIO()
genr = PythonParserGenerator(rules, out)
assert rules['start'].nullable is False # Not None!
assert rules['sign'].nullable
assert rules['start'].left_recursive
assert not rules['sign'].left_recursive
def test_advanced_left_recursive():
grammar = """
start: NUMBER | sign start
sign: ['-']
"""
grammar = parse_string(grammar, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar, out)
rules = grammar.rules
assert rules["start"].nullable is False # Not None!
assert rules["sign"].nullable
assert rules["start"].left_recursive
assert not rules["sign"].left_recursive
def test_advanced_left_recursive(self) -> None:
grammar_source = """
start: NUMBER | sign start
sign: ['-']
"""
grammar: Grammar = parse_string(grammar_source, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar, out)
rules = grammar.rules
self.assertFalse(rules["start"].nullable) # Not None!
self.assertTrue(rules["sign"].nullable)
self.assertTrue(rules["start"].left_recursive)
self.assertFalse(rules["sign"].left_recursive)
def test_nasty_mutually_left_recursive():
# This grammar does not recognize 'x - + =', much to my chagrin.
# But that's the way PEG works.
# [Breathlessly]
# The problem is that the toplevel target call
# recurses into maybe, which recognizes 'x - +',
# and then the toplevel target looks for another '+',
# which fails, so it retreats to NAME,
# which succeeds, so we end up just recognizing 'x',
# and then start fails because there's no '=' after that.
grammar = """
start: target '='
target: maybe '+' | NAME
maybe: maybe '-' | target
"""
grammar = parse_string(grammar, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar, out)
genr.generate("<string>")
ns = {}
exec(out.getvalue(), ns)
parser_class = ns["GeneratedParser"]
with pytest.raises(SyntaxError):
parse_string("x - + =", parser_class)
def build_generator(
tokenizer,
rules,
grammar_file,
output_file,
compile_extension=False,
verbose_c_extension=False,
):
with open(output_file, "w") as file:
gen: ParserGenerator
if output_file.endswith(".c"):
gen = CParserGenerator(rules.rules, file)
elif output_file.endswith(".py"):
gen = PythonParserGenerator(rules.rules, file)
else:
raise Exception("Your output file must either be a .c or .py file")
gen.generate(grammar_file)
if compile_extension and output_file.endswith(".c"):
compile_c_extension(output_file, verbose=verbose_c_extension)
return gen
def test_mutually_left_recursive():
grammar = """
start: foo 'E'
foo: bar 'A' | 'B'
bar: foo 'C' | 'D'
"""
grammar = parse_string(grammar, GrammarParser)
out = io.StringIO()
genr = PythonParserGenerator(grammar, out)
rules = grammar.rules
assert not rules["start"].left_recursive
assert rules["foo"].left_recursive
assert rules["bar"].left_recursive
genr.generate("<string>")
ns = {}
exec(out.getvalue(), ns)
parser_class = ns["GeneratedParser"]
node = parse_string("D A C A E", parser_class)
assert node == [
[
[
[
[
TokenInfo(type=NAME,
string="D",
start=(1, 0),
end=(1, 1),
line="D A C A E")
],
TokenInfo(type=NAME,
string="A",
start=(1, 2),
end=(1, 3),
line="D A C A E"),
],
TokenInfo(type=NAME,
string="C",
start=(1, 4),
end=(1, 5),
line="D A C A E"),
],
TokenInfo(type=NAME,
string="A",
start=(1, 6),
end=(1, 7),
line="D A C A E"),
],
TokenInfo(type=NAME,
string="E",
start=(1, 8),
end=(1, 9),
line="D A C A E"),
]
node = parse_string("B C A E", parser_class)
assert node != None
assert node == [
[
[
[
TokenInfo(type=NAME,
string="B",
start=(1, 0),
end=(1, 1),
line="B C A E")
],
TokenInfo(type=NAME,
string="C",
start=(1, 2),
end=(1, 3),
line="B C A E"),
],
TokenInfo(type=NAME,
string="A",
start=(1, 4),
end=(1, 5),
line="B C A E"),
],
TokenInfo(type=NAME,
string="E",
start=(1, 6),
end=(1, 7),
line="B C A E"),
]
def main() -> None:
args = argparser.parse_args()
verbose = args.verbose
verbose_tokenizer = verbose >= 3
verbose_parser = verbose == 2 or verbose >= 4
t0 = time.time()
with open(args.filename) as file:
tokenizer = Tokenizer(grammar_tokenizer(
tokenize.generate_tokens(file.readline)),
verbose=verbose_tokenizer)
parser = GrammarParser(tokenizer, verbose=verbose_parser)
rules = parser.start()
if not rules:
err = parser.make_syntax_error(args.filename)
traceback.print_exception(err.__class__, err, None)
sys.exit(1)
endpos = file.tell()
if not args.quiet:
if args.verbose:
print("Raw Grammar:")
for rule in rules.rules.values():
print(" ", repr(rule))
print("Clean Grammar:")
for rule in rules.rules.values():
print(" ", rule)
output = args.output
if not output:
if args.cpython:
output = "parse.c"
else:
output = "parse.py"
with open(output, 'w') as file:
gen: ParserGenerator
if args.cpython:
gen = CParserGenerator(rules.rules, file)
else:
gen = PythonParserGenerator(rules.rules, file)
gen.generate(args.filename)
if args.cpython and args.compile_extension:
compile_c_extension(output, verbose=args.verbose)
if args.verbose:
print("First Graph:")
for src, dsts in gen.first_graph.items():
print(f" {src} -> {', '.join(dsts)}")
print("First SCCS:")
for scc in gen.first_sccs:
print(" ", scc, end="")
if len(scc) > 1:
print(" # Indirectly left-recursive")
else:
name = next(iter(scc))
if name in gen.first_graph[name]:
print(" # Left-recursive")
else:
print()
t1 = time.time()
if args.verbose:
dt = t1 - t0
diag = tokenizer.diagnose()
nlines = diag.end[0]
if diag.type == token.ENDMARKER:
nlines -= 1
print(f"Total time: {dt:.3f} sec; {nlines} lines", end="")
if endpos:
print(f" ({endpos} bytes)", end="")
if dt:
print(f"; {nlines / dt:.0f} lines/sec")
else:
print()
print("Caches sizes:")
print(f" token array : {len(tokenizer._tokens):10}")
print(f" cache : {len(parser._cache):10}")
if not print_memstats():
print("(Can't find psutil; install it for memory stats.)")
def test_mutually_left_recursive():
grammar = """
start: foo 'E'
foo: bar 'A' | 'B'
bar: foo 'C' | 'D'
"""
rules = parse_string(grammar, GrammarParser).rules
out = io.StringIO()
genr = PythonParserGenerator(rules, out)
assert not rules['start'].left_recursive
assert rules['foo'].left_recursive
assert rules['bar'].left_recursive
genr.generate("<string>")
ns = {}
exec(out.getvalue(), ns)
parser_class = ns['GeneratedParser']
node = parse_string("D A C A E", parser_class)
assert node == [[[[[
TokenInfo(type=NAME,
string='D',
start=(1, 0),
end=(1, 1),
line='D A C A E')
],
TokenInfo(type=NAME,
string='A',
start=(1, 2),
end=(1, 3),
line='D A C A E')],
TokenInfo(type=NAME,
string='C',
start=(1, 4),
end=(1, 5),
line='D A C A E')],
TokenInfo(type=NAME,
string='A',
start=(1, 6),
end=(1, 7),
line='D A C A E')],
TokenInfo(type=NAME,
string='E',
start=(1, 8),
end=(1, 9),
line='D A C A E')]
node = parse_string("B C A E", parser_class)
assert node != None
assert node == [[[[
TokenInfo(type=NAME,
string='B',
start=(1, 0),
end=(1, 1),
line='B C A E')
],
TokenInfo(type=NAME,
string='C',
start=(1, 2),
end=(1, 3),
line='B C A E')],
TokenInfo(type=NAME,
string='A',
start=(1, 4),
end=(1, 5),
line='B C A E')],
TokenInfo(type=NAME,
string='E',
start=(1, 6),
end=(1, 7),
line='B C A E')]