def generate_parser_c_extension(
grammar: Grammar,
path: pathlib.PurePath,
debug: bool = False,
library_dir: Optional[str] = None,
) -> Any:
"""Generate a parser c extension for the given grammar in the given path
Returns a module object with a parse_string() method.
TODO: express that using a Protocol.
"""
# Make sure that the working directory is empty: reusing non-empty temporary
# directories when generating extensions can lead to segmentation faults.
# Check issue #95 (https://github.com/gvanrossum/pegen/issues/95) for more
# context.
assert not os.listdir(path)
source = path / "parse.c"
with open(source, "w", encoding="utf-8") as file:
genr = CParserGenerator(grammar,
ALL_TOKENS,
EXACT_TOKENS,
NON_EXACT_TOKENS,
file,
debug=debug)
genr.generate("parse.c")
compile_c_extension(
str(source),
build_dir=str(path),
# Significant test_peg_generator speedups
disable_optimization=True,
library_dir=library_dir,
)
def generate_parser_c_extension(grammar: Grammar,
path: pathlib.PurePath,
debug: bool = False) -> Any:
"""Generate a parser c extension for the given grammar in the given path
Returns a module object with a parse_string() method.
TODO: express that using a Protocol.
"""
# Make sure that the working directory is empty: reusing non-empty temporary
# directories when generating extensions can lead to segmentation faults.
# Check issue #95 (https://github.com/gvanrossum/pegen/issues/95) for more
# context.
assert not os.listdir(path)
source = path / "parse.c"
with open(source, "w") as file:
genr = CParserGenerator(grammar, file, debug=debug)
genr.generate("parse.c")
compile_c_extension(str(source), build_dir=str(path))
def generate_parser_c_extension(rules, path):
"""Generate a parser c extension for the given rules in the given path"""
source = path / "parse.c"
with open(source, "w") as file:
genr = CParserGenerator(rules, file)
genr.generate("parse.c")
extension_path = compile_c_extension(str(source),
build_dir=str(path / "build"))
extension = import_file("parse", extension_path)
return extension
def generate_parser_c_extension(grammar: Grammar,
path: pathlib.PurePath) -> Any:
"""Generate a parser c extension for the given grammar in the given path
Returns a module object with a parse_string() method.
TODO: express that using a Protocol.
"""
source = path / "parse.c"
with open(source, "w") as file:
genr = CParserGenerator(grammar, file)
genr.generate("parse.c")
extension_path = compile_c_extension(str(source),
build_dir=str(path / "build"))
extension = import_file("parse", extension_path)
return extension
def main() -> None:
args = argparser.parse_args()
verbose = args.verbose
verbose_tokenizer = verbose >= 3
verbose_parser = verbose == 2 or verbose >= 4
t0 = time.time()
with open(args.filename) as file:
tokenizer = Tokenizer(grammar_tokenizer(
tokenize.generate_tokens(file.readline)),
verbose=verbose_tokenizer)
parser = GrammarParser(tokenizer, verbose=verbose_parser)
rules = parser.start()
if not rules:
err = parser.make_syntax_error(args.filename)
traceback.print_exception(err.__class__, err, None)
sys.exit(1)
endpos = file.tell()
if not args.quiet:
if args.verbose:
print("Raw Grammar:")
for rule in rules.rules.values():
print(" ", repr(rule))
print("Clean Grammar:")
for rule in rules.rules.values():
print(" ", rule)
output = args.output
if not output:
if args.cpython:
output = "parse.c"
else:
output = "parse.py"
with open(output, 'w') as file:
gen: ParserGenerator
if args.cpython:
gen = CParserGenerator(rules.rules, file)
else:
gen = PythonParserGenerator(rules.rules, file)
gen.generate(args.filename)
if args.cpython and args.compile_extension:
compile_c_extension(output, verbose=args.verbose)
if args.verbose:
print("First Graph:")
for src, dsts in gen.first_graph.items():
print(f" {src} -> {', '.join(dsts)}")
print("First SCCS:")
for scc in gen.first_sccs:
print(" ", scc, end="")
if len(scc) > 1:
print(" # Indirectly left-recursive")
else:
name = next(iter(scc))
if name in gen.first_graph[name]:
print(" # Left-recursive")
else:
print()
t1 = time.time()
if args.verbose:
dt = t1 - t0
diag = tokenizer.diagnose()
nlines = diag.end[0]
if diag.type == token.ENDMARKER:
nlines -= 1
print(f"Total time: {dt:.3f} sec; {nlines} lines", end="")
if endpos:
print(f" ({endpos} bytes)", end="")
if dt:
print(f"; {nlines / dt:.0f} lines/sec")
else:
print()
print("Caches sizes:")
print(f" token array : {len(tokenizer._tokens):10}")
print(f" cache : {len(parser._cache):10}")
if not print_memstats():
print("(Can't find psutil; install it for memory stats.)")