def run_benchmark_stdlib(subcommand, parser):
modes = {"compile": 2, "parse": 1, "check": 0}
extension = None
if parser == "pegen":
extension = build_parser_and_generator(
"../../Grammar/python.gram",
"peg_extension/parse.c",
compile_extension=True,
skip_actions=False,
)
for _ in range(3):
parse_directory(
"../../Lib",
"../../Grammar/python.gram",
verbose=False,
excluded_files=[
"*/bad*",
"*/lib2to3/tests/data/*",
],
skip_actions=False,
tree_arg=0,
short=True,
extension=extension,
mode=modes[subcommand],
parser=parser,
)
def main() -> None:
args = parser.parse_args()
if args.diff and not args.grammar_file:
parser.error("-d/--diff requires -g/--grammar-file")
if args.multiline:
sep = "\n"
else:
sep = " "
program = sep.join(args.program)
if args.grammar_file:
sys.path.insert(0, os.curdir)
from pegen.build import build_parser_and_generator
build_parser_and_generator(args.grammar_file,
"peg_parser/parse.c",
compile_extension=True)
from pegen.parse import parse_string # type: ignore[import]
tree = parse_string(program, mode=1)
if args.diff:
a = tree
b = ast.parse(program)
diff = diff_trees(a, b, args.verbose)
if diff:
for line in diff:
print(line)
else:
print("# Trees are the same")
else:
print(f"# Parsed using {args.grammar_file}")
print(format_tree(tree, args.verbose))
else:
tree = ast.parse(program)
print("# Parse using ast.parse()")
print(format_tree(tree, args.verbose))
def main() -> None:
args = argparser.parse_args()
tree = args.tree
extension = build.build_parser_and_generator(
"data/simpy.gram", "pegen/parse.c", compile_extension=True
)
for package in get_packages():
print(f"Extracting files from {package}... ", end="")
try:
extract_files(package)
print("Done")
except ValueError as e:
print(e)
continue
print(f"Trying to parse all python files ... ")
dirname = find_dirname(package)
status = run_tests(dirname, tree, extension)
if status == 0:
print("Done")
shutil.rmtree(dirname)
else:
print(f"Failed to parse {dirname}")
def parse_directory(
directory: str,
grammar_file: str,
verbose: bool,
excluded_files: List[str],
skip_actions: bool,
tree_arg: int,
short: bool,
extension: Any,
mode: int,
parser: str,
) -> int:
if parser == "cpython" and (tree_arg or mode == 0):
print(
"Cannot specify tree argument or mode=0 with the cpython parser.",
file=sys.stderr)
return 1
if not directory:
print("You must specify a directory of files to test.",
file=sys.stderr)
return 1
if grammar_file:
if not os.path.exists(grammar_file):
print(
f"The specified grammar file, {grammar_file}, does not exist.",
file=sys.stderr)
return 1
try:
if not extension and parser == "pegen":
build_parser_and_generator(
grammar_file,
"peg_extension/parse.c",
compile_extension=True,
skip_actions=skip_actions,
)
except Exception as err:
print(
f"{FAIL}The following error occurred when generating the parser. Please check your grammar file.\n{ENDC}",
file=sys.stderr,
)
traceback.print_exception(err.__class__, err, None)
return 1
else:
print(
"A grammar file was not provided - attempting to use existing file...\n"
)
if parser == "pegen":
try:
from peg_extension import parse # type: ignore
except:
print(
"An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.",
file=sys.stderr,
)
return 1
# For a given directory, traverse files and attempt to parse each one
# - Output success/failure for each file
errors = 0
files = []
trees = {} # Trees to compare (after everything else is done)
t0 = time.time()
for file in sorted(glob(f"{directory}/**/*.py", recursive=True)):
# Only attempt to parse Python files and files that are not excluded
should_exclude_file = False
for pattern in excluded_files:
if PurePath(file).match(pattern):
should_exclude_file = True
break
if not should_exclude_file:
try:
if tree_arg:
mode = 1
if parser == "cpython":
with open(file, "r") as f:
source = f.read()
if mode == 2:
compile(source, file, "exec")
elif mode == 1:
ast.parse(source, file, "exec")
else:
tree = parse.parse_file(file, mode=mode)
if tree_arg:
trees[file] = tree
if not short:
report_status(succeeded=True, file=file, verbose=verbose)
except Exception as error:
try:
ast.parse(file)
except Exception:
if not short:
print(
f"File {file} cannot be parsed by either pegen or the ast module."
)
else:
report_status(succeeded=False,
file=file,
verbose=verbose,
error=error,
short=short)
errors += 1
files.append(file)
t1 = time.time()
total_seconds = t1 - t0
total_files = len(files)
total_bytes = 0
total_lines = 0
for file in files:
# Count lines and bytes separately
with open(file, "rb") as f:
total_lines += sum(1 for _ in f)
total_bytes += f.tell()
print(
f"Checked {total_files:,} files, {total_lines:,} lines,",
f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.",
)
if total_seconds > 0:
print(
f"That's {total_lines / total_seconds :,.0f} lines/sec,",
f"or {total_bytes / total_seconds :,.0f} bytes/sec.",
)
if parser == "pegen":
# Dump memo stats to @data.
with open("@data", "w") as datafile:
for i, count in enumerate(parse.get_memo_stats()):
if count:
datafile.write(f"{i:4d} {count:9d}\n")
if short:
print_memstats()
if errors:
print(f"Encountered {errors} failures.", file=sys.stderr)
# Compare trees (the dict is empty unless -t is given)
compare_trees_errors = 0
for file, tree in trees.items():
if not short:
print("Comparing ASTs for", file)
if compare_trees(tree, file, verbose, tree_arg >= 2) == 1:
compare_trees_errors += 1
if errors or compare_trees_errors:
return 1
return 0
def main() -> None:
args = argparser.parse_args()
directory = args.directory
grammar_file = args.grammar_file
verbose = args.verbose
excluded_files = args.exclude
if not directory:
print("You must specify a directory of files to test.", file=sys.stderr)
sys.exit(1)
if grammar_file:
if not os.path.exists(grammar_file):
print(f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr)
sys.exit(1)
try:
build_parser_and_generator(grammar_file, "pegen/parse.c", True)
except Exception as err:
print(
f"{FAIL}The following error occurred when generating the parser. Please check your grammar file.\n{ENDC}",
file=sys.stderr,
)
traceback.print_exception(err.__class__, err, None)
sys.exit(1)
else:
print("A grammar file was not provided - attempting to use existing file...\n")
try:
from pegen import parse # type: ignore [attr-defined]
except:
print(
"An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.",
file=sys.stderr,
)
sys.exit(1)
# For a given directory, traverse files and attempt to parse each one
# - Output success/failure for each file
errors = 0
files = []
t0 = time.time()
for file in sorted(glob(f"{directory}/**/*.py", recursive=True)):
# Only attempt to parse Python files and files that are not excluded
should_exclude_file = False
for pattern in excluded_files:
if PurePath(file).match(pattern):
should_exclude_file = True
break
if not should_exclude_file:
try:
parse.parse_file(file)
if not args.short:
report_status(succeeded=True, file=file, verbose=verbose)
except Exception as error:
report_status(
succeeded=False, file=file, verbose=verbose, error=error, short=args.short
)
errors += 1
files.append(file)
t1 = time.time()
total_seconds = t1 - t0
total_files = len(files)
total_bytes = 0
total_lines = 0
for file in files:
# Count lines and bytes separately
with open(file, "rb") as f:
total_lines += sum(1 for _ in f)
total_bytes += f.tell()
print(
f"Checked {total_files:,} files, {total_lines:,} lines,",
f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.",
)
if total_seconds > 0:
print(
f"That's {total_lines / total_seconds :,.0f} lines/sec,",
f"or {total_bytes / total_seconds :,.0f} bytes/sec.",
)
if args.short:
print_memstats()
if errors:
print(f"Encountered {errors} failures.", file=sys.stderr)
sys.exit(1)
def main() -> None:
args = argparser.parse_args()
verbose = args.verbose
verbose_tokenizer = verbose >= 3
verbose_parser = verbose == 2 or verbose >= 4
t0 = time.time()
output_file = args.output
if not output_file:
if args.cpython:
output_file = "parse.c"
else:
output_file = "parse.py"
try:
grammar, parser, tokenizer, gen = build_parser_and_generator(
args.filename,
output_file,
args.compile_extension,
verbose_tokenizer,
verbose_parser,
args.verbose,
keep_asserts_in_extension=False if args.optimized else True,
skip_actions=args.skip_actions,
)
except Exception as err:
if args.verbose:
raise # Show traceback
traceback.print_exception(err.__class__, err, None)
sys.stderr.write("For full traceback, use -v\n")
sys.exit(1)
if not args.quiet:
if args.verbose:
print("Raw Grammar:")
for line in repr(grammar).splitlines():
print(" ", line)
print("Clean Grammar:")
for line in str(grammar).splitlines():
print(" ", line)
if args.verbose:
print("First Graph:")
for src, dsts in gen.first_graph.items():
print(f" {src} -> {', '.join(dsts)}")
print("First SCCS:")
for scc in gen.first_sccs:
print(" ", scc, end="")
if len(scc) > 1:
print(
" # Indirectly left-recursive; leaders:",
{name
for name in scc if grammar.rules[name].leader},
)
else:
name = next(iter(scc))
if name in gen.first_graph[name]:
print(" # Left-recursive")
else:
print()
t1 = time.time()
if args.verbose:
dt = t1 - t0
diag = tokenizer.diagnose()
nlines = diag.end[0]
if diag.type == token.ENDMARKER:
nlines -= 1
print(f"Total time: {dt:.3f} sec; {nlines} lines", end="")
if dt:
print(f"; {nlines / dt:.0f} lines/sec")
else:
print()
print("Caches sizes:")
print(f" token array : {len(tokenizer._tokens):10}")
print(f" cache : {len(parser._cache):10}")
if not print_memstats():
print("(Can't find psutil; install it for memory stats.)")
def main():
args = argparser.parse_args()
directory = args.directory
grammar_file = args.grammar_file
verbose = args.verbose
excluded_files = args.exclude
if not directory:
print("You must specify a directory of files to test.",
file=sys.stderr)
sys.exit(1)
if grammar_file:
if not os.path.exists(grammar_file):
print(
f"The specified grammar file, {grammar_file}, does not exist.",
file=sys.stderr,
)
sys.exit(1)
try:
build_parser_and_generator(grammar_file, "pegen/parse.c", True)
except Exception as err:
print(
f"{FAIL}The following error occurred when generating the parser. Please check your grammar file.\n{ENDC}",
file=sys.stderr,
)
traceback.print_exception(err.__class__, err, None)
sys.exit(1)
else:
print(
"A grammar file was not provided - attempting to use existing file...\n"
)
try:
from pegen import parse
except:
print(
"An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.",
file=sys.stderr,
)
sys.exit(1)
# For a given directory, traverse files and attempt to parse each one
# - Output success/failure for each file
errors = 0
for file in sorted(glob(f"{directory}/**/*.py", recursive=True)):
# Only attempt to parse Python files and files that are not excluded
should_exclude_file = False
for pattern in excluded_files:
if PurePath(file).match(pattern):
should_exclude_file = True
break
if not should_exclude_file:
try:
parse.parse_file(file)
if not args.short:
report_status(succeeded=True, file=file, verbose=verbose)
except Exception as error:
report_status(
succeeded=False,
file=file,
verbose=verbose,
error=error,
short=args.short,
)
errors += 1
if errors:
print(f"Encountered {errors} failures.", file=sys.stderr)
sys.exit(1)
