def run_benchmark_stdlib(subcommand, parser): modes = {"compile": 2, "parse": 1, "check": 0} extension = None if parser == "pegen": extension = build_parser_and_generator( "../../Grammar/python.gram", "peg_extension/parse.c", compile_extension=True, skip_actions=False, ) for _ in range(3): parse_directory( "../../Lib", "../../Grammar/python.gram", verbose=False, excluded_files=[ "*/bad*", "*/lib2to3/tests/data/*", ], skip_actions=False, tree_arg=0, short=True, extension=extension, mode=modes[subcommand], parser=parser, )
def main() -> None: args = parser.parse_args() if args.diff and not args.grammar_file: parser.error("-d/--diff requires -g/--grammar-file") if args.multiline: sep = "\n" else: sep = " " program = sep.join(args.program) if args.grammar_file: sys.path.insert(0, os.curdir) from pegen.build import build_parser_and_generator build_parser_and_generator(args.grammar_file, "peg_parser/parse.c", compile_extension=True) from pegen.parse import parse_string # type: ignore[import] tree = parse_string(program, mode=1) if args.diff: a = tree b = ast.parse(program) diff = diff_trees(a, b, args.verbose) if diff: for line in diff: print(line) else: print("# Trees are the same") else: print(f"# Parsed using {args.grammar_file}") print(format_tree(tree, args.verbose)) else: tree = ast.parse(program) print("# Parse using ast.parse()") print(format_tree(tree, args.verbose))
def main() -> None: args = argparser.parse_args() tree = args.tree extension = build.build_parser_and_generator( "data/simpy.gram", "pegen/parse.c", compile_extension=True ) for package in get_packages(): print(f"Extracting files from {package}... ", end="") try: extract_files(package) print("Done") except ValueError as e: print(e) continue print(f"Trying to parse all python files ... ") dirname = find_dirname(package) status = run_tests(dirname, tree, extension) if status == 0: print("Done") shutil.rmtree(dirname) else: print(f"Failed to parse {dirname}")
def parse_directory( directory: str, grammar_file: str, verbose: bool, excluded_files: List[str], skip_actions: bool, tree_arg: int, short: bool, extension: Any, mode: int, parser: str, ) -> int: if parser == "cpython" and (tree_arg or mode == 0): print( "Cannot specify tree argument or mode=0 with the cpython parser.", file=sys.stderr) return 1 if not directory: print("You must specify a directory of files to test.", file=sys.stderr) return 1 if grammar_file: if not os.path.exists(grammar_file): print( f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr) return 1 try: if not extension and parser == "pegen": build_parser_and_generator( grammar_file, "peg_extension/parse.c", compile_extension=True, skip_actions=skip_actions, ) except Exception as err: print( f"{FAIL}The following error occurred when generating the parser. Please check your grammar file.\n{ENDC}", file=sys.stderr, ) traceback.print_exception(err.__class__, err, None) return 1 else: print( "A grammar file was not provided - attempting to use existing file...\n" ) if parser == "pegen": try: from peg_extension import parse # type: ignore except: print( "An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.", file=sys.stderr, ) return 1 # For a given directory, traverse files and attempt to parse each one # - Output success/failure for each file errors = 0 files = [] trees = {} # Trees to compare (after everything else is done) t0 = time.time() for file in sorted(glob(f"{directory}/**/*.py", recursive=True)): # Only attempt to parse Python files and files that are not excluded should_exclude_file = False for pattern in excluded_files: if PurePath(file).match(pattern): should_exclude_file = True break if not should_exclude_file: try: if tree_arg: mode = 1 if parser == "cpython": with open(file, "r") as f: source = f.read() if mode == 2: compile(source, file, "exec") elif mode == 1: ast.parse(source, file, "exec") else: tree = parse.parse_file(file, mode=mode) if tree_arg: trees[file] = tree if not short: report_status(succeeded=True, file=file, verbose=verbose) except Exception as error: try: ast.parse(file) except Exception: if not short: print( f"File {file} cannot be parsed by either pegen or the ast module." ) else: report_status(succeeded=False, file=file, verbose=verbose, error=error, short=short) errors += 1 files.append(file) t1 = time.time() total_seconds = t1 - t0 total_files = len(files) total_bytes = 0 total_lines = 0 for file in files: # Count lines and bytes separately with open(file, "rb") as f: total_lines += sum(1 for _ in f) total_bytes += f.tell() print( f"Checked {total_files:,} files, {total_lines:,} lines,", f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.", ) if total_seconds > 0: print( f"That's {total_lines / total_seconds :,.0f} lines/sec,", f"or {total_bytes / total_seconds :,.0f} bytes/sec.", ) if parser == "pegen": # Dump memo stats to @data. with open("@data", "w") as datafile: for i, count in enumerate(parse.get_memo_stats()): if count: datafile.write(f"{i:4d} {count:9d}\n") if short: print_memstats() if errors: print(f"Encountered {errors} failures.", file=sys.stderr) # Compare trees (the dict is empty unless -t is given) compare_trees_errors = 0 for file, tree in trees.items(): if not short: print("Comparing ASTs for", file) if compare_trees(tree, file, verbose, tree_arg >= 2) == 1: compare_trees_errors += 1 if errors or compare_trees_errors: return 1 return 0
def main() -> None: args = argparser.parse_args() directory = args.directory grammar_file = args.grammar_file verbose = args.verbose excluded_files = args.exclude if not directory: print("You must specify a directory of files to test.", file=sys.stderr) sys.exit(1) if grammar_file: if not os.path.exists(grammar_file): print(f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr) sys.exit(1) try: build_parser_and_generator(grammar_file, "pegen/parse.c", True) except Exception as err: print( f"{FAIL}The following error occurred when generating the parser. Please check your grammar file.\n{ENDC}", file=sys.stderr, ) traceback.print_exception(err.__class__, err, None) sys.exit(1) else: print("A grammar file was not provided - attempting to use existing file...\n") try: from pegen import parse # type: ignore [attr-defined] except: print( "An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.", file=sys.stderr, ) sys.exit(1) # For a given directory, traverse files and attempt to parse each one # - Output success/failure for each file errors = 0 files = [] t0 = time.time() for file in sorted(glob(f"{directory}/**/*.py", recursive=True)): # Only attempt to parse Python files and files that are not excluded should_exclude_file = False for pattern in excluded_files: if PurePath(file).match(pattern): should_exclude_file = True break if not should_exclude_file: try: parse.parse_file(file) if not args.short: report_status(succeeded=True, file=file, verbose=verbose) except Exception as error: report_status( succeeded=False, file=file, verbose=verbose, error=error, short=args.short ) errors += 1 files.append(file) t1 = time.time() total_seconds = t1 - t0 total_files = len(files) total_bytes = 0 total_lines = 0 for file in files: # Count lines and bytes separately with open(file, "rb") as f: total_lines += sum(1 for _ in f) total_bytes += f.tell() print( f"Checked {total_files:,} files, {total_lines:,} lines,", f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.", ) if total_seconds > 0: print( f"That's {total_lines / total_seconds :,.0f} lines/sec,", f"or {total_bytes / total_seconds :,.0f} bytes/sec.", ) if args.short: print_memstats() if errors: print(f"Encountered {errors} failures.", file=sys.stderr) sys.exit(1)
def main() -> None: args = argparser.parse_args() verbose = args.verbose verbose_tokenizer = verbose >= 3 verbose_parser = verbose == 2 or verbose >= 4 t0 = time.time() output_file = args.output if not output_file: if args.cpython: output_file = "parse.c" else: output_file = "parse.py" try: grammar, parser, tokenizer, gen = build_parser_and_generator( args.filename, output_file, args.compile_extension, verbose_tokenizer, verbose_parser, args.verbose, keep_asserts_in_extension=False if args.optimized else True, skip_actions=args.skip_actions, ) except Exception as err: if args.verbose: raise # Show traceback traceback.print_exception(err.__class__, err, None) sys.stderr.write("For full traceback, use -v\n") sys.exit(1) if not args.quiet: if args.verbose: print("Raw Grammar:") for line in repr(grammar).splitlines(): print(" ", line) print("Clean Grammar:") for line in str(grammar).splitlines(): print(" ", line) if args.verbose: print("First Graph:") for src, dsts in gen.first_graph.items(): print(f" {src} -> {', '.join(dsts)}") print("First SCCS:") for scc in gen.first_sccs: print(" ", scc, end="") if len(scc) > 1: print( " # Indirectly left-recursive; leaders:", {name for name in scc if grammar.rules[name].leader}, ) else: name = next(iter(scc)) if name in gen.first_graph[name]: print(" # Left-recursive") else: print() t1 = time.time() if args.verbose: dt = t1 - t0 diag = tokenizer.diagnose() nlines = diag.end[0] if diag.type == token.ENDMARKER: nlines -= 1 print(f"Total time: {dt:.3f} sec; {nlines} lines", end="") if dt: print(f"; {nlines / dt:.0f} lines/sec") else: print() print("Caches sizes:") print(f" token array : {len(tokenizer._tokens):10}") print(f" cache : {len(parser._cache):10}") if not print_memstats(): print("(Can't find psutil; install it for memory stats.)")
def main(): args = argparser.parse_args() directory = args.directory grammar_file = args.grammar_file verbose = args.verbose excluded_files = args.exclude if not directory: print("You must specify a directory of files to test.", file=sys.stderr) sys.exit(1) if grammar_file: if not os.path.exists(grammar_file): print( f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr, ) sys.exit(1) try: build_parser_and_generator(grammar_file, "pegen/parse.c", True) except Exception as err: print( f"{FAIL}The following error occurred when generating the parser. Please check your grammar file.\n{ENDC}", file=sys.stderr, ) traceback.print_exception(err.__class__, err, None) sys.exit(1) else: print( "A grammar file was not provided - attempting to use existing file...\n" ) try: from pegen import parse except: print( "An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.", file=sys.stderr, ) sys.exit(1) # For a given directory, traverse files and attempt to parse each one # - Output success/failure for each file errors = 0 for file in sorted(glob(f"{directory}/**/*.py", recursive=True)): # Only attempt to parse Python files and files that are not excluded should_exclude_file = False for pattern in excluded_files: if PurePath(file).match(pattern): should_exclude_file = True break if not should_exclude_file: try: parse.parse_file(file) if not args.short: report_status(succeeded=True, file=file, verbose=verbose) except Exception as error: report_status( succeeded=False, file=file, verbose=verbose, error=error, short=args.short, ) errors += 1 if errors: print(f"Encountered {errors} failures.", file=sys.stderr) sys.exit(1)