def generate_c_code( args: argparse.Namespace, ) -> Tuple[Grammar, Parser, Tokenizer, ParserGenerator]: from pegen.build import build_c_parser_and_generator verbose = args.verbose verbose_tokenizer = verbose >= 3 verbose_parser = verbose == 2 or verbose >= 4 try: grammar, parser, tokenizer, gen = build_c_parser_and_generator( args.grammar_filename, args.tokens_filename, args.output, args.compile_extension, verbose_tokenizer, verbose_parser, args.verbose, keep_asserts_in_extension=False if args.optimized else True, skip_actions=args.skip_actions, ) return grammar, parser, tokenizer, gen except Exception as err: if args.verbose: raise # Show traceback traceback.print_exception(err.__class__, err, None) sys.stderr.write("For full traceback, use -v\n") sys.exit(1)
def run_benchmark_stdlib(subcommand, parser): modes = {"compile": 2, "parse": 1, "check": 0} extension = None if parser == "pegen": extension = build_c_parser_and_generator( "../../Grammar/python.gram", "../../Grammar/Tokens", "peg_extension/parse.c", compile_extension=True, skip_actions=False, ) for _ in range(3): parse_directory( "../../Lib", "../../Grammar/python.gram", verbose=False, excluded_files=[ "*/bad*", "*/lib2to3/tests/data/*", ], skip_actions=False, tree_arg=0, short=True, extension=extension, mode=modes[subcommand], parser=parser, )
def main() -> None: args = parser.parse_args() if args.diff and not args.grammar_file: parser.error("-d/--diff requires -g/--grammar-file") if args.multiline: sep = "\n" else: sep = " " program = sep.join(args.program) if args.grammar_file: sys.path.insert(0, os.curdir) from pegen.build import build_c_parser_and_generator build_c_parser_and_generator(args.grammar_file, "data/Tokens", "pegen/parse.c", compile_extension=True) from pegen.parse import parse_string # type: ignore[import] tree = parse_string(program, mode=1) if args.diff: a = tree b = ast.parse(program) diff = diff_trees(a, b, args.verbose) if diff: for line in diff: print(line) else: print("# Trees are the same") else: print(f"# Parsed using {args.grammar_file}") print(format_tree(tree, args.verbose)) else: tree = ast.parse(program) print("# Parse using ast.parse()") print(format_tree(tree, args.verbose))
def parse_directory( directory: str, grammar_file: str, tokens_file: str, verbose: bool, excluded_files: List[str], skip_actions: bool, tree_arg: int, short: bool, extension: Any, mode: int, parser: str, ) -> int: if parser == "cpython" and (tree_arg or mode == 0): print( "Cannot specify tree argument or mode=0 with the cpython parser.", file=sys.stderr) return 1 if not directory: print("You must specify a directory of files to test.", file=sys.stderr) return 1 if grammar_file and tokens_file: if not os.path.exists(grammar_file): print( f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr) return 1 try: if not extension and parser == "pegen": build_c_parser_and_generator( grammar_file, tokens_file, "peg_extension/parse.c", compile_extension=True, skip_actions=skip_actions, ) except Exception as err: print( f"{FAIL}The following error occurred when generating the parser. Please check your grammar file.\n{ENDC}", file=sys.stderr, ) traceback.print_exception(err.__class__, err, None) return 1 else: print( "A grammar file or a tokens file was not provided - attempting to use existing parser from stdlib...\n" ) if parser == "pegen": try: from peg_extension import parse # type: ignore except Exception as e: print( "An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.", file=sys.stderr, ) return 1 # For a given directory, traverse files and attempt to parse each one # - Output success/failure for each file errors = 0 files = [] trees = {} # Trees to compare (after everything else is done) t0 = time.time() for file in sorted(glob(f"{directory}/**/*.py", recursive=True)): # Only attempt to parse Python files and files that are not excluded should_exclude_file = False for pattern in excluded_files: if PurePath(file).match(pattern): should_exclude_file = True break if not should_exclude_file: try: if tree_arg: mode = 1 if parser == "cpython": with open(file, "r") as f: source = f.read() if mode == 2: compile(source, file, "exec") elif mode == 1: ast.parse(source, file, "exec") else: tree = parse.parse_file(file, mode=mode) if tree_arg: trees[file] = tree if not short: report_status(succeeded=True, file=file, verbose=verbose) except Exception as error: try: ast.parse(file) except Exception: if not short: print( f"File {file} cannot be parsed by either pegen or the ast module." ) else: report_status(succeeded=False, file=file, verbose=verbose, error=error, short=short) errors += 1 files.append(file) t1 = time.time() total_seconds = t1 - t0 total_files = len(files) total_bytes = 0 total_lines = 0 for file in files: # Count lines and bytes separately with open(file, "rb") as f: total_lines += sum(1 for _ in f) total_bytes += f.tell() print( f"Checked {total_files:,} files, {total_lines:,} lines,", f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.", ) if total_seconds > 0: print( f"That's {total_lines / total_seconds :,.0f} lines/sec,", f"or {total_bytes / total_seconds :,.0f} bytes/sec.", ) if parser == "pegen": # Dump memo stats to @data. with open("@data", "w") as datafile: for i, count in enumerate(parse.get_memo_stats()): if count: datafile.write(f"{i:4d} {count:9d}\n") if short: print_memstats() if errors: print(f"Encountered {errors} failures.", file=sys.stderr) # Compare trees (the dict is empty unless -t is given) compare_trees_errors = 0 for file, tree in trees.items(): if not short: print("Comparing ASTs for", file) if compare_trees(tree, file, verbose, tree_arg >= 2) == 1: compare_trees_errors += 1 if errors or compare_trees_errors: return 1 return 0