def main() -> None: tok: Optional[tokenize.TokenInfo] t0 = time.time() ntoks = 0 nlines = 0 line_ends = (token.NL, token.NEWLINE) for filename in sys.argv[1:]: print(f"{nlines:10} lines", end="\r", file=sys.stderr) try: with open(filename) as file: toks = tokenize.generate_tokens(file.readline) for tok in toks: ntoks += 1 if tok.type in line_ends: nlines += 1 except Exception as err: print("Error:", err, file=sys.stderr) tok = None t1 = time.time() dt = t1 - t0 print(f"{ntoks} tokens, {nlines} lines in {dt:.3f} secs", file=sys.stderr) if dt: print(f"{ntoks/dt:.0f} tokens/sec, {nlines/dt:.0f} lines/sec", file=sys.stderr) print_memstats()
def main() -> None: filename = sys.argv[1] t0 = time.time() drv = driver.Driver(pygram.python_grammar, convert=pytree.convert) tree = drv.parse_file(filename) t1 = time.time() dt = t1 - t0 with open(filename) as file: nlines = len(file.readlines()) print("%.3f seconds for %d lines; %.0f lines/sec" % (dt, nlines, nlines / (dt or 1e-9))) print_memstats()
def main() -> None: t0 = time.time() for filename in sys.argv[1:]: print(filename, end="\r") try: with open(filename) as file: source = file.read() tree = ast.parse(source, filename) except Exception as err: print(f"{filename}: {err.__class__.__name__}: {err}", file=sys.stderr) tok = None t1 = time.time() dt = t1 - t0 print(f"Parsed in {dt:.3f} secs", file=sys.stderr) print_memstats()
def main() -> None: from pegen.testutil import print_memstats args = argparser.parse_args() if "func" not in args: argparser.error( "Must specify the target language mode ('c' or 'python')") t0 = time.time() grammar, parser, tokenizer, gen = args.func(args) t1 = time.time() validate_grammar(grammar) if not args.quiet: if args.verbose: print("Raw Grammar:") for line in repr(grammar).splitlines(): print(" ", line) print("Clean Grammar:") for line in str(grammar).splitlines(): print(" ", line) if args.verbose: print("First Graph:") for src, dsts in gen.first_graph.items(): print(f" {src} -> {', '.join(dsts)}") print("First SCCS:") for scc in gen.first_sccs: print(" ", scc, end="") if len(scc) > 1: print( " # Indirectly left-recursive; leaders:", {name for name in scc if grammar.rules[name].leader}, ) else: name = next(iter(scc)) if name in gen.first_graph[name]: print(" # Left-recursive") else: print() if args.verbose: dt = t1 - t0 diag = tokenizer.diagnose() nlines = diag.end[0] if diag.type == token.ENDMARKER: nlines -= 1 print(f"Total time: {dt:.3f} sec; {nlines} lines", end="") if dt: print(f"; {nlines / dt:.0f} lines/sec") else: print() print("Caches sizes:") print(f" token array : {len(tokenizer._tokens):10}") print(f" cache : {len(parser._cache):10}") if not print_memstats(): print("(Can't find psutil; install it for memory stats.)")
def parse_directory(directory: str, verbose: bool, excluded_files: List[str], short: bool) -> int: # For a given directory, traverse files and attempt to parse each one # - Output success/failure for each file errors = 0 files = [] total_seconds = 0 for file in sorted( glob(os.path.join(escape(directory), f"**/*.py"), recursive=True)): # Only attempt to parse Python files and files that are not excluded if any(PurePath(file).match(pattern) for pattern in excluded_files): continue with tokenize.open(file) as f: source = f.read() try: result, dt = parse_file(source, file) total_seconds += dt report_status(succeeded=True, file=file, verbose=verbose, short=short) except SyntaxError as error: report_status(succeeded=False, file=file, verbose=verbose, error=error, short=short) errors += 1 files.append(file) generate_time_stats(files, total_seconds) if short: print_memstats() if errors: print(f"Encountered {errors} failures.", file=sys.stderr) return 1 return 0
def parse_directory( directory: str, grammar_file: str, verbose: bool, excluded_files: List[str], skip_actions: bool, tree_arg: int, short: bool, extension: Any, mode: int, parser: str, ) -> int: if parser == "cpython" and (tree_arg or mode == 0): print( "Cannot specify tree argument or mode=0 with the cpython parser.", file=sys.stderr) return 1 if not directory: print("You must specify a directory of files to test.", file=sys.stderr) return 1 if grammar_file: if not os.path.exists(grammar_file): print( f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr) return 1 try: if not extension and parser == "pegen": build_parser_and_generator( grammar_file, "peg_extension/parse.c", compile_extension=True, skip_actions=skip_actions, ) except Exception as err: print( f"{FAIL}The following error occurred when generating the parser. Please check your grammar file.\n{ENDC}", file=sys.stderr, ) traceback.print_exception(err.__class__, err, None) return 1 else: print( "A grammar file was not provided - attempting to use existing file...\n" ) if parser == "pegen": try: from peg_extension import parse # type: ignore except: print( "An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.", file=sys.stderr, ) return 1 # For a given directory, traverse files and attempt to parse each one # - Output success/failure for each file errors = 0 files = [] trees = {} # Trees to compare (after everything else is done) t0 = time.time() for file in sorted(glob(f"{directory}/**/*.py", recursive=True)): # Only attempt to parse Python files and files that are not excluded should_exclude_file = False for pattern in excluded_files: if PurePath(file).match(pattern): should_exclude_file = True break if not should_exclude_file: try: if tree_arg: mode = 1 if parser == "cpython": with open(file, "r") as f: source = f.read() if mode == 2: compile(source, file, "exec") elif mode == 1: ast.parse(source, file, "exec") else: tree = parse.parse_file(file, mode=mode) if tree_arg: trees[file] = tree if not short: report_status(succeeded=True, file=file, verbose=verbose) except Exception as error: try: ast.parse(file) except Exception: if not short: print( f"File {file} cannot be parsed by either pegen or the ast module." ) else: report_status(succeeded=False, file=file, verbose=verbose, error=error, short=short) errors += 1 files.append(file) t1 = time.time() total_seconds = t1 - t0 total_files = len(files) total_bytes = 0 total_lines = 0 for file in files: # Count lines and bytes separately with open(file, "rb") as f: total_lines += sum(1 for _ in f) total_bytes += f.tell() print( f"Checked {total_files:,} files, {total_lines:,} lines,", f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.", ) if total_seconds > 0: print( f"That's {total_lines / total_seconds :,.0f} lines/sec,", f"or {total_bytes / total_seconds :,.0f} bytes/sec.", ) if parser == "pegen": # Dump memo stats to @data. with open("@data", "w") as datafile: for i, count in enumerate(parse.get_memo_stats()): if count: datafile.write(f"{i:4d} {count:9d}\n") if short: print_memstats() if errors: print(f"Encountered {errors} failures.", file=sys.stderr) # Compare trees (the dict is empty unless -t is given) compare_trees_errors = 0 for file, tree in trees.items(): if not short: print("Comparing ASTs for", file) if compare_trees(tree, file, verbose, tree_arg >= 2) == 1: compare_trees_errors += 1 if errors or compare_trees_errors: return 1 return 0
def main() -> None: args = argparser.parse_args() directory = args.directory grammar_file = args.grammar_file verbose = args.verbose excluded_files = args.exclude if not directory: print("You must specify a directory of files to test.", file=sys.stderr) sys.exit(1) if grammar_file: if not os.path.exists(grammar_file): print(f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr) sys.exit(1) try: build_parser_and_generator(grammar_file, "pegen/parse.c", True) except Exception as err: print( f"{FAIL}The following error occurred when generating the parser. Please check your grammar file.\n{ENDC}", file=sys.stderr, ) traceback.print_exception(err.__class__, err, None) sys.exit(1) else: print("A grammar file was not provided - attempting to use existing file...\n") try: from pegen import parse # type: ignore [attr-defined] except: print( "An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.", file=sys.stderr, ) sys.exit(1) # For a given directory, traverse files and attempt to parse each one # - Output success/failure for each file errors = 0 files = [] t0 = time.time() for file in sorted(glob(f"{directory}/**/*.py", recursive=True)): # Only attempt to parse Python files and files that are not excluded should_exclude_file = False for pattern in excluded_files: if PurePath(file).match(pattern): should_exclude_file = True break if not should_exclude_file: try: parse.parse_file(file) if not args.short: report_status(succeeded=True, file=file, verbose=verbose) except Exception as error: report_status( succeeded=False, file=file, verbose=verbose, error=error, short=args.short ) errors += 1 files.append(file) t1 = time.time() total_seconds = t1 - t0 total_files = len(files) total_bytes = 0 total_lines = 0 for file in files: # Count lines and bytes separately with open(file, "rb") as f: total_lines += sum(1 for _ in f) total_bytes += f.tell() print( f"Checked {total_files:,} files, {total_lines:,} lines,", f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.", ) if total_seconds > 0: print( f"That's {total_lines / total_seconds :,.0f} lines/sec,", f"or {total_bytes / total_seconds :,.0f} bytes/sec.", ) if args.short: print_memstats() if errors: print(f"Encountered {errors} failures.", file=sys.stderr) sys.exit(1)
def parse_directory( directory: str, verbose: bool, excluded_files: List[str], tree_arg: int, short: bool, mode: int, oldparser: bool, ) -> int: if tree_arg: assert mode == PARSE, "Mode should be 1 (parse), when comparing the generated trees" if oldparser and tree_arg: print("Cannot specify tree argument with the cpython parser.", file=sys.stderr) return 1 # For a given directory, traverse files and attempt to parse each one # - Output success/failure for each file errors = 0 files = [] trees = {} # Trees to compare (after everything else is done) total_seconds = 0 for file in sorted( glob(os.path.join(escape(directory), f"**/*.py"), recursive=True)): # Only attempt to parse Python files and files that are not excluded if any(PurePath(file).match(pattern) for pattern in excluded_files): continue with tokenize.open(file) as f: source = f.read() try: result, dt = parse_file(source, file, mode, oldparser) total_seconds += dt if tree_arg: trees[file] = result report_status(succeeded=True, file=file, verbose=verbose, short=short) except SyntaxError as error: if is_parsing_failure(source): print(f"File {file} cannot be parsed by either parser.") else: report_status(succeeded=False, file=file, verbose=verbose, error=error, short=short) errors += 1 files.append(file) t1 = time.time() generate_time_stats(files, total_seconds) if short: print_memstats() if errors: print(f"Encountered {errors} failures.", file=sys.stderr) # Compare trees (the dict is empty unless -t is given) compare_trees_errors = 0 for file, tree in trees.items(): if not short: print("Comparing ASTs for", file) if compare_trees(tree, file, verbose, tree_arg >= 2) == 1: compare_trees_errors += 1 if errors or compare_trees_errors: return 1 return 0
def main() -> None: args = argparser.parse_args() verbose = args.verbose verbose_tokenizer = verbose >= 3 verbose_parser = verbose == 2 or verbose >= 4 t0 = time.time() output_file = args.output if not output_file: if args.cpython: output_file = "parse.c" else: output_file = "parse.py" try: grammar, parser, tokenizer, gen = build_parser_and_generator( args.filename, output_file, args.compile_extension, verbose_tokenizer, verbose_parser, args.verbose, keep_asserts_in_extension=False if args.optimized else True, skip_actions=args.skip_actions, ) except Exception as err: if args.verbose: raise # Show traceback traceback.print_exception(err.__class__, err, None) sys.stderr.write("For full traceback, use -v\n") sys.exit(1) if not args.quiet: if args.verbose: print("Raw Grammar:") for line in repr(grammar).splitlines(): print(" ", line) print("Clean Grammar:") for line in str(grammar).splitlines(): print(" ", line) if args.verbose: print("First Graph:") for src, dsts in gen.first_graph.items(): print(f" {src} -> {', '.join(dsts)}") print("First SCCS:") for scc in gen.first_sccs: print(" ", scc, end="") if len(scc) > 1: print( " # Indirectly left-recursive; leaders:", {name for name in scc if grammar.rules[name].leader}, ) else: name = next(iter(scc)) if name in gen.first_graph[name]: print(" # Left-recursive") else: print() t1 = time.time() if args.verbose: dt = t1 - t0 diag = tokenizer.diagnose() nlines = diag.end[0] if diag.type == token.ENDMARKER: nlines -= 1 print(f"Total time: {dt:.3f} sec; {nlines} lines", end="") if dt: print(f"; {nlines / dt:.0f} lines/sec") else: print() print("Caches sizes:") print(f" token array : {len(tokenizer._tokens):10}") print(f" cache : {len(parser._cache):10}") if not print_memstats(): print("(Can't find psutil; install it for memory stats.)")
def parse_directory( directory: str, grammar_file: str, tokens_file: str, verbose: bool, excluded_files: List[str], skip_actions: bool, tree_arg: int, short: bool, mode: int, parser: str, ) -> int: if parser == "cpython" and (tree_arg or mode == 0): print("Cannot specify tree argument or mode=0 with the cpython parser.", file=sys.stderr) return 1 if not directory: print("You must specify a directory of files to test.", file=sys.stderr) return 1 if grammar_file and tokens_file: if not os.path.exists(grammar_file): print(f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr) return 1 else: print( "A grammar file or a tokens file was not provided - attempting to use existing parser from stdlib...\n" ) if tree_arg: assert mode == 1, "Mode should be 1 (parse), when comparing the generated trees" # For a given directory, traverse files and attempt to parse each one # - Output success/failure for each file errors = 0 files = [] trees = {} # Trees to compare (after everything else is done) total_seconds = 0 for file in sorted(glob(f"{directory}/**/*.py", recursive=True)): # Only attempt to parse Python files and files that are not excluded should_exclude_file = False for pattern in excluded_files: if PurePath(file).match(pattern): should_exclude_file = True break if not should_exclude_file: with tokenize.open(file) as f: source = f.read() try: t0 = time.time() if mode == 2: result = _peg_parser.compile_string( source, filename=file, oldparser=parser == "cpython", ) else: result = _peg_parser.parse_string( source, filename=file, oldparser=parser == "cpython" ) t1 = time.time() total_seconds += (t1 - t0) if tree_arg: trees[file] = result if not short: report_status(succeeded=True, file=file, verbose=verbose) except Exception as error: try: _peg_parser.parse_string(source, mode="exec", oldparser=True) except Exception: if not short: print(f"File {file} cannot be parsed by either pegen or the ast module.") else: report_status( succeeded=False, file=file, verbose=verbose, error=error, short=short ) errors += 1 files.append(file) t1 = time.time() total_files = len(files) total_bytes = 0 total_lines = 0 for file in files: # Count lines and bytes separately with open(file, "rb") as f: total_lines += sum(1 for _ in f) total_bytes += f.tell() print( f"Checked {total_files:,} files, {total_lines:,} lines,", f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.", ) if total_seconds > 0: print( f"That's {total_lines / total_seconds :,.0f} lines/sec,", f"or {total_bytes / total_seconds :,.0f} bytes/sec.", ) if short: print_memstats() if errors: print(f"Encountered {errors} failures.", file=sys.stderr) # Compare trees (the dict is empty unless -t is given) compare_trees_errors = 0 for file, tree in trees.items(): if not short: print("Comparing ASTs for", file) if compare_trees(tree, file, verbose, tree_arg >= 2) == 1: compare_trees_errors += 1 if errors or compare_trees_errors: return 1 return 0
except SyntaxError as error: if is_parsing_failure(source): print(f"File {file} cannot be parsed by either parser.") else: report_status( succeeded=False, file=file, verbose=verbose, error=error, short=short ) errors += 1 files.append(file) t1 = time.time() >>>>>>> 3.9 generate_time_stats(files, total_seconds) if short: print_memstats() if errors: print(f"Encountered {errors} failures.", file=sys.stderr) return 1 return 0 def main() -> None: args = argparser.parse_args() directory = args.directory verbose = args.verbose excluded_files = args.exclude <<<<<<< HEAD short = args.short