def main() -> None: print( f"Testing {GRAMMAR_FILE} starting at nesting depth of {INITIAL_NESTING_DEPTH}..." ) with TemporaryDirectory() as tmp_dir: nesting_depth = INITIAL_NESTING_DEPTH rules, parser, tokenizer = build_parser(GRAMMAR_FILE) python_parser = generate_parser(rules) c_parser = generate_parser_c_extension(rules, Path(tmp_dir)) c_succeeded = True python_succeeded = True while c_succeeded or python_succeeded: expr = f"{'(' * nesting_depth}0{')' * nesting_depth}" if c_succeeded: c_succeeded = check_nested_expr(nesting_depth, c_parser, "C") if python_succeeded: python_succeeded = check_nested_expr(nesting_depth, python_parser, "Python") nesting_depth += NESTED_INCR_AMT sys.exit(1)
def main() -> None: args = argparser.parse_args() try: grammar, parser, tokenizer = build_parser(args.grammar_file) except Exception as err: print("ERROR: Failed to parse grammar file", file=sys.stderr) sys.exit(1) references = {} for name, rule in grammar.rules.items(): references[name] = set(references_for_item(rule)) # Flatten the start node if has only a single reference root_node = { "exec": "file", "eval": "eval", "single": "interactive" }[args.start] print("digraph g1 {") print('\toverlap="scale";' ) # Force twopi to scale the graph to avoid overlaps print(f'\troot="{root_node}";') print(f"\t{root_node} [color=green, shape=circle];") for name, refs in references.items(): for ref in refs: print(f"\t{name} -> {ref};") print("}")
def python_parse_str(): grammar_path = Path(__file__).parent.parent.parent / "data/python.gram" grammar = build_parser(grammar_path)[0] source_path = str(Path(__file__).parent / "parser_cache" / "py_parser.py") parser_cls = generate_parser(grammar, source_path, "parse_string") return parser_cls
def main() -> None: args = argparser.parse_args() try: grammar, parser, tokenizer = build_parser(args.grammar_file) except Exception as err: print("ERROR: Failed to parse grammar file", file=sys.stderr) sys.exit(1) firs_sets = FirstSetCalculator(grammar.rules).calculate() pprint.pprint(firs_sets)
def main() -> None: args = argparser.parse_args() try: rules, parser, tokenizer = build_parser(args.filename) except Exception as err: print("ERROR: Failed to parse grammar file", file=sys.stderr) sys.exit(1) visitor = ASTGrammarPrinter() visitor.print_grammar_ast(rules)
def main() -> None: args = argparser.parse_args() try: grammar, parser, tokenizer = build_parser(args.grammar_file) except Exception as err: print("ERROR: Failed to parse grammar file", file=sys.stderr) sys.exit(1) references = {} for name, rule in grammar.rules.items(): references[name] = set(references_for_item(rule)) # Flatten the start node if has only a single reference root_node = "start" if start := references.get("start"): if len(start) == 1: root_node = list(start)[0] del references["start"]
def parse_directory( directory: str, grammar_file: str, verbose: bool, excluded_files: List[str], skip_actions: bool, tree_arg: int, short: bool, parser: Any, ) -> int: if not directory: print("You must specify a directory of files to test.", file=sys.stderr) return 1 if grammar_file: if not os.path.exists(grammar_file): print(f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr) return 1 try: if not parser: grammar = build_parser(grammar_file)[0] GeneratedParser = generate_parser(grammar) # TODO: skip_actions except Exception as err: print( f"{FAIL}The following error occurred when generating the parser." f" Please check your grammar file.\n{ENDC}", file=sys.stderr, ) traceback.print_exception(err.__class__, err, None) return 1 else: print("A grammar file was not provided - attempting to use existing file...\n") try: sys.path.insert(0, sys.path.insert(0, os.path.join(os.getcwd(), "data"))) from python_parser import GeneratedParser except: print( "An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.", file=sys.stderr, ) return 1 try: import tokenize from pegen.tokenizer import Tokenizer def parse(filepath): with open(filepath) as f: tokengen = tokenize.generate_tokens(f.readline) tokenizer = Tokenizer(tokengen, verbose=False) parser = GeneratedParser(tokenizer, verbose=verbose) return parser.start() except: print( "An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.", file=sys.stderr, ) return 1 # For a given directory, traverse files and attempt to parse each one # - Output success/failure for each file errors = 0 files = [] trees = {} # Trees to compare (after everything else is done) t0 = time.time() for file in sorted(glob(f"{directory}/**/*.py", recursive=True)): # Only attempt to parse Python files and files that are not excluded should_exclude_file = False for pattern in excluded_files: if PurePath(file).match(pattern): should_exclude_file = True break if not should_exclude_file: try: if tree_arg: tree = parse(file) trees[file] = tree if not short: report_status(succeeded=True, file=file, verbose=verbose) except Exception as error: try: with open(file) as f: ast.parse(f.read()) except Exception: if not short: print(f"File {file} cannot be parsed by either pegen or the ast module.") else: report_status( succeeded=False, file=file, verbose=verbose, error=error, short=short ) errors += 1 files.append(file) t1 = time.time() total_seconds = t1 - t0 total_files = len(files) total_bytes = 0 total_lines = 0 for file in files: # Count lines and bytes separately with open(file, "rb") as f: total_lines += sum(1 for _ in f) total_bytes += f.tell() print( f"Checked {total_files:,} files, {total_lines:,} lines,", f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.", ) if total_seconds > 0: print( f"That's {total_lines / total_seconds :,.0f} lines/sec,", f"or {total_bytes / total_seconds :,.0f} bytes/sec.", ) if short: print_memstats() if errors: print(f"Encountered {errors} failures.", file=sys.stderr) # Compare trees (the dict is empty unless -t is given) compare_trees_errors = 0 for file, tree in trees.items(): if not short: print("Comparing ASTs for", file) if compare_trees(tree, file, verbose, tree_arg >= 2) == 1: compare_trees_errors += 1 if errors or compare_trees_errors: return 1 return 0