def main() -> None:
print(
f"Testing {GRAMMAR_FILE} starting at nesting depth of {INITIAL_NESTING_DEPTH}..."
)
with TemporaryDirectory() as tmp_dir:
nesting_depth = INITIAL_NESTING_DEPTH
rules, parser, tokenizer = build_parser(GRAMMAR_FILE)
python_parser = generate_parser(rules)
c_parser = generate_parser_c_extension(rules, Path(tmp_dir))
c_succeeded = True
python_succeeded = True
while c_succeeded or python_succeeded:
expr = f"{'(' * nesting_depth}0{')' * nesting_depth}"
if c_succeeded:
c_succeeded = check_nested_expr(nesting_depth, c_parser, "C")
if python_succeeded:
python_succeeded = check_nested_expr(nesting_depth,
python_parser, "Python")
nesting_depth += NESTED_INCR_AMT
sys.exit(1)
def main() -> None:
args = argparser.parse_args()
try:
grammar, parser, tokenizer = build_parser(args.grammar_file)
except Exception as err:
print("ERROR: Failed to parse grammar file", file=sys.stderr)
sys.exit(1)
references = {}
for name, rule in grammar.rules.items():
references[name] = set(references_for_item(rule))
# Flatten the start node if has only a single reference
root_node = {
"exec": "file",
"eval": "eval",
"single": "interactive"
}[args.start]
print("digraph g1 {")
print('\toverlap="scale";'
) # Force twopi to scale the graph to avoid overlaps
print(f'\troot="{root_node}";')
print(f"\t{root_node} [color=green, shape=circle];")
for name, refs in references.items():
for ref in refs:
print(f"\t{name} -> {ref};")
print("}")
def python_parse_str():
grammar_path = Path(__file__).parent.parent.parent / "data/python.gram"
grammar = build_parser(grammar_path)[0]
source_path = str(Path(__file__).parent / "parser_cache" / "py_parser.py")
parser_cls = generate_parser(grammar, source_path, "parse_string")
return parser_cls
def main() -> None:
args = argparser.parse_args()
try:
grammar, parser, tokenizer = build_parser(args.grammar_file)
except Exception as err:
print("ERROR: Failed to parse grammar file", file=sys.stderr)
sys.exit(1)
firs_sets = FirstSetCalculator(grammar.rules).calculate()
pprint.pprint(firs_sets)
def main() -> None:
args = argparser.parse_args()
try:
rules, parser, tokenizer = build_parser(args.filename)
except Exception as err:
print("ERROR: Failed to parse grammar file", file=sys.stderr)
sys.exit(1)
visitor = ASTGrammarPrinter()
visitor.print_grammar_ast(rules)
def main() -> None:
args = argparser.parse_args()
try:
grammar, parser, tokenizer = build_parser(args.grammar_file)
except Exception as err:
print("ERROR: Failed to parse grammar file", file=sys.stderr)
sys.exit(1)
references = {}
for name, rule in grammar.rules.items():
references[name] = set(references_for_item(rule))
# Flatten the start node if has only a single reference
root_node = "start"
if start := references.get("start"):
if len(start) == 1:
root_node = list(start)[0]
del references["start"]
def parse_directory(
directory: str,
grammar_file: str,
verbose: bool,
excluded_files: List[str],
skip_actions: bool,
tree_arg: int,
short: bool,
parser: Any,
) -> int:
if not directory:
print("You must specify a directory of files to test.", file=sys.stderr)
return 1
if grammar_file:
if not os.path.exists(grammar_file):
print(f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr)
return 1
try:
if not parser:
grammar = build_parser(grammar_file)[0]
GeneratedParser = generate_parser(grammar) # TODO: skip_actions
except Exception as err:
print(
f"{FAIL}The following error occurred when generating the parser."
f" Please check your grammar file.\n{ENDC}",
file=sys.stderr,
)
traceback.print_exception(err.__class__, err, None)
return 1
else:
print("A grammar file was not provided - attempting to use existing file...\n")
try:
sys.path.insert(0, sys.path.insert(0, os.path.join(os.getcwd(), "data")))
from python_parser import GeneratedParser
except:
print(
"An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.",
file=sys.stderr,
)
return 1
try:
import tokenize
from pegen.tokenizer import Tokenizer
def parse(filepath):
with open(filepath) as f:
tokengen = tokenize.generate_tokens(f.readline)
tokenizer = Tokenizer(tokengen, verbose=False)
parser = GeneratedParser(tokenizer, verbose=verbose)
return parser.start()
except:
print(
"An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.",
file=sys.stderr,
)
return 1
# For a given directory, traverse files and attempt to parse each one
# - Output success/failure for each file
errors = 0
files = []
trees = {} # Trees to compare (after everything else is done)
t0 = time.time()
for file in sorted(glob(f"{directory}/**/*.py", recursive=True)):
# Only attempt to parse Python files and files that are not excluded
should_exclude_file = False
for pattern in excluded_files:
if PurePath(file).match(pattern):
should_exclude_file = True
break
if not should_exclude_file:
try:
if tree_arg:
tree = parse(file)
trees[file] = tree
if not short:
report_status(succeeded=True, file=file, verbose=verbose)
except Exception as error:
try:
with open(file) as f:
ast.parse(f.read())
except Exception:
if not short:
print(f"File {file} cannot be parsed by either pegen or the ast module.")
else:
report_status(
succeeded=False, file=file, verbose=verbose, error=error, short=short
)
errors += 1
files.append(file)
t1 = time.time()
total_seconds = t1 - t0
total_files = len(files)
total_bytes = 0
total_lines = 0
for file in files:
# Count lines and bytes separately
with open(file, "rb") as f:
total_lines += sum(1 for _ in f)
total_bytes += f.tell()
print(
f"Checked {total_files:,} files, {total_lines:,} lines,",
f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.",
)
if total_seconds > 0:
print(
f"That's {total_lines / total_seconds :,.0f} lines/sec,",
f"or {total_bytes / total_seconds :,.0f} bytes/sec.",
)
if short:
print_memstats()
if errors:
print(f"Encountered {errors} failures.", file=sys.stderr)
# Compare trees (the dict is empty unless -t is given)
compare_trees_errors = 0
for file, tree in trees.items():
if not short:
print("Comparing ASTs for", file)
if compare_trees(tree, file, verbose, tree_arg >= 2) == 1:
compare_trees_errors += 1
if errors or compare_trees_errors:
return 1
return 0
