def main() -> None:
tok: Optional[tokenize.TokenInfo]
t0 = time.time()
ntoks = 0
nlines = 0
line_ends = (token.NL, token.NEWLINE)
for filename in sys.argv[1:]:
print(f"{nlines:10} lines", end="\r", file=sys.stderr)
try:
with open(filename) as file:
toks = tokenize.generate_tokens(file.readline)
for tok in toks:
ntoks += 1
if tok.type in line_ends:
nlines += 1
except Exception as err:
print("Error:", err, file=sys.stderr)
tok = None
t1 = time.time()
dt = t1 - t0
print(f"{ntoks} tokens, {nlines} lines in {dt:.3f} secs", file=sys.stderr)
if dt:
print(f"{ntoks/dt:.0f} tokens/sec, {nlines/dt:.0f} lines/sec",
file=sys.stderr)
print_memstats()
def main() -> None:
filename = sys.argv[1]
t0 = time.time()
drv = driver.Driver(pygram.python_grammar, convert=pytree.convert)
tree = drv.parse_file(filename)
t1 = time.time()
dt = t1 - t0
with open(filename) as file:
nlines = len(file.readlines())
print("%.3f seconds for %d lines; %.0f lines/sec" % (dt, nlines, nlines /
(dt or 1e-9)))
print_memstats()
def main() -> None:
t0 = time.time()
for filename in sys.argv[1:]:
print(filename, end="\r")
try:
with open(filename) as file:
source = file.read()
tree = ast.parse(source, filename)
except Exception as err:
print(f"{filename}: {err.__class__.__name__}: {err}", file=sys.stderr)
tok = None
t1 = time.time()
dt = t1 - t0
print(f"Parsed in {dt:.3f} secs", file=sys.stderr)
print_memstats()
def main() -> None:
from pegen.testutil import print_memstats
args = argparser.parse_args()
if "func" not in args:
argparser.error(
"Must specify the target language mode ('c' or 'python')")
t0 = time.time()
grammar, parser, tokenizer, gen = args.func(args)
t1 = time.time()
validate_grammar(grammar)
if not args.quiet:
if args.verbose:
print("Raw Grammar:")
for line in repr(grammar).splitlines():
print(" ", line)
print("Clean Grammar:")
for line in str(grammar).splitlines():
print(" ", line)
if args.verbose:
print("First Graph:")
for src, dsts in gen.first_graph.items():
print(f" {src} -> {', '.join(dsts)}")
print("First SCCS:")
for scc in gen.first_sccs:
print(" ", scc, end="")
if len(scc) > 1:
print(
" # Indirectly left-recursive; leaders:",
{name
for name in scc if grammar.rules[name].leader},
)
else:
name = next(iter(scc))
if name in gen.first_graph[name]:
print(" # Left-recursive")
else:
print()
if args.verbose:
dt = t1 - t0
diag = tokenizer.diagnose()
nlines = diag.end[0]
if diag.type == token.ENDMARKER:
nlines -= 1
print(f"Total time: {dt:.3f} sec; {nlines} lines", end="")
if dt:
print(f"; {nlines / dt:.0f} lines/sec")
else:
print()
print("Caches sizes:")
print(f" token array : {len(tokenizer._tokens):10}")
print(f" cache : {len(parser._cache):10}")
if not print_memstats():
print("(Can't find psutil; install it for memory stats.)")
def parse_directory(directory: str, verbose: bool, excluded_files: List[str],
short: bool) -> int:
# For a given directory, traverse files and attempt to parse each one
# - Output success/failure for each file
errors = 0
files = []
total_seconds = 0
for file in sorted(
glob(os.path.join(escape(directory), f"**/*.py"), recursive=True)):
# Only attempt to parse Python files and files that are not excluded
if any(PurePath(file).match(pattern) for pattern in excluded_files):
continue
with tokenize.open(file) as f:
source = f.read()
try:
result, dt = parse_file(source, file)
total_seconds += dt
report_status(succeeded=True,
file=file,
verbose=verbose,
short=short)
except SyntaxError as error:
report_status(succeeded=False,
file=file,
verbose=verbose,
error=error,
short=short)
errors += 1
files.append(file)
generate_time_stats(files, total_seconds)
if short:
print_memstats()
if errors:
print(f"Encountered {errors} failures.", file=sys.stderr)
return 1
return 0
def parse_directory(
directory: str,
grammar_file: str,
verbose: bool,
excluded_files: List[str],
skip_actions: bool,
tree_arg: int,
short: bool,
extension: Any,
mode: int,
parser: str,
) -> int:
if parser == "cpython" and (tree_arg or mode == 0):
print(
"Cannot specify tree argument or mode=0 with the cpython parser.",
file=sys.stderr)
return 1
if not directory:
print("You must specify a directory of files to test.",
file=sys.stderr)
return 1
if grammar_file:
if not os.path.exists(grammar_file):
print(
f"The specified grammar file, {grammar_file}, does not exist.",
file=sys.stderr)
return 1
try:
if not extension and parser == "pegen":
build_parser_and_generator(
grammar_file,
"peg_extension/parse.c",
compile_extension=True,
skip_actions=skip_actions,
)
except Exception as err:
print(
f"{FAIL}The following error occurred when generating the parser. Please check your grammar file.\n{ENDC}",
file=sys.stderr,
)
traceback.print_exception(err.__class__, err, None)
return 1
else:
print(
"A grammar file was not provided - attempting to use existing file...\n"
)
if parser == "pegen":
try:
from peg_extension import parse # type: ignore
except:
print(
"An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.",
file=sys.stderr,
)
return 1
# For a given directory, traverse files and attempt to parse each one
# - Output success/failure for each file
errors = 0
files = []
trees = {} # Trees to compare (after everything else is done)
t0 = time.time()
for file in sorted(glob(f"{directory}/**/*.py", recursive=True)):
# Only attempt to parse Python files and files that are not excluded
should_exclude_file = False
for pattern in excluded_files:
if PurePath(file).match(pattern):
should_exclude_file = True
break
if not should_exclude_file:
try:
if tree_arg:
mode = 1
if parser == "cpython":
with open(file, "r") as f:
source = f.read()
if mode == 2:
compile(source, file, "exec")
elif mode == 1:
ast.parse(source, file, "exec")
else:
tree = parse.parse_file(file, mode=mode)
if tree_arg:
trees[file] = tree
if not short:
report_status(succeeded=True, file=file, verbose=verbose)
except Exception as error:
try:
ast.parse(file)
except Exception:
if not short:
print(
f"File {file} cannot be parsed by either pegen or the ast module."
)
else:
report_status(succeeded=False,
file=file,
verbose=verbose,
error=error,
short=short)
errors += 1
files.append(file)
t1 = time.time()
total_seconds = t1 - t0
total_files = len(files)
total_bytes = 0
total_lines = 0
for file in files:
# Count lines and bytes separately
with open(file, "rb") as f:
total_lines += sum(1 for _ in f)
total_bytes += f.tell()
print(
f"Checked {total_files:,} files, {total_lines:,} lines,",
f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.",
)
if total_seconds > 0:
print(
f"That's {total_lines / total_seconds :,.0f} lines/sec,",
f"or {total_bytes / total_seconds :,.0f} bytes/sec.",
)
if parser == "pegen":
# Dump memo stats to @data.
with open("@data", "w") as datafile:
for i, count in enumerate(parse.get_memo_stats()):
if count:
datafile.write(f"{i:4d} {count:9d}\n")
if short:
print_memstats()
if errors:
print(f"Encountered {errors} failures.", file=sys.stderr)
# Compare trees (the dict is empty unless -t is given)
compare_trees_errors = 0
for file, tree in trees.items():
if not short:
print("Comparing ASTs for", file)
if compare_trees(tree, file, verbose, tree_arg >= 2) == 1:
compare_trees_errors += 1
if errors or compare_trees_errors:
return 1
return 0
def main() -> None:
args = argparser.parse_args()
directory = args.directory
grammar_file = args.grammar_file
verbose = args.verbose
excluded_files = args.exclude
if not directory:
print("You must specify a directory of files to test.", file=sys.stderr)
sys.exit(1)
if grammar_file:
if not os.path.exists(grammar_file):
print(f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr)
sys.exit(1)
try:
build_parser_and_generator(grammar_file, "pegen/parse.c", True)
except Exception as err:
print(
f"{FAIL}The following error occurred when generating the parser. Please check your grammar file.\n{ENDC}",
file=sys.stderr,
)
traceback.print_exception(err.__class__, err, None)
sys.exit(1)
else:
print("A grammar file was not provided - attempting to use existing file...\n")
try:
from pegen import parse # type: ignore [attr-defined]
except:
print(
"An existing parser was not found. Please run `make` or specify a grammar file with the `-g` flag.",
file=sys.stderr,
)
sys.exit(1)
# For a given directory, traverse files and attempt to parse each one
# - Output success/failure for each file
errors = 0
files = []
t0 = time.time()
for file in sorted(glob(f"{directory}/**/*.py", recursive=True)):
# Only attempt to parse Python files and files that are not excluded
should_exclude_file = False
for pattern in excluded_files:
if PurePath(file).match(pattern):
should_exclude_file = True
break
if not should_exclude_file:
try:
parse.parse_file(file)
if not args.short:
report_status(succeeded=True, file=file, verbose=verbose)
except Exception as error:
report_status(
succeeded=False, file=file, verbose=verbose, error=error, short=args.short
)
errors += 1
files.append(file)
t1 = time.time()
total_seconds = t1 - t0
total_files = len(files)
total_bytes = 0
total_lines = 0
for file in files:
# Count lines and bytes separately
with open(file, "rb") as f:
total_lines += sum(1 for _ in f)
total_bytes += f.tell()
print(
f"Checked {total_files:,} files, {total_lines:,} lines,",
f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.",
)
if total_seconds > 0:
print(
f"That's {total_lines / total_seconds :,.0f} lines/sec,",
f"or {total_bytes / total_seconds :,.0f} bytes/sec.",
)
if args.short:
print_memstats()
if errors:
print(f"Encountered {errors} failures.", file=sys.stderr)
sys.exit(1)
def parse_directory(
directory: str,
verbose: bool,
excluded_files: List[str],
tree_arg: int,
short: bool,
mode: int,
oldparser: bool,
) -> int:
if tree_arg:
assert mode == PARSE, "Mode should be 1 (parse), when comparing the generated trees"
if oldparser and tree_arg:
print("Cannot specify tree argument with the cpython parser.",
file=sys.stderr)
return 1
# For a given directory, traverse files and attempt to parse each one
# - Output success/failure for each file
errors = 0
files = []
trees = {} # Trees to compare (after everything else is done)
total_seconds = 0
for file in sorted(
glob(os.path.join(escape(directory), f"**/*.py"), recursive=True)):
# Only attempt to parse Python files and files that are not excluded
if any(PurePath(file).match(pattern) for pattern in excluded_files):
continue
with tokenize.open(file) as f:
source = f.read()
try:
result, dt = parse_file(source, file, mode, oldparser)
total_seconds += dt
if tree_arg:
trees[file] = result
report_status(succeeded=True,
file=file,
verbose=verbose,
short=short)
except SyntaxError as error:
if is_parsing_failure(source):
print(f"File {file} cannot be parsed by either parser.")
else:
report_status(succeeded=False,
file=file,
verbose=verbose,
error=error,
short=short)
errors += 1
files.append(file)
t1 = time.time()
generate_time_stats(files, total_seconds)
if short:
print_memstats()
if errors:
print(f"Encountered {errors} failures.", file=sys.stderr)
# Compare trees (the dict is empty unless -t is given)
compare_trees_errors = 0
for file, tree in trees.items():
if not short:
print("Comparing ASTs for", file)
if compare_trees(tree, file, verbose, tree_arg >= 2) == 1:
compare_trees_errors += 1
if errors or compare_trees_errors:
return 1
return 0
def main() -> None:
args = argparser.parse_args()
verbose = args.verbose
verbose_tokenizer = verbose >= 3
verbose_parser = verbose == 2 or verbose >= 4
t0 = time.time()
output_file = args.output
if not output_file:
if args.cpython:
output_file = "parse.c"
else:
output_file = "parse.py"
try:
grammar, parser, tokenizer, gen = build_parser_and_generator(
args.filename,
output_file,
args.compile_extension,
verbose_tokenizer,
verbose_parser,
args.verbose,
keep_asserts_in_extension=False if args.optimized else True,
skip_actions=args.skip_actions,
)
except Exception as err:
if args.verbose:
raise # Show traceback
traceback.print_exception(err.__class__, err, None)
sys.stderr.write("For full traceback, use -v\n")
sys.exit(1)
if not args.quiet:
if args.verbose:
print("Raw Grammar:")
for line in repr(grammar).splitlines():
print(" ", line)
print("Clean Grammar:")
for line in str(grammar).splitlines():
print(" ", line)
if args.verbose:
print("First Graph:")
for src, dsts in gen.first_graph.items():
print(f" {src} -> {', '.join(dsts)}")
print("First SCCS:")
for scc in gen.first_sccs:
print(" ", scc, end="")
if len(scc) > 1:
print(
" # Indirectly left-recursive; leaders:",
{name
for name in scc if grammar.rules[name].leader},
)
else:
name = next(iter(scc))
if name in gen.first_graph[name]:
print(" # Left-recursive")
else:
print()
t1 = time.time()
if args.verbose:
dt = t1 - t0
diag = tokenizer.diagnose()
nlines = diag.end[0]
if diag.type == token.ENDMARKER:
nlines -= 1
print(f"Total time: {dt:.3f} sec; {nlines} lines", end="")
if dt:
print(f"; {nlines / dt:.0f} lines/sec")
else:
print()
print("Caches sizes:")
print(f" token array : {len(tokenizer._tokens):10}")
print(f" cache : {len(parser._cache):10}")
if not print_memstats():
print("(Can't find psutil; install it for memory stats.)")
def parse_directory(
directory: str,
grammar_file: str,
tokens_file: str,
verbose: bool,
excluded_files: List[str],
skip_actions: bool,
tree_arg: int,
short: bool,
mode: int,
parser: str,
) -> int:
if parser == "cpython" and (tree_arg or mode == 0):
print("Cannot specify tree argument or mode=0 with the cpython parser.", file=sys.stderr)
return 1
if not directory:
print("You must specify a directory of files to test.", file=sys.stderr)
return 1
if grammar_file and tokens_file:
if not os.path.exists(grammar_file):
print(f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr)
return 1
else:
print(
"A grammar file or a tokens file was not provided - attempting to use existing parser from stdlib...\n"
)
if tree_arg:
assert mode == 1, "Mode should be 1 (parse), when comparing the generated trees"
# For a given directory, traverse files and attempt to parse each one
# - Output success/failure for each file
errors = 0
files = []
trees = {} # Trees to compare (after everything else is done)
total_seconds = 0
for file in sorted(glob(f"{directory}/**/*.py", recursive=True)):
# Only attempt to parse Python files and files that are not excluded
should_exclude_file = False
for pattern in excluded_files:
if PurePath(file).match(pattern):
should_exclude_file = True
break
if not should_exclude_file:
with tokenize.open(file) as f:
source = f.read()
try:
t0 = time.time()
if mode == 2:
result = _peg_parser.compile_string(
source,
filename=file,
oldparser=parser == "cpython",
)
else:
result = _peg_parser.parse_string(
source,
filename=file,
oldparser=parser == "cpython"
)
t1 = time.time()
total_seconds += (t1 - t0)
if tree_arg:
trees[file] = result
if not short:
report_status(succeeded=True, file=file, verbose=verbose)
except Exception as error:
try:
_peg_parser.parse_string(source, mode="exec", oldparser=True)
except Exception:
if not short:
print(f"File {file} cannot be parsed by either pegen or the ast module.")
else:
report_status(
succeeded=False, file=file, verbose=verbose, error=error, short=short
)
errors += 1
files.append(file)
t1 = time.time()
total_files = len(files)
total_bytes = 0
total_lines = 0
for file in files:
# Count lines and bytes separately
with open(file, "rb") as f:
total_lines += sum(1 for _ in f)
total_bytes += f.tell()
print(
f"Checked {total_files:,} files, {total_lines:,} lines,",
f"{total_bytes:,} bytes in {total_seconds:,.3f} seconds.",
)
if total_seconds > 0:
print(
f"That's {total_lines / total_seconds :,.0f} lines/sec,",
f"or {total_bytes / total_seconds :,.0f} bytes/sec.",
)
if short:
print_memstats()
if errors:
print(f"Encountered {errors} failures.", file=sys.stderr)
# Compare trees (the dict is empty unless -t is given)
compare_trees_errors = 0
for file, tree in trees.items():
if not short:
print("Comparing ASTs for", file)
if compare_trees(tree, file, verbose, tree_arg >= 2) == 1:
compare_trees_errors += 1
if errors or compare_trees_errors:
return 1
return 0
except SyntaxError as error:
if is_parsing_failure(source):
print(f"File {file} cannot be parsed by either parser.")
else:
report_status(
succeeded=False, file=file, verbose=verbose, error=error, short=short
)
errors += 1
files.append(file)
t1 = time.time()
>>>>>>> 3.9
generate_time_stats(files, total_seconds)
if short:
print_memstats()
if errors:
print(f"Encountered {errors} failures.", file=sys.stderr)
return 1
return 0
def main() -> None:
args = argparser.parse_args()
directory = args.directory
verbose = args.verbose
excluded_files = args.exclude
<<<<<<< HEAD
short = args.short
