def __init__(self, grammar_file=_PATTERN_GRAMMAR_FILE):
"""Initializer.
Takes an optional alternative filename for the pattern grammar.
"""
self.grammar = driver.load_grammar(grammar_file)
self.syms = pygram.Symbols(self.grammar)
self.pygrammar = pygram.python_grammar
self.pysyms = pygram.python_symbols
self.driver = driver.Driver(self.grammar, convert=pattern_convert)
def main():
gr = driver.load_grammar("Grammar.txt")
dr = driver.Driver(gr, convert=pytree.convert)
fn = "example.py"
tree = dr.parse_file(fn, debug=True)
if not diff(fn, tree):
print("No diffs.")
if not sys.argv[1:]:
return # Pass a dummy argument to run the complete test suite below
problems = []
# Process every imported module
for name in sys.modules:
mod = sys.modules[name]
if mod is None or not hasattr(mod, "__file__"):
continue
fn = mod.__file__
if fn.endswith(".pyc"):
fn = fn[:-1]
if not fn.endswith(".py"):
continue
print("Parsing", fn, file=sys.stderr)
tree = dr.parse_file(fn, debug=True)
if diff(fn, tree):
problems.append(fn)
# Process every single module on sys.path (but not in packages)
for dir in sys.path:
try:
names = os.listdir(dir)
except os.error:
continue
print("Scanning", dir, "...", file=sys.stderr)
for name in names:
if not name.endswith(".py"):
continue
print("Parsing", name, file=sys.stderr)
fn = os.path.join(dir, name)
try:
tree = dr.parse_file(fn, debug=True)
except pgen2.parse.ParseError as err:
print("ParseError:", err)
else:
if diff(fn, tree):
problems.append(fn)
# Show summary of problem files
if not problems:
print("No problems. Congratulations!")
else:
print("Problems in following files:")
for fn in problems:
print("***", fn)
def main():
gr = driver.load_grammar("Grammar.txt")
dr = driver.Driver(gr, convert=pytree.convert)
fn = "example.py"
tree = dr.parse_file(fn, debug=True)
if not diff(fn, tree):
print("No diffs.")
if not sys.argv[1:]:
return # Pass a dummy argument to run the complete test suite below
problems = []
# Process every imported module
for name in sys.modules:
mod = sys.modules[name]
if mod is None or not hasattr(mod, "__file__"):
continue
fn = mod.__file__
if fn.endswith(".pyc"):
fn = fn[:-1]
if not fn.endswith(".py"):
continue
print("Parsing", fn, file=sys.stderr)
tree = dr.parse_file(fn, debug=True)
if diff(fn, tree):
problems.append(fn)
# Process every single module on sys.path (but not in packages)
for dir in sys.path:
try:
names = os.listdir(dir)
except OSError:
continue
print("Scanning", dir, "...", file=sys.stderr)
for name in names:
if not name.endswith(".py"):
continue
print("Parsing", name, file=sys.stderr)
fn = os.path.join(dir, name)
try:
tree = dr.parse_file(fn, debug=True)
except pgen2.parse.ParseError as err:
print("ParseError:", err)
else:
if diff(fn, tree):
problems.append(fn)
# Show summary of problem files
if not problems:
print("No problems. Congratulations!")
else:
print("Problems in following files:")
for fn in problems:
print("***", fn)
def main(argv):
grammar_path = argv[1]
# NOTE: This is cached as a pickle
grammar = driver.load_grammar(grammar_path)
FILE_INPUT = grammar.symbol2number['file_input']
symbols = Symbols(grammar)
pytree.Init(symbols) # for type_repr() pretty printing
transformer.Init(symbols) # for _names and other dicts
# In Python 2 code, always use from __future__ import print_function.
try:
del grammar.keywords["print"]
except KeyError:
pass
#do_glue = False
do_glue = True
if do_glue: # Make it a flag
# Emulating parser.st structures from parsermodule.c.
# They have a totuple() method, which outputs tuples like this.
def py2st(grammar, raw_node):
type, value, context, children = raw_node
# See pytree.Leaf
if context:
_, (lineno, column) = context
else:
lineno = 0 # default in Leaf
column = 0
if children:
return (type,) + tuple(children)
else:
return (type, value, lineno, column)
convert = py2st
else:
convert = pytree.convert
d = driver.Driver(grammar, convert=convert)
action = argv[2]
if action == 'stdlib-parse':
# This is what the compiler/ package was written against.
import parser
py_path = argv[3]
with open(py_path) as f:
st = parser.suite(f.read())
tree = st.totuple()
n = transformer.CountTupleTree(tree)
log('COUNT %d', n)
printer = transformer.TupleTreePrinter(HostStdlibNames())
printer.Print(tree)
elif action == 'parse':
py_path = argv[3]
with open(py_path) as f:
tokens = tokenize.generate_tokens(f.readline)
tree = d.parse_tokens(tokens, start_symbol=FILE_INPUT)
if isinstance(tree, tuple):
n = transformer.CountTupleTree(tree)
log('COUNT %d', n)
printer = transformer.TupleTreePrinter(transformer._names)
printer.Print(tree)
else:
tree.PrettyPrint(sys.stdout)
log('\tChildren: %d' % len(tree.children), file=sys.stderr)
elif action == 'compile':
py_path = argv[3]
out_path = argv[4]
if do_glue:
py_parser = Pgen2PythonParser(d, FILE_INPUT)
printer = transformer.TupleTreePrinter(transformer._names)
tr = transformer.Pgen2Transformer(py_parser, printer)
else:
tr = transformer.Transformer()
# for Python 2.7 compatibility:
if _READ_SOURCE_AS_UNICODE:
f = codecs.open(py_path, encoding='utf-8')
else:
f = open(py_path)
contents = f.read()
co = pycodegen.compile(contents, py_path, 'exec', transformer=tr)
file_size = os.path.getsize(py_path)
log("Code length: %d", len(co.co_code))
# Write the .pyc file
with open(out_path, 'wb') as out_f:
h = pycodegen.getPycHeader(py_path)
out_f.write(h)
marshal.dump(co, out_f)
elif action == 'compile2':
in_path = argv[3]
out_path = argv[4]
from compiler2 import pycodegen as pycodegen2
from misc import stdlib_compile
stdlib_compile.compileAndWrite(in_path, out_path, pycodegen2.compile)
else:
raise RuntimeError('Invalid action %r' % action)
# Licensed to PSF under a Contributor Agreement.
"""Export the Python grammar and symbols."""
# Python imports
import os
# Local imports
from pgen2 import token
from pgen2 import driver
import pytree
# The grammar file
_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "Grammar.txt")
class Symbols(object):
def __init__(self, grammar):
"""Initializer.
Creates an attribute for each grammar symbol (nonterminal),
whose value is the symbol's type (an int >= 256).
"""
for name, symbol in grammar.symbol2number.iteritems():
setattr(self, name, symbol)
python_grammar = driver.load_grammar(_GRAMMAR_FILE)
python_symbols = Symbols(python_grammar)
# Copyright 2006 Google, Inc. All Rights Reserved.
# Licensed to PSF under a Contributor Agreement.
"""Export the Python grammar and symbols."""
# Python imports
import os
# Local imports
from pgen2 import token
from pgen2 import driver
import pytree
# The grammar file
_GRAMMAR_FILE = os.path.join(os.path.dirname(__file__), "Grammar.txt")
class Symbols(object):
def __init__(self, grammar):
"""Initializer.
Creates an attribute for each grammar symbol (nonterminal),
whose value is the symbol's type (an int >= 256).
"""
for name, symbol in grammar.symbol2number.iteritems():
setattr(self, name, symbol)
python_grammar = driver.load_grammar(_GRAMMAR_FILE)
python_symbols = Symbols(python_grammar)
