def main(codegen=pythoncg):
args = parse_args()
if args.whitespace:
args.whitespace = eval_escapes(args.whitespace)
outfile = args.outfile
prepare_for_output(outfile)
prepare_for_output(args.object_model_outfile)
grammar = codecs.open(args.filename, encoding='utf-8').read()
try:
model = compile(grammar,
args.name,
trace=args.trace,
filename=args.filename,
colorize=args.color)
model.whitespace = args.whitespace
model.nameguard = args.nameguard
model.left_recursion = args.left_recursion
if args.draw:
from tatsu import diagrams
diagrams.draw(outfile, model)
else:
if args.pretty:
result = model.pretty()
elif args.pretty_lean:
result = model.pretty_lean()
elif args.object_model:
result = objectmodel.codegen(model, base_type=args.base_type)
else:
result = codegen(model)
if outfile:
save(outfile, result)
else:
print(result)
# if requested, always save it
if args.object_model_outfile:
save(args.object_model_outfile,
objectmodel.codegen(model, base_type=args.base_type))
print('-' * 72, file=sys.stderr)
print('{:12,d} lines in grammar'.format(len(grammar.split())),
file=sys.stderr)
print('{:12,d} rules in grammar'.format(len(model.rules)),
file=sys.stderr) # noqa
print('{:12,d} nodes in AST'.format(model.nodecount()),
file=sys.stderr)
except ParseException as e:
print(e, file=sys.stderr)
sys.exit(1)
def test_dot(self):
grammar = '''
start = "foo\\nbar" $;
'''
try:
from tatsu.diagrams import draw
except ImportError:
return
m = compile(grammar, 'Diagram')
draw('tmp/diagram.png', m)
def main(codegen=pythoncg):
args = parse_args()
if args.whitespace:
args.whitespace = eval_escapes(args.whitespace)
outfile = args.outfile
prepare_for_output(outfile)
prepare_for_output(args.object_model_outfile)
grammar = codecs.open(args.filename, 'r', encoding='utf-8').read()
try:
model = compile(
grammar,
args.name,
trace=args.trace,
filename=args.filename,
colorize=args.color
)
model.whitespace = args.whitespace
model.nameguard = args.nameguard
model.left_recursion = args.left_recursion
if args.draw:
from tatsu import diagrams
diagrams.draw(outfile, model)
else:
if args.pretty:
result = model.pretty()
elif args.pretty_lean:
result = model.pretty_lean()
elif args.object_model:
result = objectmodel.codegen(model)
else:
result = codegen(model)
if outfile:
save(outfile, result)
else:
print(result)
# if requested, always save it
if args.object_model_outfile:
save(args.object_model_outfile, objectmodel.codegen(model))
print('-' * 72, file=sys.stderr)
print('{:12,d} lines in grammar'.format(len(grammar.split())), file=sys.stderr)
print('{:12,d} rules in grammar'.format(len(model.rules)), file=sys.stderr) # noqa
print('{:12,d} nodes in AST'.format(model.nodecount()), file=sys.stderr)
except ParseException as e:
print(e, file=sys.stderr)
sys.exit(1)
def test_bootstrap(self):
print()
if os.path.isfile('./tmp/00.ast'):
shutil.rmtree('./tmp')
if not os.path.isdir('./tmp'):
os.mkdir('./tmp')
print('-' * 20, 'phase 00 - parse using the bootstrap grammar')
with open('grammar/tatsu.ebnf') as f:
text = str(f.read())
g = EBNFParser('EBNFBootstrap')
grammar0 = g.parse(text)
ast0 = json.dumps(asjson(grammar0), indent=2)
with open('./tmp/00.ast', 'w') as f:
f.write(ast0)
print('-' * 20, 'phase 01 - parse with parser generator')
with open('grammar/tatsu.ebnf') as f:
text = str(f.read())
g = GrammarGenerator('EBNFBootstrap')
g.parse(text)
generated_grammar1 = str(g.ast['start'])
with open('./tmp/01.ebnf', 'w') as f:
f.write(generated_grammar1)
print('-' * 20, 'phase 02 - parse previous output with the parser generator')
with open('./tmp/01.ebnf') as f:
text = str(f.read())
g = GrammarGenerator('EBNFBootstrap')
g.parse(text)
generated_grammar2 = str(g.ast['start'])
with open('./tmp/02.ebnf', 'w') as f:
f.write(generated_grammar2)
self.assertEqual(generated_grammar2, generated_grammar1)
print('-' * 20, 'phase 03 - repeat')
with open('./tmp/02.ebnf') as f:
text = f.read()
g = EBNFParser('EBNFBootstrap')
ast3 = g.parse(text)
with open('./tmp/03.ast', 'w') as f:
f.write(json.dumps(asjson(ast3), indent=2))
print('-' * 20, 'phase 04 - repeat')
with open('./tmp/02.ebnf') as f:
text = f.read()
g = GrammarGenerator('EBNFBootstrap')
g.parse(text)
parser = g.ast['start']
# pprint(parser.first_sets, indent=2, depth=3)
generated_grammar4 = str(parser)
with open('./tmp/04.ebnf', 'w') as f:
f.write(generated_grammar4)
self.assertEqual(generated_grammar4, generated_grammar2)
print('-' * 20, 'phase 05 - parse using the grammar model')
with open('./tmp/04.ebnf') as f:
text = f.read()
ast5 = parser.parse(text)
with open('./tmp/05.ast', 'w') as f:
f.write(json.dumps(asjson(ast5), indent=2))
print('-' * 20, 'phase 06 - generate parser code')
gencode6 = codegen(parser)
with open('./tmp/g06.py', 'w') as f:
f.write(gencode6)
print('-' * 20, 'phase 07 - import generated code')
py_compile.compile('./tmp/g06.py', doraise=True)
# g06 = __import__('g06')
# GenParser = g06.EBNFBootstrapParser
# print('-' * 20, 'phase 08 - compile using generated code')
# parser = GenParser(trace=False)
# result = parser.parse(
# text,
# 'start',
# comments_re=COMMENTS_RE,
# eol_comments_re=EOL_COMMENTS_RE
# )
# self.assertEqual(result, parser.ast['start'])
# ast8 = parser.ast['start']
# json8 = json.dumps(asjson(ast8), indent=2)
# open('./tmp/08.ast', 'w').write(json8)
# self.assertEqual(ast5, ast8)
print('-' * 20, 'phase 09 - Generate parser with semantics')
with open('grammar/tatsu.ebnf') as f:
text = f.read()
parser = GrammarGenerator('EBNFBootstrap')
g9 = parser.parse(text)
generated_grammar9 = str(g9)
with open('./tmp/09.ebnf', 'w') as f:
f.write(generated_grammar9)
self.assertEqual(generated_grammar9, generated_grammar1)
print('-' * 20, 'phase 10 - Parse with a model using a semantics')
g10 = g9.parse(
text,
start_rule='start',
semantics=EBNFGrammarSemantics('EBNFBootstrap')
)
generated_grammar10 = str(g10)
with open('./tmp/10.ebnf', 'w') as f:
f.write(generated_grammar10)
gencode10 = codegen(g10)
with open('./tmp/g10.py', 'w') as f:
f.write(gencode10)
print('-' * 20, 'phase 11 - Pickle the model and try again.')
with open('./tmp/11.tatsu', 'wb') as f:
pickle.dump(g10, f, protocol=2)
with open('./tmp/11.tatsu', 'rb') as f:
g11 = pickle.load(f)
r11 = g11.parse(
text,
start_rule='start',
semantics=EBNFGrammarSemantics('EBNFBootstrap')
)
with open('./tmp/11.ebnf', 'w') as f:
f.write(str(g11))
gencode11 = codegen(r11)
with open('./tmp/g11.py', 'w') as f:
f.write(gencode11)
print('-' * 20, 'phase 12 - Walker')
class PrintNameWalker(DepthFirstWalker):
def __init__(self):
self.walked = []
def walk_default(self, o, children):
self.walked.append(o.__class__.__name__)
v = PrintNameWalker()
v.walk(g11)
with open('./tmp/12.txt', 'w') as f:
f.write('\n'.join(v.walked))
# note: pygraphviz not yet updated
if sys.version_info >= (3, 7):
return
print('-' * 20, 'phase 13 - Graphics')
try:
from tatsu.diagrams import draw
except ImportError:
print('PyGraphViz not found!')
else:
if not util.PY37:
draw('./tmp/13.png', g11)