def main(codegen=pythoncg): args = parse_args() if args.whitespace: args.whitespace = eval_escapes(args.whitespace) outfile = args.outfile prepare_for_output(outfile) prepare_for_output(args.object_model_outfile) grammar = codecs.open(args.filename, encoding='utf-8').read() try: model = compile(grammar, args.name, trace=args.trace, filename=args.filename, colorize=args.color) model.whitespace = args.whitespace model.nameguard = args.nameguard model.left_recursion = args.left_recursion if args.draw: from tatsu import diagrams diagrams.draw(outfile, model) else: if args.pretty: result = model.pretty() elif args.pretty_lean: result = model.pretty_lean() elif args.object_model: result = objectmodel.codegen(model, base_type=args.base_type) else: result = codegen(model) if outfile: save(outfile, result) else: print(result) # if requested, always save it if args.object_model_outfile: save(args.object_model_outfile, objectmodel.codegen(model, base_type=args.base_type)) print('-' * 72, file=sys.stderr) print('{:12,d} lines in grammar'.format(len(grammar.split())), file=sys.stderr) print('{:12,d} rules in grammar'.format(len(model.rules)), file=sys.stderr) # noqa print('{:12,d} nodes in AST'.format(model.nodecount()), file=sys.stderr) except ParseException as e: print(e, file=sys.stderr) sys.exit(1)
def test_dot(self): grammar = ''' start = "foo\\nbar" $; ''' try: from tatsu.diagrams import draw except ImportError: return m = compile(grammar, 'Diagram') draw('tmp/diagram.png', m)
def main(codegen=pythoncg): args = parse_args() if args.whitespace: args.whitespace = eval_escapes(args.whitespace) outfile = args.outfile prepare_for_output(outfile) prepare_for_output(args.object_model_outfile) grammar = codecs.open(args.filename, 'r', encoding='utf-8').read() try: model = compile( grammar, args.name, trace=args.trace, filename=args.filename, colorize=args.color ) model.whitespace = args.whitespace model.nameguard = args.nameguard model.left_recursion = args.left_recursion if args.draw: from tatsu import diagrams diagrams.draw(outfile, model) else: if args.pretty: result = model.pretty() elif args.pretty_lean: result = model.pretty_lean() elif args.object_model: result = objectmodel.codegen(model) else: result = codegen(model) if outfile: save(outfile, result) else: print(result) # if requested, always save it if args.object_model_outfile: save(args.object_model_outfile, objectmodel.codegen(model)) print('-' * 72, file=sys.stderr) print('{:12,d} lines in grammar'.format(len(grammar.split())), file=sys.stderr) print('{:12,d} rules in grammar'.format(len(model.rules)), file=sys.stderr) # noqa print('{:12,d} nodes in AST'.format(model.nodecount()), file=sys.stderr) except ParseException as e: print(e, file=sys.stderr) sys.exit(1)
def test_bootstrap(self): print() if os.path.isfile('./tmp/00.ast'): shutil.rmtree('./tmp') if not os.path.isdir('./tmp'): os.mkdir('./tmp') print('-' * 20, 'phase 00 - parse using the bootstrap grammar') with open('grammar/tatsu.ebnf') as f: text = str(f.read()) g = EBNFParser('EBNFBootstrap') grammar0 = g.parse(text) ast0 = json.dumps(asjson(grammar0), indent=2) with open('./tmp/00.ast', 'w') as f: f.write(ast0) print('-' * 20, 'phase 01 - parse with parser generator') with open('grammar/tatsu.ebnf') as f: text = str(f.read()) g = GrammarGenerator('EBNFBootstrap') g.parse(text) generated_grammar1 = str(g.ast['start']) with open('./tmp/01.ebnf', 'w') as f: f.write(generated_grammar1) print('-' * 20, 'phase 02 - parse previous output with the parser generator') with open('./tmp/01.ebnf') as f: text = str(f.read()) g = GrammarGenerator('EBNFBootstrap') g.parse(text) generated_grammar2 = str(g.ast['start']) with open('./tmp/02.ebnf', 'w') as f: f.write(generated_grammar2) self.assertEqual(generated_grammar2, generated_grammar1) print('-' * 20, 'phase 03 - repeat') with open('./tmp/02.ebnf') as f: text = f.read() g = EBNFParser('EBNFBootstrap') ast3 = g.parse(text) with open('./tmp/03.ast', 'w') as f: f.write(json.dumps(asjson(ast3), indent=2)) print('-' * 20, 'phase 04 - repeat') with open('./tmp/02.ebnf') as f: text = f.read() g = GrammarGenerator('EBNFBootstrap') g.parse(text) parser = g.ast['start'] # pprint(parser.first_sets, indent=2, depth=3) generated_grammar4 = str(parser) with open('./tmp/04.ebnf', 'w') as f: f.write(generated_grammar4) self.assertEqual(generated_grammar4, generated_grammar2) print('-' * 20, 'phase 05 - parse using the grammar model') with open('./tmp/04.ebnf') as f: text = f.read() ast5 = parser.parse(text) with open('./tmp/05.ast', 'w') as f: f.write(json.dumps(asjson(ast5), indent=2)) print('-' * 20, 'phase 06 - generate parser code') gencode6 = codegen(parser) with open('./tmp/g06.py', 'w') as f: f.write(gencode6) print('-' * 20, 'phase 07 - import generated code') py_compile.compile('./tmp/g06.py', doraise=True) # g06 = __import__('g06') # GenParser = g06.EBNFBootstrapParser # print('-' * 20, 'phase 08 - compile using generated code') # parser = GenParser(trace=False) # result = parser.parse( # text, # 'start', # comments_re=COMMENTS_RE, # eol_comments_re=EOL_COMMENTS_RE # ) # self.assertEqual(result, parser.ast['start']) # ast8 = parser.ast['start'] # json8 = json.dumps(asjson(ast8), indent=2) # open('./tmp/08.ast', 'w').write(json8) # self.assertEqual(ast5, ast8) print('-' * 20, 'phase 09 - Generate parser with semantics') with open('grammar/tatsu.ebnf') as f: text = f.read() parser = GrammarGenerator('EBNFBootstrap') g9 = parser.parse(text) generated_grammar9 = str(g9) with open('./tmp/09.ebnf', 'w') as f: f.write(generated_grammar9) self.assertEqual(generated_grammar9, generated_grammar1) print('-' * 20, 'phase 10 - Parse with a model using a semantics') g10 = g9.parse( text, start_rule='start', semantics=EBNFGrammarSemantics('EBNFBootstrap') ) generated_grammar10 = str(g10) with open('./tmp/10.ebnf', 'w') as f: f.write(generated_grammar10) gencode10 = codegen(g10) with open('./tmp/g10.py', 'w') as f: f.write(gencode10) print('-' * 20, 'phase 11 - Pickle the model and try again.') with open('./tmp/11.tatsu', 'wb') as f: pickle.dump(g10, f, protocol=2) with open('./tmp/11.tatsu', 'rb') as f: g11 = pickle.load(f) r11 = g11.parse( text, start_rule='start', semantics=EBNFGrammarSemantics('EBNFBootstrap') ) with open('./tmp/11.ebnf', 'w') as f: f.write(str(g11)) gencode11 = codegen(r11) with open('./tmp/g11.py', 'w') as f: f.write(gencode11) print('-' * 20, 'phase 12 - Walker') class PrintNameWalker(DepthFirstWalker): def __init__(self): self.walked = [] def walk_default(self, o, children): self.walked.append(o.__class__.__name__) v = PrintNameWalker() v.walk(g11) with open('./tmp/12.txt', 'w') as f: f.write('\n'.join(v.walked)) # note: pygraphviz not yet updated if sys.version_info >= (3, 7): return print('-' * 20, 'phase 13 - Graphics') try: from tatsu.diagrams import draw except ImportError: print('PyGraphViz not found!') else: if not util.PY37: draw('./tmp/13.png', g11)