def test_parseinfo_directive(self): grammar = ''' @@parseinfo @@parseinfo :: True test = value:"test" $; ''' model = grako.compile(grammar, "test") ast = model.parse("test") self.assertIsNotNone(ast.parseinfo) code = codegen(model) self.assertTrue('parseinfo=True' in code) compile(code, 'test.py', EXEC) grammar = ''' @@parseinfo :: False test = value:"test" $; ''' model = grako.compile(grammar, "test") ast = model.parse("test") self.assertIsNone(ast.parseinfo) code = codegen(model) self.assertTrue('parseinfo=False' in code) compile(code, 'test.py', EXEC)
def test_right_join(self): grammar = ''' start = (op)>{number}+ $ ; op = '+' | '-' ; number = /\d+/ ; ''' text = '1 + 2 - 3 + 4' model = compile(grammar, "test") self.assertEqual(trim(grammar).strip(), str(model).strip()) codegen(model) ast = model.parse(text) self.assertEqual(('+', '1', ('-', '2', ('+', '3', '4'))), ast)
def test_empty_closure(self): grammar = ''' start = {'x'}+ {} 'y'$; ''' model = genmodel("test", grammar) codegen(model) ast = model.parse("xxxy", nameguard=False) self.assertEquals([['x', 'x', 'x'], [], 'y'], ast)
def test_empty_closure(self): grammar = ''' start = {'x'}+ {} 'y'$; ''' model = compile(grammar, "test") codegen(model) ast = model.parse("xxxy", nameguard=False) self.assertEqual([['x', 'x', 'x'], [], 'y'], ast)
def test_check_keywords(self): import parser grammar = ''' @@keyword :: A start = {id}+ $ ; @name id = /\w+/ ; ''' model = compile(grammar, 'test') c = codegen(model) parser.suite(c) ast = model.parse('hello world') self.assertEqual(['hello', 'world'], ast) try: ast = model.parse("hello A world") self.assertEqual(['hello', 'A', 'world'], ast) self.fail('accepted keyword as name') except FailedParse as e: self.assertTrue('"A" is a reserved word' in str(e)) pass
def test_normal_join(self): grammar = ''' start = ','%{'x' 'y'} 'z' ; ''' model = compile(grammar, "test") codegen(model) ast = model.parse("x y, x y z", nameguard=False) self.assertEqual([[['x', 'y'], ',', ['x', 'y']], 'z'], ast) ast = model.parse("x y z", nameguard=False) self.assertEqual([[['x', 'y']], 'z'], ast) ast = model.parse("z", nameguard=False) self.assertEqual([[], 'z'], ast)
def test_36_unichars(self): grammar = ''' start = { rule_positional | rule_keywords | rule_all }* $ ; rule_positional("ÄÖÜäöüß") = 'a' ; rule_keywords(k1='äöüÄÖÜß') = 'b' ; rule_all('ßÄÖÜäöü', k1="ßäöüÄÖÜ") = 'c' ; ''' def _trydelete(pymodule): import os try: os.unlink(pymodule + ".py") except EnvironmentError: pass try: os.unlink(pymodule + ".pyc") except EnvironmentError: pass try: os.unlink(pymodule + ".pyo") except EnvironmentError: pass def assert_equal(target, value): self.assertEqual(target, value) class UnicharsSemantics(object): """Check all rule parameters for expected types and values""" def rule_positional(self, ast, p1): assert_equal("ÄÖÜäöüß", p1) return ast def rule_keyword(self, ast, k1): assert_equal("äöüÄÖÜß", k1) return ast def rule_all(self, ast, p1, k1): assert_equal("ßÄÖÜäöü", p1) assert_equal("ßäöüÄÖÜ", k1) return ast m = genmodel("UnicodeRuleArguments", grammar) ast = m.parse("a b c") self.assertEqual(['a', 'b', 'c'], ast) semantics = UnicharsSemantics() ast = m.parse("a b c", semantics=semantics) self.assertEqual(['a', 'b', 'c'], ast) code = codegen(m) import codecs with codecs.open(TMP_DIR + "tc36unicharstest.py", "w", "utf-8") as f: f.write(code) import tc36unicharstest tc36unicharstest _trydelete("tc36unicharstest")
def test_36_unichars(self): grammar = ''' start = { rule_positional | rule_keywords | rule_all }* $ ; rule_positional("ÄÖÜäöüß") = 'a' ; rule_keywords(k1='äöüÄÖÜß') = 'b' ; rule_all('ßÄÖÜäöü', k1="ßäöüÄÖÜ") = 'c' ; ''' def _trydelete(pymodule): import os try: os.unlink(pymodule + ".py") except EnvironmentError: pass try: os.unlink(pymodule + ".pyc") except EnvironmentError: pass try: os.unlink(pymodule + ".pyo") except EnvironmentError: pass def assert_equal(target, value): self.assertEqual(target, value) class UnicharsSemantics(object): """Check all rule parameters for expected types and values""" def rule_positional(self, ast, p1): assert_equal("ÄÖÜäöüß", p1) return ast def rule_keyword(self, ast, k1): assert_equal("äöüÄÖÜß", k1) return ast def rule_all(self, ast, p1, k1): assert_equal("ßÄÖÜäöü", p1) assert_equal("ßäöüÄÖÜ", k1) return ast m = genmodel("UnicodeRuleArguments", grammar) ast = m.parse("a b c") self.assertEqual(['a', 'b', 'c'], ast) semantics = UnicharsSemantics() ast = m.parse("a b c", semantics=semantics) self.assertEqual(['a', 'b', 'c'], ast) code = codegen(m) import codecs with codecs.open("tc36unicharstest.py", "w", "utf-8") as f: f.write(code) import tc36unicharstest tc36unicharstest _trydelete("tc36unicharstest")
def test_eol_comments_re_directive(self): grammar = ''' @@eol_comments :: /#.*?$/ test = "test" $; ''' model = genmodel("test", grammar) code = codegen(model) compile(code, 'test.py', 'exec')
def test_eol_comments_re_directive(self): grammar = ''' @@eol_comments :: /#.*?$/ test = "test" $; ''' model = grako.compile(grammar, "test") code = codegen(model) compile(code, 'test.py', EXEC)
def test_whitespace_directive(self): grammar = ''' @@whitespace :: /[\t ]+/ test = "test" $; ''' model = genmodel("test", grammar) code = codegen(model) compile(code, 'test.py', 'exec')
def test_whitespace_directive(self): grammar = ''' @@whitespace :: /[\t ]+/ test = "test" $; ''' model = grako.compile(grammar, "test") code = codegen(model) compile('test.py', code, EXEC)
def test_define_keywords(self): import parser grammar = ''' @@keyword :: B C @@keyword :: 'A' start = ('a' 'b').{'x'}+ ; ''' model = compile(grammar, "test") c = codegen(model) parser.suite(c) grammar2 = str(model) model2 = compile(grammar2, "test") c2 = codegen(model2) parser.suite(c2) self.assertEqual(grammar2, str(model2))
def test_failed_ref(self): grammar = """ final = object; type = /[^\s=()]+/; object = '('type')' '{' @:{pair} {',' @:{pair}}* [','] '}'; pair = key '=' value; list = '('type')' '[' @:{object} {',' @:{object}}* [','] ']'; key = /[^\s=]+/; value = @:(string|list|object|unset|boolean|number|null) [',']; null = '('type')' @:{ 'null' }; boolean = /(true|false)/; unset = '<unset>'; string = '"' @:/[^"]*/ '"'; number = /-?[0-9]+/; """ model = genmodel("final", grammar) codegen(model) model.parse('(sometype){boolean = true}')
def test_define_keywords(self): import parser grammar = ''' @@keyword :: B C @@keyword :: 'A' start = ('a' 'b').{'x'} ; ''' model = genmodel("test", grammar) c = codegen(model) parser.suite(c) grammar2 = str(model) model2 = genmodel("test", grammar2) c2 = codegen(model2) parser.suite(c2) self.assertEqual(grammar2, str(model2))
def test_failed_ref(self): grammar = """ final = object; type = /[^\s=()]+/; object = '('type')' '{' @:{pair} {',' @:{pair}}* [','] '}'; pair = key '=' value; list = '('type')' '[' @:{object} {',' @:{object}}* [','] ']'; key = /[^\s=]+/; value = @:(string|list|object|unset|boolean|number|null) [',']; null = '('type')' @:{ 'null' }; boolean = /(true|false)/; unset = '<unset>'; string = '"' @:/[^"]*/ '"'; number = /-?[0-9]+/; """ model = compile(grammar, "final") codegen(model) model.parse('(sometype){boolean = true}')
def test_right_join(self): grammar = ''' start = (op)>{number}+ $ ; op = '+' | '-' ; number = /\d+/ ; ''' text = '1 + 2 - 3 + 4' model = compile(grammar, "test") self.assertEqual(trim(grammar).strip(), str(model).strip()) codegen(model) ast = model.parse(text) self.assertEqual( ( '+', '1', ( '-', '2', ( '+', '3', '4' ) ) ), ast )
def test_group_join(self): grammar = ''' start = ('a' 'b')%{'x'}+ ; ''' model = compile(grammar, "test") c = codegen(model) import parser parser.suite(c) ast = model.parse("x a b x", nameguard=False) self.assertEqual(['x', ['a', 'b'], 'x'], ast)
def test_group_join(self): grammar = ''' start = ('a' 'b').{'x'} ; ''' model = genmodel("test", grammar) c = codegen(model) import parser parser.suite(c) ast = model.parse("x a b x", nameguard=False) self.assertEquals(['x', 'x'], ast)
def test_left_recursion_directive(self): grammar = ''' @@left_recursion :: False test = "test" $; ''' model = genmodel("test", grammar) self.assertFalse(model.directives.get('left_recursion')) self.assertFalse(model.left_recursion) code = codegen(model) compile(code, 'test.py', 'exec')
def test_keyword_params(self): grammar = ''' start(k1=1, k2=2) = {'a'} $ ; ''' g = GrakoGrammarGenerator('Keywords') model = g.parse(grammar, trace=False) code = codegen(model) self.assertEquals('#!/usr/bin/env python', code.splitlines()[0]) pass
def test_left_recursion_directive(self): grammar = ''' @@left_recursion :: False test = "test" $; ''' model = grako.compile(grammar, "test") self.assertFalse(model.directives.get('left_recursion')) self.assertFalse(model.left_recursion) code = codegen(model) compile('test.py', code, EXEC)
def test_grammar_directive(self): grammar = ''' @@grammar :: Test start = test $; test = "test"; ''' model = grako.compile(grammar=grammar) self.assertEqual('Test', model.directives.get('grammar')) self.assertEqual('Test', model.name) code = codegen(model) module = compile(code, 'test.py', EXEC) assert 'TestParser' in module.co_names
def test_positive_gather(self): grammar = ''' start = ','.{'x' 'y'}+ ; ''' grammar2 = ''' start = (','.{'x'}+|{}) ; ''' grammar3 = ''' start = [','.{'x'}+] ; ''' model = compile(grammar, "test") codegen(model) ast = model.parse("x y, x y", nameguard=False) self.assertEqual([['x', 'y'], ['x', 'y']], ast) ast = model.parse("x y x y", nameguard=False) self.assertEqual([['x', 'y']], ast) try: ast = model.parse("y x", nameguard=False) self.Fail('closure not positive') except FailedParse: pass model = compile(grammar2, "test") ast = model.parse("y x", nameguard=False) self.assertEqual([], ast) ast = model.parse("x", nameguard=False) self.assertEqual(['x'], ast) ast = model.parse("x,x", nameguard=False) self.assertEqual(['x', 'x'], ast) model = compile(grammar3, "test") ast = model.parse("y x", nameguard=False) self.assertEqual(None, ast)
def test_join(self): grammar = ''' start = ','.{'x' 'y'} ; ''' grammar2 = ''' start = (','.{'x'}|{}) ; ''' grammar3 = ''' start = [','.{'x'}] ; ''' model = genmodel("test", grammar) codegen(model) ast = model.parse("x y, x y", nameguard=False) self.assertEquals([['x', 'y'], ['x', 'y']], ast) ast = model.parse("x y x y", nameguard=False) self.assertEquals([['x', 'y']], ast) try: ast = model.parse("y x", nameguard=False) self.Fail('closure not positive') except FailedParse: pass model = genmodel("test", grammar2) ast = model.parse("y x", nameguard=False) self.assertEquals([], ast) ast = model.parse("x", nameguard=False) self.assertEquals(['x'], ast) ast = model.parse("x,x", nameguard=False) self.assertEquals(['x', 'x'], ast) model = genmodel("test", grammar3) ast = model.parse("y x", nameguard=False) self.assertEquals(None, ast)
def main(): grammar = regex_parser.translate('(a|b)*') model = grako.compile(grammar, 'Regexp') model.parse('aaabbaba', 'S0') try: model.parse('aaaCbbaba', 'S0') raise Exception('Should not have parsed!') except grako.exceptions.FailedParse: pass print('Grammar:', file=sys.stderr) print(grammar) sys.stdout.flush() with open(PARSER_FILENAME, 'w') as f: f.write(codegen(model)) print('Generated parser saved as:', PARSER_FILENAME, file=sys.stderr) print(file=sys.stderr)
def main(): grammar = regex_parser.translate('(a|b)*') model = grako.genmodel('Regexp', grammar) model.parse('aaabbaba', 'S0') try: model.parse('aaaCbbaba', 'S0') raise Exception('Should not have parsed!') except grako.exceptions.FailedParse: pass print('Grammar:', file=sys.stderr) print(grammar) sys.stdout.flush() with open(PARSER_FILENAME, 'w') as f: f.write(codegen(model)) print('Generated parser saved as:', PARSER_FILENAME, file=sys.stderr) print(file=sys.stderr)
def test_check_keywords(self): import parser grammar = ''' @@keyword :: A start = {id}+ $ ; @name id = /\w+/ ; ''' model = genmodel('test', grammar) c = codegen(model) parser.suite(c) ast = model.parse('hello world') self.assertEqual(['hello', 'world'], ast) try: ast = model.parse("hello A world") self.assertEqual(['hello', 'A', 'world'], ast) self.fail('accepted keyword as name') except FailedSemantics: pass
def test_36_param_combinations(self): def assert_equal(target, value): self.assertEqual(target, value) class TC36Semantics(object): """Check all rule parameters for expected types and values""" def rule_positional(self, ast, p1, p2, p3, p4): assert_equal("ABC", p1) assert_equal(123, p2) assert_equal('=', p3) assert_equal("+", p4) return ast def rule_keyword(self, ast, k1, k2, k3, k4): assert_equal("ABC", k1) assert_equal(123, k2) assert_equal('=', k3) assert_equal('+', k4) return ast def rule_all(self, ast, p1, p2, p3, p4, k1, k2, k3, k4): assert_equal("DEF", p1) assert_equal(456, p2) assert_equal('=', p3) assert_equal("+", p4) assert_equal("HIJ", k1) assert_equal(789, k2) assert_equal('=', k3) assert_equal('+', k4) return ast grammar = ''' @@ignorecase::False @@nameguard start = {rule_positional | rule_keywords | rule_all} $ ; rule_positional('ABC', 123, '=', '+') = 'a' ; rule_keywords(k1=ABC, k3='=', k4='+', k2=123) = 'b' ; rule_all('DEF', 456, '=', '+', k1=HIJ, k3='=', k4='+', k2=789) = 'c' ; ''' pretty = ''' @@ignorecase :: False @@nameguard :: True start = {rule_positional | rule_keywords | rule_all} $ ; rule_positional(ABC, 123, '=', '+') = 'a' ; rule_keywords(k1=ABC, k3='=', k4='+', k2=123) = 'b' ; rule_all(DEF, 456, '=', '+', k1=HIJ, k3='=', k4='+', k2=789) = 'c' ; ''' model = genmodel('RuleArguments', grammar) self.assertEqual(trim(pretty), ustr(model)) model = genmodel('RuleArguments', pretty) ast = model.parse("a b c") self.assertEqual(['a', 'b', 'c'], ast) semantics = TC36Semantics() ast = model.parse("a b c", semantics=semantics) self.assertEqual(['a', 'b', 'c'], ast) codegen(model)
def test_bootstrap(self): print() if os.path.isfile(TMP_DIR_PATH + '00.ast'): shutil.rmtree(TMP_DIR_PATH) if not os.path.isdir(TMP_DIR_PATH): os.mkdir(TMP_DIR_PATH) print('-' * 20, 'phase 00 - parse using the bootstrap grammar') with open(ETC_DIR_PATH + 'grako.ebnf') as f: text = str(f.read()) g = GrakoParser('GrakoBootstrap') grammar0 = g.parse(text) ast0 = json.dumps(grammar0, indent=2) with open(TMP_DIR_PATH + '00.ast', 'w') as f: f.write(ast0) print('-' * 20, 'phase 01 - parse with parser generator') with open(ETC_DIR_PATH + 'grako.ebnf') as f: text = str(f.read()) g = GrakoGrammarGenerator('GrakoBootstrap') g.parse(text, trace=False) generated_grammar1 = str(g.ast['start']) with open(TMP_DIR_PATH + '01.ebnf', 'w') as f: f.write(generated_grammar1) print('-' * 20, 'phase 02 - parse previous output with the parser generator') with open(TMP_DIR_PATH + '01.ebnf', 'r') as f: text = str(f.read()) g = GrakoGrammarGenerator('GrakoBootstrap') g.parse(text, trace=False) generated_grammar2 = str(g.ast['start']) with open(TMP_DIR_PATH + '02.ebnf', 'w') as f: f.write(generated_grammar2) self.assertEqual(generated_grammar2, generated_grammar1) print('-' * 20, 'phase 03 - repeat') with open(TMP_DIR_PATH + '02.ebnf') as f: text = f.read() g = GrakoParser('GrakoBootstrap') ast3 = g.parse(text) with open(TMP_DIR_PATH + '03.ast', 'w') as f: f.write(json.dumps(ast3, indent=2)) print('-' * 20, 'phase 04 - repeat') with open(TMP_DIR_PATH + '02.ebnf') as f: text = f.read() g = GrakoGrammarGenerator('GrakoBootstrap') g.parse(text) parser = g.ast['start'] # pprint(parser.first_sets, indent=2, depth=3) generated_grammar4 = str(parser) with open(TMP_DIR_PATH + '04.ebnf', 'w') as f: f.write(generated_grammar4) self.assertEqual(generated_grammar4, generated_grammar2) print('-' * 20, 'phase 05 - parse using the grammar model') with open(TMP_DIR_PATH + '04.ebnf') as f: text = f.read() ast5 = parser.parse(text) with open(TMP_DIR_PATH + '05.ast', 'w') as f: f.write(json.dumps(ast5, indent=2)) print('-' * 20, 'phase 06 - generate parser code') gencode6 = codegen(parser) with open(TMP_DIR_PATH + 'g06.py', 'w') as f: f.write(gencode6) print('-' * 20, 'phase 07 - import generated code') from g06 import GrakoBootstrapParser as GenParser # @UnresolvedImport print('-' * 20, 'phase 08 - compile using generated code') parser = GenParser(trace=False) result = parser.parse( text, 'start', comments_re=COMMENTS_RE, eol_comments_re=EOL_COMMENTS_RE ) self.assertEqual(result, parser.ast['start']) ast8 = parser.ast['start'] json8 = json.dumps(ast8, indent=2) open(TMP_DIR_PATH + '08.ast', 'w').write(json8) self.assertEqual(ast5, ast8) print('-' * 20, 'phase 09 - Generate parser with semantics') with open(ETC_DIR_PATH + 'grako.ebnf') as f: text = f.read() parser = GrakoGrammarGenerator('GrakoBootstrap') g9 = parser.parse(text) generated_grammar9 = str(g9) with open(TMP_DIR_PATH + '09.ebnf', 'w') as f: f.write(generated_grammar9) self.assertEqual(generated_grammar9, generated_grammar1) print('-' * 20, 'phase 10 - Parse with a model using a semantics') g10 = g9.parse( text, start_rule='start', semantics=GrakoSemantics('GrakoBootstrap') ) generated_grammar10 = str(g10) with open(TMP_DIR_PATH + '10.ebnf', 'w') as f: f.write(generated_grammar10) gencode10 = codegen(g10) with open(TMP_DIR_PATH + 'g10.py', 'w') as f: f.write(gencode10) print('-' * 20, 'phase 11 - Pickle the model and try again.') with open(TMP_DIR_PATH + '11.grako', 'wb') as f: pickle.dump(g10, f, protocol=2) with open(TMP_DIR_PATH + '11.grako', 'rb') as f: g11 = pickle.load(f) r11 = g11.parse( text, start_rule='start', semantics=GrakoSemantics('GrakoBootstrap') ) with open(TMP_DIR_PATH + '11.ebnf', 'w') as f: f.write(str(g11)) gencode11 = codegen(r11) with open(TMP_DIR_PATH + 'g11.py', 'w') as f: f.write(gencode11) print('-' * 20, 'phase 12 - Walker') class PrintNameWalker(DepthFirstWalker): def __init__(self): self.walked = [] def walk_default(self, o, children): self.walked.append(o.__class__.__name__) v = PrintNameWalker() v.walk(g11) with open(TMP_DIR_PATH + '12.txt', 'w') as f: f.write('\n'.join(v.walked)) print('-' * 20, 'phase 13 - Graphics') try: from grako.diagrams import draw except ImportError: print('PyGraphViz not found!') else: draw('tmp/13.png', g11)