def test_builder_semantics(self): grammar = ''' start::sum = {number}+ $ ; number::int = /\d+/ ; ''' text = '5 4 3 2 1' semantics = ModelBuilderSemantics() model = genmodel('test', grammar) ast = model.parse(text, semantics=semantics) self.assertEqual(15, ast) import functools dotted = functools.partial(type('').join, '.') dotted.__name__ = 'dotted' grammar = ''' start::dotted = {number}+ $ ; number = /\d+/ ; ''' semantics = ModelBuilderSemantics(types=[dotted]) model = genmodel('test', grammar) ast = model.parse(text, semantics=semantics) self.assertEqual('5.4.3.2.1', ast)
def test_no_left_recursion(self): grammar = ''' start = expre $ ; expre = expre '+' number | expre '*' number | number ; number = ?/[0-9]+/? ; ''' try: genmodel("test", grammar, left_recursion=False) self.fail('expected left recursion failure') except FailedLeftRecursion: pass
def test_empty_match_token(self): grammar = """ table = { row }+ ; row = (cell1:cell "|" cell2:cell) "\n"; cell = /[a-z]+/ ; """ try: genmodel("model", grammar) self.fail('allowed empty token') except FailedParse: pass
def test_optional_sequence(self): grammar = ''' start = '1' ['2' '3'] '4' $ ; ''' model = genmodel("test", grammar) ast = model.parse("1234", nameguard=False) self.assertEquals(['1', '2', '3', '4'], ast) grammar = ''' start = '1' foo:['2' '3'] '4' $ ; ''' model = genmodel("test", grammar) ast = model.parse("1234", nameguard=False) self.assertEquals(['2', '3'], ast.foo)
def test_ast_assignment(self): grammar = ''' n = @: {"a"}* $ ; f = @+: {"a"}* $ ; nn = @: {"a"}* @: {"b"}* $ ; nf = @: {"a"}* @+: {"b"}* $ ; fn = @+: {"a"}* @: {"b"}* $ ; ff = @+: {"a"}* @+: {"b"}* $ ; ''' model = genmodel("test", grammar) def p(input, rule): return model.parse(input, start=rule, whitespace='') e = self.assertEqual e([], p('', 'n')) e(['a'], p('a', 'n')) e(['a', 'a'], p('aa', 'n')) e([[]], p('', 'f')) e([['a']], p('a', 'f')) e([['a', 'a']], p('aa', 'f')) for r in ('nn', 'nf', 'fn', 'ff'): e([[], []], p('', r)) e([['a'], []], p('a', r)) e([[], ['b']], p('b', r)) e([['a', 'a'], []], p('aa', r)) e([[], ['b', 'b']], p('bb', r)) e([['a', 'a'], ['b']], p('aab', r))
def test_group_ast(self): grammar = ''' start = '1' ('2' '3') '4' $ ; ''' model = genmodel("test", grammar) ast = model.parse("1234", nameguard=False) self.assertEquals(['1', '2', '3', '4'], ast)
def test_patterns_with_newlines(self): grammar = ''' start = blanklines $ ; blanklines = blankline [blanklines] ; blankline = /^[^\n]*\n?$/ ; blankline2 = ?/^[^\n]*\n?$/? ; ''' model = genmodel("test", grammar) ast = model.parse('\n\n') self.assertEqual('', ustr(ast))
def test_numbers_and_unicode(self): grammar = ''' rúle(1, -23, 4.56, 7.89e-11, 0xABCDEF, Añez) = 'a' ; ''' rule2 = ''' rulé::Añez = '\\xf1' ; ''' rule3 = ''' rúlé::Añez = 'ñ' ; ''' if PY3: grammar += rule3 else: grammar += rule2 model = genmodel("test", grammar) self.assertEquals(trim(grammar), ustr(model))
def test_stateful(self): # Parser for mediawiki-style unordered lists. grammar = r''' document = @:ul [ nl ] $ ; ul = "*" ul_start el+:li { nl el:li } * ul_end ; li = ul | li_text ; (* Quirk: If a text line is followed by a sublist, the sublist does not get its own li. *) li_text = text:text [ ul:li_followed_by_ul ] ; li_followed_by_ul = nl @:ul ; text = ?/.*/? ; nl = ?/\n/? ul_marker ; (* The following rules are placeholders for state transitions. *) ul_start = () ; ul_end = () ; (* The following rules are placeholders for state validations and grammar rules. *) ul_marker = () ; ''' class StatefulSemantics(object): def __init__(self, parser): self._context = parser def ul_start(self, ast): ctx = self._context ctx._state = 1 if ctx._state is None else ctx._state + 1 return ast def ul_end(self, ast): ctx = self._context ctx._state = None if ctx._state is None or ctx._state <= 1 else ctx._state - 1 return ast def ul_marker(self, ast): ctx = self._context if ctx._state is not None: if not ctx.buf.match("*" * ctx._state): raise FailedSemantics("not at correct level") return ast def ul(self, ast): return "<ul>" + "".join(ast.el) + "</ul>" def li(self, ast): return "<li>" + ast + "</li>" def li_text(self, ast): return ast.text if ast.ul is None else ast.text + ast.ul model = genmodel("test", grammar) context = ModelContext(model.rules, whitespace='', nameguard=False) ast = model.parse('*abc', "document", context=context, semantics=StatefulSemantics(context), whitespace='', nameguard=False) self.assertEqual(ast, "<ul><li>abc</li></ul>") ast = model.parse('*abc\n', "document", context=context, semantics=StatefulSemantics(context), whitespace='', nameguard=False) self.assertEqual("<ul><li>abc</li></ul>", ast) ast = model.parse('*abc\n*def\n', "document", context=context, semantics=StatefulSemantics(context), whitespace='', nameguard=False) self.assertEqual("<ul><li>abc</li><li>def</li></ul>", ast) ast = model.parse('**abc', "document", context=context, semantics=StatefulSemantics(context), whitespace='', nameguard=False) self.assertEqual("<ul><li><ul><li>abc</li></ul></li></ul>", ast) ast = model.parse('*abc\n**def\n', "document", context=context, semantics=StatefulSemantics(context), whitespace='', nameguard=False) self.assertEqual("<ul><li>abc<ul><li>def</li></ul></li></ul>", ast)
def test_no_left_recursion(self): grammar = ''' start = expre $ ; expre = expre '+' number | expre '*' number | number ; number = ?/[0-9]+/? ; ''' model = genmodel("test", grammar) model.parse("1*2+3*5") try: model.parse("1*2+3*5", left_recursion=False) self.Fail('expected left recursion failure') except FailedParse: pass
def test_no_left_recursion(self): grammar = ''' start = expre $ ; expre = expre '+' number | expre '*' number | number ; number = ?/[0-9]+/? ; ''' model = genmodel("test", grammar) model.parse("1*2+3*5") try: model.parse("1*2+3*5", left_recursion=False) self.fail('expected left recursion failure') except FailedParse: pass
def test_36_unichars(self): grammar = ''' start = { rule_positional | rule_keywords | rule_all }* $ ; rule_positional("ÄÖÜäöüß") = 'a' ; rule_keywords(k1='äöüÄÖÜß') = 'b' ; rule_all('ßÄÖÜäöü', k1="ßäöüÄÖÜ") = 'c' ; ''' def _trydelete(pymodule): import os try: os.unlink(pymodule + ".py") except EnvironmentError: pass try: os.unlink(pymodule + ".pyc") except EnvironmentError: pass try: os.unlink(pymodule + ".pyo") except EnvironmentError: pass def assert_equal(target, value): self.assertEqual(target, value) class UnicharsSemantics(object): """Check all rule parameters for expected types and values""" def rule_positional(self, ast, p1): assert_equal("ÄÖÜäöüß", p1) return ast def rule_keyword(self, ast, k1): assert_equal("äöüÄÖÜß", k1) return ast def rule_all(self, ast, p1, k1): assert_equal("ßÄÖÜäöü", p1) assert_equal("ßäöüÄÖÜ", k1) return ast m = genmodel("UnicodeRuleArguments", grammar) ast = m.parse("a b c") self.assertEqual(['a', 'b', 'c'], ast) semantics = UnicharsSemantics() ast = m.parse("a b c", semantics=semantics) self.assertEqual(['a', 'b', 'c'], ast) code = codegen(m) import codecs with codecs.open("tc36unicharstest.py", "w", "utf-8") as f: f.write(code) import tc36unicharstest tc36unicharstest _trydelete("tc36unicharstest")
def test_36_unichars(self): grammar = ''' start = { rule_positional | rule_keywords | rule_all }* $ ; rule_positional("ÄÖÜäöüß") = 'a' ; rule_keywords(k1='äöüÄÖÜß') = 'b' ; rule_all('ßÄÖÜäöü', k1="ßäöüÄÖÜ") = 'c' ; ''' def _trydelete(pymodule): import os try: os.unlink(pymodule + ".py") except EnvironmentError: pass try: os.unlink(pymodule + ".pyc") except EnvironmentError: pass try: os.unlink(pymodule + ".pyo") except EnvironmentError: pass def assert_equal(target, value): self.assertEqual(target, value) class UnicharsSemantics(object): """Check all rule parameters for expected types and values""" def rule_positional(self, ast, p1): assert_equal("ÄÖÜäöüß", p1) return ast def rule_keyword(self, ast, k1): assert_equal("äöüÄÖÜß", k1) return ast def rule_all(self, ast, p1, k1): assert_equal("ßÄÖÜäöü", p1) assert_equal("ßäöüÄÖÜ", k1) return ast m = genmodel("UnicodeRuleArguments", grammar) ast = m.parse("a b c") self.assertEqual(['a', 'b', 'c'], ast) semantics = UnicharsSemantics() ast = m.parse("a b c", semantics=semantics) self.assertEqual(['a', 'b', 'c'], ast) code = codegen(m) import codecs with codecs.open(TMP_DIR + "tc36unicharstest.py", "w", "utf-8") as f: f.write(code) import tc36unicharstest tc36unicharstest _trydelete("tc36unicharstest")
def test_update_ast(self): grammar = ''' foo = name:"1" [ name: bar ] ; bar = { "2" } * ; ''' m = genmodel('Keywords', grammar) ast = m.parse('1 2') self.assertEqual(['1', ['2']], ast.name) grammar = ''' start = items: { item } * $ ; item = @:{ subitem } * "0" ; subitem = ?/1+/? ; ''' m = genmodel('Update', grammar) ast = m.parse("1101110100", nameguard=False) self.assertEquals([['11'], ['111'], ['1'], []], ast.items_)
def test_empty_closure(self): grammar = ''' start = {'x'}+ {} 'y'$; ''' model = genmodel("test", grammar) codegen(model) ast = model.parse("xxxy", nameguard=False) self.assertEquals([['x', 'x', 'x'], [], 'y'], ast)
def test_36_params_and_keyword_params(self): grammar = ''' rule(A, kwdB=B) = 'a' ; ''' model = genmodel("test", grammar) self.assertEquals(trim(grammar), ustr(model))
def test_whitespace_directive(self): grammar = ''' @@whitespace :: /[\t ]+/ test = "test" $; ''' model = genmodel("test", grammar) code = codegen(model) compile(code, 'test.py', 'exec')
def test_eol_comments_re_directive(self): grammar = ''' @@eol_comments :: /#.*?$/ test = "test" $; ''' model = genmodel("test", grammar) code = codegen(model) compile(code, 'test.py', 'exec')
def test_indirect_left_recursion_with_cut(self): grammar = ''' start = x $ ; x = expr ; expr = x '-' ~ num | num; num = ?/[0-9]+/? ; ''' model = genmodel("test", grammar) ast = model.parse("5-87-32") self.assertEquals(['5', '-', '87', '-', '32'], ast)
def test_rule_include(self): grammar = ''' start = b $; a = @:'a' ; b = >a {@:'b'} ; ''' model = genmodel("test", grammar) ast = model.parse("abb", nameguard=False) self.assertEquals(['a', 'b', 'b'], ast)
def test_define_keywords(self): import parser grammar = ''' @@keyword :: B C @@keyword :: 'A' start = ('a' 'b').{'x'} ; ''' model = genmodel("test", grammar) c = codegen(model) parser.suite(c) grammar2 = str(model) model2 = genmodel("test", grammar2) c2 = codegen(model2) parser.suite(c2) self.assertEqual(grammar2, str(model2))
def test_optional_closure(self): grammar = 'start = foo+:"x" foo:{"y"}* {foo:"z"}* ;' model = genmodel("test", grammar) ast = model.parse("xyyzz", nameguard=False) self.assertEquals(['x', ['y', 'y'], 'z', 'z'], ast.foo) grammar = 'start = foo+:"x" [foo+:{"y"}*] {foo:"z"}* ;' model = genmodel("test", grammar) ast = model.parse("xyyzz", nameguard=False) self.assertEquals(['x', ['y', 'y'], 'z', 'z'], ast.foo) grammar = 'start = foo+:"x" foo:[{"y"}*] {foo:"z"}* ;' model = genmodel("test", grammar) ast = model.parse("xyyzz", nameguard=False) self.assertEquals(['x', ['y', 'y'], 'z', 'z'], ast.foo) grammar = 'start = foo+:"x" [foo:{"y"}*] {foo:"z"}* ;' model = genmodel("test", grammar) ast = model.parse("xyyzz", nameguard=False) self.assertEquals(['x', ['y', 'y'], 'z', 'z'], ast.foo)
def test_group_join(self): grammar = ''' start = ('a' 'b').{'x'} ; ''' model = genmodel("test", grammar) c = codegen(model) import parser parser.suite(c) ast = model.parse("x a b x", nameguard=False) self.assertEquals(['x', 'x'], ast)
def test_new_override(self): grammar = ''' start = @:'a' {@:'b'} $ ; ''' model = genmodel("test", grammar) ast = model.parse("abb", nameguard=False) self.assertEquals(['a', 'b', 'b'], ast)
def test_dot(self): grammar = ''' start = "foo\\nbar" $; ''' try: from grako.diagrams import draw except ImportError: return m = genmodel('Diagram', grammar) draw('tmp/diagram.png', m)
def test_48_rule_override(self): grammar = ''' start = ab $; ab = 'xyz' ; @override ab = @:'a' {@:'b'} ; ''' model = genmodel("test", grammar) ast = model.parse("abb", nameguard=False) self.assertEquals(['a', 'b', 'b'], ast)
def test_left_recursion_directive(self): grammar = ''' @@left_recursion :: False test = "test" $; ''' model = genmodel("test", grammar) self.assertFalse(model.directives.get('left_recursion')) self.assertFalse(model.left_recursion) code = codegen(model) compile(code, 'test.py', 'exec')
def test_keywords_in_rule_names(self): grammar = ''' start = whitespace ; whitespace = {'x'}+ ; ''' m = genmodel('Keywords', grammar) m.parse('x')
def test_indirect_left_recursion_complex(self): grammar = ''' start = Primary $ ; Primary = PrimaryNoNewArray ; PrimaryNoNewArray = ClassInstanceCreationExpression | MethodInvocation | FieldAccess | ArrayAccess | 'this' ; ClassInstanceCreationExpression = 'new' ClassOrInterfaceType '(' ')' | Primary '.new' Identifier '()' ; MethodInvocation = Primary '.' MethodName '()' | MethodName '()' ; FieldAccess = Primary '.' Identifier | 'super.' Identifier ; ArrayAccess = Primary '[' Expression ']' | ExpressionName '[' Expression ']' ; ClassOrInterfaceType = ClassName | InterfaceTypeName ; ClassName = 'C' | 'D' ; InterfaceTypeName = 'I' | 'J' ; Identifier = 'x' | 'y' | ClassOrInterfaceType ; MethodName = 'm' | 'n' ; ExpressionName = Identifier ; Expression = 'i' | 'j' ; ''' model = genmodel("test", grammar) ast = model.parse("this") self.assertEquals('this', ast) ast = model.parse("this.x") self.assertEquals(['this', '.', 'x'], ast) ast = model.parse("this.x.y") self.assertEquals(['this', '.', 'x', '.', 'y'], ast) ast = model.parse("this.x.m()") self.assertEquals(['this', '.', 'x', '.', 'm', '()'], ast) ast = model.parse("x[i][j].y") self.assertEquals(['x', '[', 'i', ']', '[', 'j', ']', '.', 'y'], ast)
def test_join(self): grammar = ''' start = ','.{'x' 'y'} ; ''' grammar2 = ''' start = (','.{'x'}|{}) ; ''' grammar3 = ''' start = [','.{'x'}] ; ''' model = genmodel("test", grammar) codegen(model) ast = model.parse("x y, x y", nameguard=False) self.assertEquals([['x', 'y'], ['x', 'y']], ast) ast = model.parse("x y x y", nameguard=False) self.assertEquals([['x', 'y']], ast) try: ast = model.parse("y x", nameguard=False) self.Fail('closure not positive') except FailedParse: pass model = genmodel("test", grammar2) ast = model.parse("y x", nameguard=False) self.assertEquals([], ast) ast = model.parse("x", nameguard=False) self.assertEquals(['x'], ast) ast = model.parse("x,x", nameguard=False) self.assertEquals(['x', 'x'], ast) model = genmodel("test", grammar3) ast = model.parse("y x", nameguard=False) self.assertEquals(None, ast)
def test_grammar_directive(self): grammar = ''' @@grammar :: Test start = test $; test = "test"; ''' model = genmodel(grammar=grammar) self.assertEquals('Test', model.directives.get('grammar')) self.assertEquals('Test', model.name) code = codegen(model) module = compile(code, 'test.py', 'exec') assert 'TestParser' in module.co_names
def test_nested_left_recursion(self): grammar_a = ''' s = e $ ; e = [e '+'] t ; t = [t '*'] a ; a = ?/[0-9]/? ; ''' grammar_b = ''' s = e $ ; e = [e '+'] a ; a = n | p ; n = ?/[0-9]/? ; p = '(' @:e ')' ; ''' model_a = genmodel("test", grammar_a) model_b = genmodel("test", grammar_b) ast = model_a.parse("1*2+3*4") self.assertEquals(['1', '*', '2', '+', ['3', '*', '4']], ast) ast = model_b.parse("(1+2)+(3+4)") self.assertEquals(['1', '+', '2', '+', ['3', '+', '4']], ast) ast = model_a.parse("1*2*3") self.assertEquals(['1', '*', '2', '*', '3'], ast) ast = model_b.parse("(((1+2)))") self.assertEquals(['1', '+', '2'], ast)
def test_partial_choice(self): grammar = ''' start = o:[o] x:'A' $ ; o = 'A' a:'A' | 'A' b:'B' ; ''' model = genmodel("test", grammar) ast = model.parse("A", nameguard=False) self.assertEquals({'x': 'A', 'o': None}, ast)
def test_failed_ref(self): grammar = """ final = object; type = /[^\s=()]+/; object = '('type')' '{' @:{pair} {',' @:{pair}}* [','] '}'; pair = key '=' value; list = '('type')' '[' @:{object} {',' @:{object}}* [','] ']'; key = /[^\s=]+/; value = @:(string|list|object|unset|boolean|number|null) [',']; null = '('type')' @:{ 'null' }; boolean = /(true|false)/; unset = '<unset>'; string = '"' @:/[^"]*/ '"'; number = /-?[0-9]+/; """ model = genmodel("final", grammar) codegen(model) model.parse('(sometype){boolean = true}')