def test_builder_semantics(self):
grammar = '''
start::sum = {number}+ $ ;
number::int = /\d+/ ;
'''
text = '5 4 3 2 1'
semantics = ModelBuilderSemantics()
model = genmodel('test', grammar)
ast = model.parse(text, semantics=semantics)
self.assertEqual(15, ast)
import functools
dotted = functools.partial(type('').join, '.')
dotted.__name__ = 'dotted'
grammar = '''
start::dotted = {number}+ $ ;
number = /\d+/ ;
'''
semantics = ModelBuilderSemantics(types=[dotted])
model = genmodel('test', grammar)
ast = model.parse(text, semantics=semantics)
self.assertEqual('5.4.3.2.1', ast)
def test_no_left_recursion(self):
grammar = '''
start
=
expre $
;
expre
=
expre '+' number
|
expre '*' number
|
number
;
number
=
?/[0-9]+/?
;
'''
try:
genmodel("test", grammar, left_recursion=False)
self.fail('expected left recursion failure')
except FailedLeftRecursion:
pass
def test_builder_semantics(self):
grammar = '''
start::sum = {number}+ $ ;
number::int = /\d+/ ;
'''
text = '5 4 3 2 1'
semantics = ModelBuilderSemantics()
model = genmodel('test', grammar)
ast = model.parse(text, semantics=semantics)
self.assertEqual(15, ast)
import functools
dotted = functools.partial(type('').join, '.')
dotted.__name__ = 'dotted'
grammar = '''
start::dotted = {number}+ $ ;
number = /\d+/ ;
'''
semantics = ModelBuilderSemantics(types=[dotted])
model = genmodel('test', grammar)
ast = model.parse(text, semantics=semantics)
self.assertEqual('5.4.3.2.1', ast)
def test_empty_match_token(self):
grammar = """
table = { row }+ ;
row = (cell1:cell "|" cell2:cell) "\n";
cell = /[a-z]+/ ;
"""
try:
genmodel("model", grammar)
self.fail('allowed empty token')
except FailedParse:
pass
def test_empty_match_token(self):
grammar = """
table = { row }+ ;
row = (cell1:cell "|" cell2:cell) "\n";
cell = /[a-z]+/ ;
"""
try:
genmodel("model", grammar)
self.fail('allowed empty token')
except FailedParse:
pass
def test_optional_sequence(self):
grammar = '''
start = '1' ['2' '3'] '4' $ ;
'''
model = genmodel("test", grammar)
ast = model.parse("1234", nameguard=False)
self.assertEquals(['1', '2', '3', '4'], ast)
grammar = '''
start = '1' foo:['2' '3'] '4' $ ;
'''
model = genmodel("test", grammar)
ast = model.parse("1234", nameguard=False)
self.assertEquals(['2', '3'], ast.foo)
def test_optional_sequence(self):
grammar = '''
start = '1' ['2' '3'] '4' $ ;
'''
model = genmodel("test", grammar)
ast = model.parse("1234", nameguard=False)
self.assertEquals(['1', '2', '3', '4'], ast)
grammar = '''
start = '1' foo:['2' '3'] '4' $ ;
'''
model = genmodel("test", grammar)
ast = model.parse("1234", nameguard=False)
self.assertEquals(['2', '3'], ast.foo)
def test_ast_assignment(self):
grammar = '''
n = @: {"a"}* $ ;
f = @+: {"a"}* $ ;
nn = @: {"a"}* @: {"b"}* $ ;
nf = @: {"a"}* @+: {"b"}* $ ;
fn = @+: {"a"}* @: {"b"}* $ ;
ff = @+: {"a"}* @+: {"b"}* $ ;
'''
model = genmodel("test", grammar)
def p(input, rule):
return model.parse(input, start=rule, whitespace='')
e = self.assertEqual
e([], p('', 'n'))
e(['a'], p('a', 'n'))
e(['a', 'a'], p('aa', 'n'))
e([[]], p('', 'f'))
e([['a']], p('a', 'f'))
e([['a', 'a']], p('aa', 'f'))
for r in ('nn', 'nf', 'fn', 'ff'):
e([[], []], p('', r))
e([['a'], []], p('a', r))
e([[], ['b']], p('b', r))
e([['a', 'a'], []], p('aa', r))
e([[], ['b', 'b']], p('bb', r))
e([['a', 'a'], ['b']], p('aab', r))
def test_group_ast(self):
grammar = '''
start = '1' ('2' '3') '4' $ ;
'''
model = genmodel("test", grammar)
ast = model.parse("1234", nameguard=False)
self.assertEquals(['1', '2', '3', '4'], ast)
def test_patterns_with_newlines(self):
grammar = '''
start
=
blanklines $
;
blanklines
=
blankline [blanklines]
;
blankline
=
/^[^\n]*\n?$/
;
blankline2 =
?/^[^\n]*\n?$/?
;
'''
model = genmodel("test", grammar)
ast = model.parse('\n\n')
self.assertEqual('', ustr(ast))
def test_group_ast(self):
grammar = '''
start = '1' ('2' '3') '4' $ ;
'''
model = genmodel("test", grammar)
ast = model.parse("1234", nameguard=False)
self.assertEquals(['1', '2', '3', '4'], ast)
def test_numbers_and_unicode(self):
grammar = '''
rúle(1, -23, 4.56, 7.89e-11, 0xABCDEF, Añez)
=
'a'
;
'''
rule2 = '''
rulé::Añez
=
'\\xf1'
;
'''
rule3 = '''
rúlé::Añez
=
'ñ'
;
'''
if PY3:
grammar += rule3
else:
grammar += rule2
model = genmodel("test", grammar)
self.assertEquals(trim(grammar), ustr(model))
def test_stateful(self):
# Parser for mediawiki-style unordered lists.
grammar = r'''
document = @:ul [ nl ] $ ;
ul = "*" ul_start el+:li { nl el:li } * ul_end ;
li = ul | li_text ;
(* Quirk: If a text line is followed by a sublist, the sublist does not get its own li. *)
li_text = text:text [ ul:li_followed_by_ul ] ;
li_followed_by_ul = nl @:ul ;
text = ?/.*/? ;
nl = ?/\n/? ul_marker ;
(* The following rules are placeholders for state transitions. *)
ul_start = () ;
ul_end = () ;
(* The following rules are placeholders for state validations and grammar rules. *)
ul_marker = () ;
'''
class StatefulSemantics(object):
def __init__(self, parser):
self._context = parser
def ul_start(self, ast):
ctx = self._context
ctx._state = 1 if ctx._state is None else ctx._state + 1
return ast
def ul_end(self, ast):
ctx = self._context
ctx._state = None if ctx._state is None or ctx._state <= 1 else ctx._state - 1
return ast
def ul_marker(self, ast):
ctx = self._context
if ctx._state is not None:
if not ctx.buf.match("*" * ctx._state):
raise FailedSemantics("not at correct level")
return ast
def ul(self, ast):
return "<ul>" + "".join(ast.el) + "</ul>"
def li(self, ast):
return "<li>" + ast + "</li>"
def li_text(self, ast):
return ast.text if ast.ul is None else ast.text + ast.ul
model = genmodel("test", grammar)
context = ModelContext(model.rules, whitespace='', nameguard=False)
ast = model.parse('*abc', "document", context=context, semantics=StatefulSemantics(context), whitespace='', nameguard=False)
self.assertEqual(ast, "<ul><li>abc</li></ul>")
ast = model.parse('*abc\n', "document", context=context, semantics=StatefulSemantics(context), whitespace='', nameguard=False)
self.assertEqual("<ul><li>abc</li></ul>", ast)
ast = model.parse('*abc\n*def\n', "document", context=context, semantics=StatefulSemantics(context), whitespace='', nameguard=False)
self.assertEqual("<ul><li>abc</li><li>def</li></ul>", ast)
ast = model.parse('**abc', "document", context=context, semantics=StatefulSemantics(context), whitespace='', nameguard=False)
self.assertEqual("<ul><li><ul><li>abc</li></ul></li></ul>", ast)
ast = model.parse('*abc\n**def\n', "document", context=context, semantics=StatefulSemantics(context), whitespace='', nameguard=False)
self.assertEqual("<ul><li>abc<ul><li>def</li></ul></li></ul>", ast)
def test_patterns_with_newlines(self):
grammar = '''
start
=
blanklines $
;
blanklines
=
blankline [blanklines]
;
blankline
=
/^[^\n]*\n?$/
;
blankline2 =
?/^[^\n]*\n?$/?
;
'''
model = genmodel("test", grammar)
ast = model.parse('\n\n')
self.assertEqual('', ustr(ast))
def test_no_left_recursion(self):
grammar = '''
start
=
expre $
;
expre
=
expre '+' number
|
expre '*' number
|
number
;
number
=
?/[0-9]+/?
;
'''
model = genmodel("test", grammar)
model.parse("1*2+3*5")
try:
model.parse("1*2+3*5", left_recursion=False)
self.Fail('expected left recursion failure')
except FailedParse:
pass
def test_ast_assignment(self):
grammar = '''
n = @: {"a"}* $ ;
f = @+: {"a"}* $ ;
nn = @: {"a"}* @: {"b"}* $ ;
nf = @: {"a"}* @+: {"b"}* $ ;
fn = @+: {"a"}* @: {"b"}* $ ;
ff = @+: {"a"}* @+: {"b"}* $ ;
'''
model = genmodel("test", grammar)
def p(input, rule):
return model.parse(input, start=rule, whitespace='')
e = self.assertEqual
e([], p('', 'n'))
e(['a'], p('a', 'n'))
e(['a', 'a'], p('aa', 'n'))
e([[]], p('', 'f'))
e([['a']], p('a', 'f'))
e([['a', 'a']], p('aa', 'f'))
for r in ('nn', 'nf', 'fn', 'ff'):
e([[], []], p('', r))
e([['a'], []], p('a', r))
e([[], ['b']], p('b', r))
e([['a', 'a'], []], p('aa', r))
e([[], ['b', 'b']], p('bb', r))
e([['a', 'a'], ['b']], p('aab', r))
def test_no_left_recursion(self):
grammar = '''
start
=
expre $
;
expre
=
expre '+' number
|
expre '*' number
|
number
;
number
=
?/[0-9]+/?
;
'''
model = genmodel("test", grammar)
model.parse("1*2+3*5")
try:
model.parse("1*2+3*5", left_recursion=False)
self.fail('expected left recursion failure')
except FailedParse:
pass
def test_numbers_and_unicode(self):
grammar = '''
rúle(1, -23, 4.56, 7.89e-11, 0xABCDEF, Añez)
=
'a'
;
'''
rule2 = '''
rulé::Añez
=
'\\xf1'
;
'''
rule3 = '''
rúlé::Añez
=
'ñ'
;
'''
if PY3:
grammar += rule3
else:
grammar += rule2
model = genmodel("test", grammar)
self.assertEquals(trim(grammar), ustr(model))
def test_36_unichars(self):
grammar = '''
start = { rule_positional | rule_keywords | rule_all }* $ ;
rule_positional("ÄÖÜäöüß") = 'a' ;
rule_keywords(k1='äöüÄÖÜß') = 'b' ;
rule_all('ßÄÖÜäöü', k1="ßäöüÄÖÜ") = 'c' ;
'''
def _trydelete(pymodule):
import os
try:
os.unlink(pymodule + ".py")
except EnvironmentError:
pass
try:
os.unlink(pymodule + ".pyc")
except EnvironmentError:
pass
try:
os.unlink(pymodule + ".pyo")
except EnvironmentError:
pass
def assert_equal(target, value):
self.assertEqual(target, value)
class UnicharsSemantics(object):
"""Check all rule parameters for expected types and values"""
def rule_positional(self, ast, p1):
assert_equal("ÄÖÜäöüß", p1)
return ast
def rule_keyword(self, ast, k1):
assert_equal("äöüÄÖÜß", k1)
return ast
def rule_all(self, ast, p1, k1):
assert_equal("ßÄÖÜäöü", p1)
assert_equal("ßäöüÄÖÜ", k1)
return ast
m = genmodel("UnicodeRuleArguments", grammar)
ast = m.parse("a b c")
self.assertEqual(['a', 'b', 'c'], ast)
semantics = UnicharsSemantics()
ast = m.parse("a b c", semantics=semantics)
self.assertEqual(['a', 'b', 'c'], ast)
code = codegen(m)
import codecs
with codecs.open("tc36unicharstest.py", "w", "utf-8") as f:
f.write(code)
import tc36unicharstest
tc36unicharstest
_trydelete("tc36unicharstest")
def test_36_unichars(self):
grammar = '''
start = { rule_positional | rule_keywords | rule_all }* $ ;
rule_positional("ÄÖÜäöüß") = 'a' ;
rule_keywords(k1='äöüÄÖÜß') = 'b' ;
rule_all('ßÄÖÜäöü', k1="ßäöüÄÖÜ") = 'c' ;
'''
def _trydelete(pymodule):
import os
try:
os.unlink(pymodule + ".py")
except EnvironmentError:
pass
try:
os.unlink(pymodule + ".pyc")
except EnvironmentError:
pass
try:
os.unlink(pymodule + ".pyo")
except EnvironmentError:
pass
def assert_equal(target, value):
self.assertEqual(target, value)
class UnicharsSemantics(object):
"""Check all rule parameters for expected types and values"""
def rule_positional(self, ast, p1):
assert_equal("ÄÖÜäöüß", p1)
return ast
def rule_keyword(self, ast, k1):
assert_equal("äöüÄÖÜß", k1)
return ast
def rule_all(self, ast, p1, k1):
assert_equal("ßÄÖÜäöü", p1)
assert_equal("ßäöüÄÖÜ", k1)
return ast
m = genmodel("UnicodeRuleArguments", grammar)
ast = m.parse("a b c")
self.assertEqual(['a', 'b', 'c'], ast)
semantics = UnicharsSemantics()
ast = m.parse("a b c", semantics=semantics)
self.assertEqual(['a', 'b', 'c'], ast)
code = codegen(m)
import codecs
with codecs.open(TMP_DIR + "tc36unicharstest.py", "w", "utf-8") as f:
f.write(code)
import tc36unicharstest
tc36unicharstest
_trydelete("tc36unicharstest")
def test_update_ast(self):
grammar = '''
foo = name:"1" [ name: bar ] ;
bar = { "2" } * ;
'''
m = genmodel('Keywords', grammar)
ast = m.parse('1 2')
self.assertEqual(['1', ['2']], ast.name)
grammar = '''
start = items: { item } * $ ;
item = @:{ subitem } * "0" ;
subitem = ?/1+/? ;
'''
m = genmodel('Update', grammar)
ast = m.parse("1101110100", nameguard=False)
self.assertEquals([['11'], ['111'], ['1'], []], ast.items_)
def test_update_ast(self):
grammar = '''
foo = name:"1" [ name: bar ] ;
bar = { "2" } * ;
'''
m = genmodel('Keywords', grammar)
ast = m.parse('1 2')
self.assertEqual(['1', ['2']], ast.name)
grammar = '''
start = items: { item } * $ ;
item = @:{ subitem } * "0" ;
subitem = ?/1+/? ;
'''
m = genmodel('Update', grammar)
ast = m.parse("1101110100", nameguard=False)
self.assertEquals([['11'], ['111'], ['1'], []], ast.items_)
def test_empty_closure(self):
grammar = '''
start = {'x'}+ {} 'y'$;
'''
model = genmodel("test", grammar)
codegen(model)
ast = model.parse("xxxy", nameguard=False)
self.assertEquals([['x', 'x', 'x'], [], 'y'], ast)
def test_empty_closure(self):
grammar = '''
start = {'x'}+ {} 'y'$;
'''
model = genmodel("test", grammar)
codegen(model)
ast = model.parse("xxxy", nameguard=False)
self.assertEquals([['x', 'x', 'x'], [], 'y'], ast)
def test_36_params_and_keyword_params(self):
grammar = '''
rule(A, kwdB=B)
=
'a'
;
'''
model = genmodel("test", grammar)
self.assertEquals(trim(grammar), ustr(model))
def test_whitespace_directive(self):
grammar = '''
@@whitespace :: /[\t ]+/
test = "test" $;
'''
model = genmodel("test", grammar)
code = codegen(model)
compile(code, 'test.py', 'exec')
def test_36_params_and_keyword_params(self):
grammar = '''
rule(A, kwdB=B)
=
'a'
;
'''
model = genmodel("test", grammar)
self.assertEquals(trim(grammar), ustr(model))
def test_eol_comments_re_directive(self):
grammar = '''
@@eol_comments :: /#.*?$/
test = "test" $;
'''
model = genmodel("test", grammar)
code = codegen(model)
compile(code, 'test.py', 'exec')
def test_whitespace_directive(self):
grammar = '''
@@whitespace :: /[\t ]+/
test = "test" $;
'''
model = genmodel("test", grammar)
code = codegen(model)
compile(code, 'test.py', 'exec')
def test_eol_comments_re_directive(self):
grammar = '''
@@eol_comments :: /#.*?$/
test = "test" $;
'''
model = genmodel("test", grammar)
code = codegen(model)
compile(code, 'test.py', 'exec')
def test_indirect_left_recursion_with_cut(self):
grammar = '''
start = x $ ;
x = expr ;
expr = x '-' ~ num | num;
num = ?/[0-9]+/? ;
'''
model = genmodel("test", grammar)
ast = model.parse("5-87-32")
self.assertEquals(['5', '-', '87', '-', '32'], ast)
def test_rule_include(self):
grammar = '''
start = b $;
a = @:'a' ;
b = >a {@:'b'} ;
'''
model = genmodel("test", grammar)
ast = model.parse("abb", nameguard=False)
self.assertEquals(['a', 'b', 'b'], ast)
def test_define_keywords(self):
import parser
grammar = '''
@@keyword :: B C
@@keyword :: 'A'
start = ('a' 'b').{'x'} ;
'''
model = genmodel("test", grammar)
c = codegen(model)
parser.suite(c)
grammar2 = str(model)
model2 = genmodel("test", grammar2)
c2 = codegen(model2)
parser.suite(c2)
self.assertEqual(grammar2, str(model2))
def test_rule_include(self):
grammar = '''
start = b $;
a = @:'a' ;
b = >a {@:'b'} ;
'''
model = genmodel("test", grammar)
ast = model.parse("abb", nameguard=False)
self.assertEquals(['a', 'b', 'b'], ast)
def test_define_keywords(self):
import parser
grammar = '''
@@keyword :: B C
@@keyword :: 'A'
start = ('a' 'b').{'x'} ;
'''
model = genmodel("test", grammar)
c = codegen(model)
parser.suite(c)
grammar2 = str(model)
model2 = genmodel("test", grammar2)
c2 = codegen(model2)
parser.suite(c2)
self.assertEqual(grammar2, str(model2))
def test_indirect_left_recursion_with_cut(self):
grammar = '''
start = x $ ;
x = expr ;
expr = x '-' ~ num | num;
num = ?/[0-9]+/? ;
'''
model = genmodel("test", grammar)
ast = model.parse("5-87-32")
self.assertEquals(['5', '-', '87', '-', '32'], ast)
def test_optional_closure(self):
grammar = 'start = foo+:"x" foo:{"y"}* {foo:"z"}* ;'
model = genmodel("test", grammar)
ast = model.parse("xyyzz", nameguard=False)
self.assertEquals(['x', ['y', 'y'], 'z', 'z'], ast.foo)
grammar = 'start = foo+:"x" [foo+:{"y"}*] {foo:"z"}* ;'
model = genmodel("test", grammar)
ast = model.parse("xyyzz", nameguard=False)
self.assertEquals(['x', ['y', 'y'], 'z', 'z'], ast.foo)
grammar = 'start = foo+:"x" foo:[{"y"}*] {foo:"z"}* ;'
model = genmodel("test", grammar)
ast = model.parse("xyyzz", nameguard=False)
self.assertEquals(['x', ['y', 'y'], 'z', 'z'], ast.foo)
grammar = 'start = foo+:"x" [foo:{"y"}*] {foo:"z"}* ;'
model = genmodel("test", grammar)
ast = model.parse("xyyzz", nameguard=False)
self.assertEquals(['x', ['y', 'y'], 'z', 'z'], ast.foo)
def test_group_join(self):
grammar = '''
start = ('a' 'b').{'x'} ;
'''
model = genmodel("test", grammar)
c = codegen(model)
import parser
parser.suite(c)
ast = model.parse("x a b x", nameguard=False)
self.assertEquals(['x', 'x'], ast)
def test_new_override(self):
grammar = '''
start
=
@:'a' {@:'b'}
$
;
'''
model = genmodel("test", grammar)
ast = model.parse("abb", nameguard=False)
self.assertEquals(['a', 'b', 'b'], ast)
def test_group_join(self):
grammar = '''
start = ('a' 'b').{'x'} ;
'''
model = genmodel("test", grammar)
c = codegen(model)
import parser
parser.suite(c)
ast = model.parse("x a b x", nameguard=False)
self.assertEquals(['x', 'x'], ast)
def test_dot(self):
grammar = '''
start = "foo\\nbar" $;
'''
try:
from grako.diagrams import draw
except ImportError:
return
m = genmodel('Diagram', grammar)
draw('tmp/diagram.png', m)
def test_optional_closure(self):
grammar = 'start = foo+:"x" foo:{"y"}* {foo:"z"}* ;'
model = genmodel("test", grammar)
ast = model.parse("xyyzz", nameguard=False)
self.assertEquals(['x', ['y', 'y'], 'z', 'z'], ast.foo)
grammar = 'start = foo+:"x" [foo+:{"y"}*] {foo:"z"}* ;'
model = genmodel("test", grammar)
ast = model.parse("xyyzz", nameguard=False)
self.assertEquals(['x', ['y', 'y'], 'z', 'z'], ast.foo)
grammar = 'start = foo+:"x" foo:[{"y"}*] {foo:"z"}* ;'
model = genmodel("test", grammar)
ast = model.parse("xyyzz", nameguard=False)
self.assertEquals(['x', ['y', 'y'], 'z', 'z'], ast.foo)
grammar = 'start = foo+:"x" [foo:{"y"}*] {foo:"z"}* ;'
model = genmodel("test", grammar)
ast = model.parse("xyyzz", nameguard=False)
self.assertEquals(['x', ['y', 'y'], 'z', 'z'], ast.foo)
def test_new_override(self):
grammar = '''
start
=
@:'a' {@:'b'}
$
;
'''
model = genmodel("test", grammar)
ast = model.parse("abb", nameguard=False)
self.assertEquals(['a', 'b', 'b'], ast)
def test_dot(self):
grammar = '''
start = "foo\\nbar" $;
'''
try:
from grako.diagrams import draw
except ImportError:
return
m = genmodel('Diagram', grammar)
draw('tmp/diagram.png', m)
def test_48_rule_override(self):
grammar = '''
start = ab $;
ab = 'xyz' ;
@override
ab = @:'a' {@:'b'} ;
'''
model = genmodel("test", grammar)
ast = model.parse("abb", nameguard=False)
self.assertEquals(['a', 'b', 'b'], ast)
def test_left_recursion_directive(self):
grammar = '''
@@left_recursion :: False
test = "test" $;
'''
model = genmodel("test", grammar)
self.assertFalse(model.directives.get('left_recursion'))
self.assertFalse(model.left_recursion)
code = codegen(model)
compile(code, 'test.py', 'exec')
def test_left_recursion_directive(self):
grammar = '''
@@left_recursion :: False
test = "test" $;
'''
model = genmodel("test", grammar)
self.assertFalse(model.directives.get('left_recursion'))
self.assertFalse(model.left_recursion)
code = codegen(model)
compile(code, 'test.py', 'exec')
def test_48_rule_override(self):
grammar = '''
start = ab $;
ab = 'xyz' ;
@override
ab = @:'a' {@:'b'} ;
'''
model = genmodel("test", grammar)
ast = model.parse("abb", nameguard=False)
self.assertEquals(['a', 'b', 'b'], ast)
def test_keywords_in_rule_names(self):
grammar = '''
start
=
whitespace
;
whitespace
=
{'x'}+
;
'''
m = genmodel('Keywords', grammar)
m.parse('x')
def test_indirect_left_recursion_complex(self):
grammar = '''
start = Primary $ ;
Primary = PrimaryNoNewArray ;
PrimaryNoNewArray =
ClassInstanceCreationExpression
| MethodInvocation
| FieldAccess
| ArrayAccess
| 'this' ;
ClassInstanceCreationExpression =
'new' ClassOrInterfaceType '(' ')'
| Primary '.new' Identifier '()' ;
MethodInvocation =
Primary '.' MethodName '()'
| MethodName '()' ;
FieldAccess =
Primary '.' Identifier
| 'super.' Identifier ;
ArrayAccess =
Primary '[' Expression ']'
| ExpressionName '[' Expression ']' ;
ClassOrInterfaceType =
ClassName
| InterfaceTypeName ;
ClassName = 'C' | 'D' ;
InterfaceTypeName = 'I' | 'J' ;
Identifier = 'x' | 'y' | ClassOrInterfaceType ;
MethodName = 'm' | 'n' ;
ExpressionName = Identifier ;
Expression = 'i' | 'j' ;
'''
model = genmodel("test", grammar)
ast = model.parse("this")
self.assertEquals('this', ast)
ast = model.parse("this.x")
self.assertEquals(['this', '.', 'x'], ast)
ast = model.parse("this.x.y")
self.assertEquals(['this', '.', 'x', '.', 'y'], ast)
ast = model.parse("this.x.m()")
self.assertEquals(['this', '.', 'x', '.', 'm', '()'], ast)
ast = model.parse("x[i][j].y")
self.assertEquals(['x', '[', 'i', ']', '[', 'j', ']', '.', 'y'], ast)
def test_keywords_in_rule_names(self):
grammar = '''
start
=
whitespace
;
whitespace
=
{'x'}+
;
'''
m = genmodel('Keywords', grammar)
m.parse('x')
def test_indirect_left_recursion_complex(self):
grammar = '''
start = Primary $ ;
Primary = PrimaryNoNewArray ;
PrimaryNoNewArray =
ClassInstanceCreationExpression
| MethodInvocation
| FieldAccess
| ArrayAccess
| 'this' ;
ClassInstanceCreationExpression =
'new' ClassOrInterfaceType '(' ')'
| Primary '.new' Identifier '()' ;
MethodInvocation =
Primary '.' MethodName '()'
| MethodName '()' ;
FieldAccess =
Primary '.' Identifier
| 'super.' Identifier ;
ArrayAccess =
Primary '[' Expression ']'
| ExpressionName '[' Expression ']' ;
ClassOrInterfaceType =
ClassName
| InterfaceTypeName ;
ClassName = 'C' | 'D' ;
InterfaceTypeName = 'I' | 'J' ;
Identifier = 'x' | 'y' | ClassOrInterfaceType ;
MethodName = 'm' | 'n' ;
ExpressionName = Identifier ;
Expression = 'i' | 'j' ;
'''
model = genmodel("test", grammar)
ast = model.parse("this")
self.assertEquals('this', ast)
ast = model.parse("this.x")
self.assertEquals(['this', '.', 'x'], ast)
ast = model.parse("this.x.y")
self.assertEquals(['this', '.', 'x', '.', 'y'], ast)
ast = model.parse("this.x.m()")
self.assertEquals(['this', '.', 'x', '.', 'm', '()'], ast)
ast = model.parse("x[i][j].y")
self.assertEquals(['x', '[', 'i', ']', '[', 'j', ']', '.', 'y'], ast)
def test_join(self):
grammar = '''
start = ','.{'x' 'y'} ;
'''
grammar2 = '''
start = (','.{'x'}|{}) ;
'''
grammar3 = '''
start = [','.{'x'}] ;
'''
model = genmodel("test", grammar)
codegen(model)
ast = model.parse("x y, x y", nameguard=False)
self.assertEquals([['x', 'y'], ['x', 'y']], ast)
ast = model.parse("x y x y", nameguard=False)
self.assertEquals([['x', 'y']], ast)
try:
ast = model.parse("y x", nameguard=False)
self.Fail('closure not positive')
except FailedParse:
pass
model = genmodel("test", grammar2)
ast = model.parse("y x", nameguard=False)
self.assertEquals([], ast)
ast = model.parse("x", nameguard=False)
self.assertEquals(['x'], ast)
ast = model.parse("x,x", nameguard=False)
self.assertEquals(['x', 'x'], ast)
model = genmodel("test", grammar3)
ast = model.parse("y x", nameguard=False)
self.assertEquals(None, ast)
def test_grammar_directive(self):
grammar = '''
@@grammar :: Test
start = test $;
test = "test";
'''
model = genmodel(grammar=grammar)
self.assertEquals('Test', model.directives.get('grammar'))
self.assertEquals('Test', model.name)
code = codegen(model)
module = compile(code, 'test.py', 'exec')
assert 'TestParser' in module.co_names
def test_nested_left_recursion(self):
grammar_a = '''
s = e $ ;
e = [e '+'] t ;
t = [t '*'] a ;
a = ?/[0-9]/? ;
'''
grammar_b = '''
s = e $ ;
e = [e '+'] a ;
a = n | p ;
n = ?/[0-9]/? ;
p = '(' @:e ')' ;
'''
model_a = genmodel("test", grammar_a)
model_b = genmodel("test", grammar_b)
ast = model_a.parse("1*2+3*4")
self.assertEquals(['1', '*', '2', '+', ['3', '*', '4']], ast)
ast = model_b.parse("(1+2)+(3+4)")
self.assertEquals(['1', '+', '2', '+', ['3', '+', '4']], ast)
ast = model_a.parse("1*2*3")
self.assertEquals(['1', '*', '2', '*', '3'], ast)
ast = model_b.parse("(((1+2)))")
self.assertEquals(['1', '+', '2'], ast)
def test_nested_left_recursion(self):
grammar_a = '''
s = e $ ;
e = [e '+'] t ;
t = [t '*'] a ;
a = ?/[0-9]/? ;
'''
grammar_b = '''
s = e $ ;
e = [e '+'] a ;
a = n | p ;
n = ?/[0-9]/? ;
p = '(' @:e ')' ;
'''
model_a = genmodel("test", grammar_a)
model_b = genmodel("test", grammar_b)
ast = model_a.parse("1*2+3*4")
self.assertEquals(['1', '*', '2', '+', ['3', '*', '4']], ast)
ast = model_b.parse("(1+2)+(3+4)")
self.assertEquals(['1', '+', '2', '+', ['3', '+', '4']], ast)
ast = model_a.parse("1*2*3")
self.assertEquals(['1', '*', '2', '*', '3'], ast)
ast = model_b.parse("(((1+2)))")
self.assertEquals(['1', '+', '2'], ast)
def test_grammar_directive(self):
grammar = '''
@@grammar :: Test
start = test $;
test = "test";
'''
model = genmodel(grammar=grammar)
self.assertEquals('Test', model.directives.get('grammar'))
self.assertEquals('Test', model.name)
code = codegen(model)
module = compile(code, 'test.py', 'exec')
assert 'TestParser' in module.co_names
def test_join(self):
grammar = '''
start = ','.{'x' 'y'} ;
'''
grammar2 = '''
start = (','.{'x'}|{}) ;
'''
grammar3 = '''
start = [','.{'x'}] ;
'''
model = genmodel("test", grammar)
codegen(model)
ast = model.parse("x y, x y", nameguard=False)
self.assertEquals([['x', 'y'], ['x', 'y']], ast)
ast = model.parse("x y x y", nameguard=False)
self.assertEquals([['x', 'y']], ast)
try:
ast = model.parse("y x", nameguard=False)
self.Fail('closure not positive')
except FailedParse:
pass
model = genmodel("test", grammar2)
ast = model.parse("y x", nameguard=False)
self.assertEquals([], ast)
ast = model.parse("x", nameguard=False)
self.assertEquals(['x'], ast)
ast = model.parse("x,x", nameguard=False)
self.assertEquals(['x', 'x'], ast)
model = genmodel("test", grammar3)
ast = model.parse("y x", nameguard=False)
self.assertEquals(None, ast)
def test_partial_choice(self):
grammar = '''
start
=
o:[o]
x:'A'
$
;
o
=
'A' a:'A'
|
'A' b:'B'
;
'''
model = genmodel("test", grammar)
ast = model.parse("A", nameguard=False)
self.assertEquals({'x': 'A', 'o': None}, ast)
def test_failed_ref(self):
grammar = """
final = object;
type = /[^\s=()]+/;
object = '('type')' '{' @:{pair} {',' @:{pair}}* [','] '}';
pair = key '=' value;
list = '('type')' '[' @:{object} {',' @:{object}}* [','] ']';
key = /[^\s=]+/;
value = @:(string|list|object|unset|boolean|number|null) [','];
null = '('type')' @:{ 'null' };
boolean = /(true|false)/;
unset = '<unset>';
string = '"' @:/[^"]*/ '"';
number = /-?[0-9]+/;
"""
model = genmodel("final", grammar)
codegen(model)
model.parse('(sometype){boolean = true}')
