def test_36_unichars(self):
grammar = '''
start = { rule_positional | rule_keywords | rule_all }* $ ;
rule_positional("ÄÖÜäöüß") = 'a' ;
rule_keywords(k1='äöüÄÖÜß') = 'b' ;
rule_all('ßÄÖÜäöü', k1="ßäöüÄÖÜ") = 'c' ;
'''
def _trydelete(pymodule):
import os
try:
os.unlink(pymodule + ".py")
except EnvironmentError:
pass
try:
os.unlink(pymodule + ".pyc")
except EnvironmentError:
pass
try:
os.unlink(pymodule + ".pyo")
except EnvironmentError:
pass
def assert_equal(target, value):
self.assertEqual(target, value)
class UnicharsSemantics(object):
"""Check all rule parameters for expected types and values"""
def rule_positional(self, ast, p1):
assert_equal("ÄÖÜäöüß", p1)
return ast
def rule_keyword(self, ast, k1):
assert_equal("äöüÄÖÜß", k1)
return ast
def rule_all(self, ast, p1, k1):
assert_equal("ßÄÖÜäöü", p1)
assert_equal("ßäöüÄÖÜ", k1)
return ast
m = genmodel("UnicodeRuleArguments", grammar)
ast = m.parse("a b c")
self.assertEqual(['a', 'b', 'c'], ast)
semantics = UnicharsSemantics()
ast = m.parse("a b c", semantics=semantics)
self.assertEqual(['a', 'b', 'c'], ast)
code = codegen(m)
import codecs
with codecs.open("tc36unicharstest.py", "w", "utf-8") as f:
f.write(code)
import tc36unicharstest
tc36unicharstest
_trydelete("tc36unicharstest")
def test_eol_comments_re_directive(self):
grammar = '''
@@eol_comments :: /#.*?$/
test = "test" $;
'''
model = genmodel("test", grammar)
code = codegen(model)
compile(code, 'test.py', 'exec')
def test_whitespace_directive(self):
grammar = '''
@@whitespace :: /[\t ]+/
test = "test" $;
'''
model = genmodel("test", grammar)
code = codegen(model)
compile(code, 'test.py', 'exec')
def test_failed_ref(self):
grammar = """
final = object;
type = /[^\s=()]+/;
object = '('type')' '{' @:{pair} {',' @:{pair}}* [','] '}';
pair = key '=' value;
list = '('type')' '[' @:{object} {',' @:{object}}* [','] ']';
key = /[^\s=]+/;
value = @:(string|list|object|unset|boolean|number|null) [','];
null = '('type')' @:{ 'null' };
boolean = /(true|false)/;
unset = '<unset>';
string = '"' @:/[^"]*/ '"';
number = /-?[0-9]+/;
"""
model = genmodel("final", grammar)
codegen(model)
model.parse('(sometype){boolean = true}')
def test_left_recursion_directive(self):
grammar = '''
@@left_recursion :: False
test = "test" $;
'''
model = genmodel("test", grammar)
self.assertFalse(model.directives.get('left_recursion'))
self.assertFalse(model.left_recursion)
code = codegen(model)
compile(code, 'test.py', 'exec')
def test_keyword_params(self):
grammar = '''
start(k1=1, k2=2)
=
{'a'} $
;
'''
g = GrakoGrammarGenerator('Keywords')
model = g.parse(grammar, trace=False)
code = codegen(model)
self.assertEquals('#!/usr/bin/env python', code.splitlines()[0])
pass
def test_bootstrap(self):
print()
if os.path.isfile('tmp/00.ast'):
shutil.rmtree('tmp')
if not os.path.isdir('tmp'):
os.mkdir('tmp')
print('-' * 20, 'phase 00 - parse using the bootstrap grammar')
with open('etc/grako.ebnf') as f:
text = str(f.read())
g = GrakoParser('GrakoBootstrap')
grammar0 = g.parse(text)
ast0 = json.dumps(grammar0, indent=2)
with open('tmp/00.ast', 'w') as f:
f.write(ast0)
print('-' * 20, 'phase 01 - parse with parser generator')
with open('etc/grako.ebnf') as f:
text = str(f.read())
g = GrakoGrammarGenerator('GrakoBootstrap')
g.parse(text, trace=False)
generated_grammar1 = str(g.ast['grammar'])
with open('tmp/01.ebnf', 'w') as f:
f.write(generated_grammar1)
print('-' * 20, 'phase 02 - parse previous output with the parser generator')
with open('tmp/01.ebnf', 'r') as f:
text = str(f.read())
g = GrakoGrammarGenerator('GrakoBootstrap')
g.parse(text, trace=False)
generated_grammar2 = str(g.ast['grammar'])
with open('tmp/02.ebnf', 'w') as f:
f.write(generated_grammar2)
self.assertEqual(generated_grammar2, generated_grammar1)
print('-' * 20, 'phase 03 - repeat')
with open('tmp/02.ebnf') as f:
text = f.read()
g = GrakoParser('GrakoBootstrap')
ast3 = g.parse(text)
with open('tmp/03.ast', 'w') as f:
f.write(json.dumps(ast3, indent=2))
print('-' * 20, 'phase 04 - repeat')
with open('tmp/02.ebnf') as f:
text = f.read()
g = GrakoGrammarGenerator('GrakoBootstrap')
g.parse(text)
parser = g.ast['grammar']
# pprint(parser.first_sets, indent=2, depth=3)
generated_grammar4 = str(parser)
with open('tmp/04.ebnf', 'w') as f:
f.write(generated_grammar4)
self.assertEqual(generated_grammar4, generated_grammar2)
print('-' * 20, 'phase 05 - parse using the grammar model')
with open('tmp/04.ebnf') as f:
text = f.read()
ast5 = parser.parse(text)
with open('tmp/05.ast', 'w') as f:
f.write(json.dumps(ast5, indent=2))
print('-' * 20, 'phase 06 - generate parser code')
gencode6 = codegen(parser)
with open('tmp/g06.py', 'w') as f:
f.write(gencode6)
print('-' * 20, 'phase 07 - import generated code')
from g06 import GrakoBootstrapParser as GenParser # @UnresolvedImport
print('-' * 20, 'phase 08 - compile using generated code')
parser = GenParser(trace=False)
result = parser.parse(
text,
'grammar',
comments_re=COMMENTS_RE,
eol_comments_re=EOL_COMMENTS_RE
)
self.assertEqual(result, parser.ast['grammar'])
ast8 = parser.ast['grammar']
json8 = json.dumps(ast8, indent=2)
open('tmp/08.ast', 'w').write(json8)
self.assertEqual(ast5, ast8)
print('-' * 20, 'phase 09 - Generate parser with semantics')
with open('etc/grako.ebnf') as f:
text = f.read()
parser = GrakoGrammarGenerator('GrakoBootstrap')
g9 = parser.parse(text)
generated_grammar9 = str(g9)
with open('tmp/09.ebnf', 'w') as f:
f.write(generated_grammar9)
self.assertEqual(generated_grammar9, generated_grammar1)
print('-' * 20, 'phase 10 - Parse with a model using a semantics')
g10 = g9.parse(
text,
start_rule='grammar',
semantics=GrakoSemantics('GrakoBootstrap')
)
generated_grammar10 = str(g10)
with open('tmp/10.ebnf', 'w') as f:
f.write(generated_grammar10)
gencode10 = codegen(g10)
with open('tmp/g10.py', 'w') as f:
f.write(gencode10)
print('-' * 20, 'phase 11 - Pickle the model and try again.')
with open('tmp/11.grako', 'wb') as f:
pickle.dump(g10, f, protocol=2)
with open('tmp/11.grako', 'rb') as f:
g11 = pickle.load(f)
r11 = g11.parse(
text,
start_rule='grammar',
semantics=GrakoSemantics('GrakoBootstrap')
)
with open('tmp/11.ebnf', 'w') as f:
f.write(str(g11))
gencode11 = codegen(r11)
with open('tmp/g11.py', 'w') as f:
f.write(gencode11)
print('-' * 20, 'phase 12 - Walker')
class PrintNameWalker(DepthFirstWalker):
def __init__(self):
self.walked = []
def walk_default(self, o, children):
self.walked.append(o.__class__.__name__)
v = PrintNameWalker()
v.walk(g11)
with open('tmp/12.txt', 'w') as f:
f.write('\n'.join(v.walked))
print('-' * 20, 'phase 13 - Graphics')
try:
from grako.diagrams import draw
except ImportError:
print('PyGraphViz not found!')
else:
draw('tmp/13.png', g11)
def test_36_param_combinations(self):
def assert_equal(target, value):
self.assertEqual(target, value)
class TC36Semantics(object):
"""Check all rule parameters for expected types and values"""
def rule_positional(self, ast, p1, p2, p3, p4):
assert_equal("ABC", p1)
assert_equal(123, p2)
assert_equal('=', p3)
assert_equal("+", p4)
return ast
def rule_keyword(self, ast, k1, k2, k3, k4):
assert_equal("ABC", k1)
assert_equal(123, k2)
assert_equal('=', k3)
assert_equal('+', k4)
return ast
def rule_all(self, ast, p1, p2, p3, p4, k1, k2, k3, k4):
assert_equal("DEF", p1)
assert_equal(456, p2)
assert_equal('=', p3)
assert_equal("+", p4)
assert_equal("HIJ", k1)
assert_equal(789, k2)
assert_equal('=', k3)
assert_equal('+', k4)
return ast
grammar = '''
start
=
{rule_positional | rule_keywords | rule_all} $
;
rule_positional('ABC', 123, '=', '+')
=
'a'
;
rule_keywords(k1=ABC, k3='=', k4='+', k2=123)
=
'b'
;
rule_all('DEF', 456, '=', '+', k1=HIJ, k3='=', k4='+', k2=789)
=
'c'
;
'''
pretty = '''
start
=
{rule_positional | rule_keywords | rule_all} $
;
rule_positional(ABC, 123, '=', '+')
=
'a'
;
rule_keywords(k1=ABC, k3='=', k4='+', k2=123)
=
'b'
;
rule_all(DEF, 456, '=', '+', k1=HIJ, k3='=', k4='+', k2=789)
=
'c'
;
'''
model = genmodel('RuleArguments', grammar)
self.assertEqual(trim(pretty), ustr(model))
model = genmodel('RuleArguments', pretty)
ast = model.parse("a b c")
self.assertEqual(['a', 'b', 'c'], ast)
semantics = TC36Semantics()
ast = model.parse("a b c", semantics=semantics)
self.assertEqual(['a', 'b', 'c'], ast)
codegen(model)
