def test_redefine_non_terminal(self):
grammar = Grammar()
grammar.add_terminal('a')
self.assertTrue(grammar.is_terminal('a'))
grammar.add_production('b', ['a', 'a'])
with self.assertRaises(ParserGenerationException):
grammar.add_terminal('b')
def test_print(self, mock_stdout):
grammar = Grammar()
grammar.add_terminal('a')
grammar.add_production('b', ['a', 'a'])
print_grammar(grammar)
grammar.dump()
self.assertTrue(mock_stdout.getvalue())
def test_dump_parse(self, mock_stdout):
""" Test the debug dump of the parser """
g = Grammar()
g.add_terminals(['a', 'b', 'c'])
g.add_production('goal', ['a', 'c', 'b'])
g.start_symbol = 'goal'
p = EarleyParser(g)
tokens = gen_tokens(['a', 'c', 'b'])
p.parse(tokens, debug_dump=True)
def test_empty(self):
""" Test empty token stream """
g = Grammar()
g.add_terminals([','])
g.add_production('input', [','])
g.start_symbol = 'input'
p = LrParserBuilder(g).generate_parser()
tokens = gen_tokens([])
with self.assertRaises(ParserException):
p.parse(tokens)
def setUp(self):
g = Grammar()
g.add_terminals(['(', ')'])
g.add_production('goal', ['list'])
g.add_production('list', ['list', 'pair'])
g.add_production('list', ['pair'])
g.add_production('pair', ['(', 'pair', ')'])
g.add_production('pair', ['(', ')'])
g.start_symbol = 'goal'
self.g = g
def test_undefined_terminal(self):
""" Test correct behavior when a terminal is undefined """
g = Grammar()
g.add_terminals(['b'])
g.add_production('goal', ['a'])
g.add_production('a', ['b'])
g.add_production('a', ['c'])
g.start_symbol = 'goal'
with self.assertRaises(ParserGenerationException):
LrParserBuilder(g).generate_parser()
def test_invalid_parse(self):
""" Check what happens when """
g = Grammar()
g.add_terminals(['a', 'b', 'c'])
g.add_production('goal', ['a', 'c', 'b'])
g.start_symbol = 'goal'
p = EarleyParser(g)
tokens = gen_tokens(['d'])
with self.assertRaises(CompilerError):
p.parse(tokens)
def test_eps_sequence(self):
""" Test epsilon terminal for use in sequences """
g = Grammar()
g.add_terminals(['a'])
g.add_production('aas', [])
g.add_production('aas', ['aas', 'a'])
g.start_symbol = 'aas'
p = LrParserBuilder(g).generate_parser()
tokens = gen_tokens(['a', 'a', 'a'])
p.parse(tokens)
tokens = gen_tokens([])
p.parse(tokens)
def test_cb(self):
""" Test callback of one rule and order or parameters """
self.cb_called = False
def cb(a, c, b):
self.cb_called = True
self.assertEqual(a.val, 'a')
self.assertEqual(b.val, 'b')
self.assertEqual(c.val, 'c')
g = Grammar()
g.add_terminals(['a', 'b', 'c'])
g.add_production('goal', ['a', 'c', 'b'], cb)
g.start_symbol = 'goal'
p = EarleyParser(g)
tokens = gen_tokens(['a', 'c', 'b'])
p.parse(tokens)
self.assertTrue(self.cb_called)
def test_shift_reduce_conflict(self):
""" Must be handled automatically by doing shift """
g = Grammar()
g.add_terminals([EOF, 'if', 'then', 'else', 'ass'])
# Ambiguous grammar:
g.add_production('if_stmt', ['if', 'then', 'stmt'])
g.add_production('if_stmt', ['if', 'then', 'stmt', 'else', 'stmt'])
g.add_production('stmt', ['if_stmt'])
g.add_production('stmt', ['ass'])
g.start_symbol = 'stmt'
p = LrParserBuilder(g).generate_parser()
# Ambiguous program:
tokens = gen_tokens(['if', 'then', 'if', 'then', 'ass', 'else', 'ass'])
p.parse(tokens)
def test_eps2(self):
g = Grammar()
g.add_terminals(['id', ':'])
g.add_production('input', ['opt_lab', 'ins', 'op1'])
g.add_production('input', ['ins', 'op1'])
g.add_production('opt_lab', ['id', ':'])
g.add_production('ins', ['id'])
g.add_production('op1', ['id'])
g.start_symbol = 'input'
p = LrParserBuilder(g).generate_parser()
tokens = gen_tokens(['id', ':', 'id', 'id']) # i.e. "lab_0: inc rax"
p.parse(tokens)
tokens = gen_tokens(['id', 'id']) # i.e. "inc rax"
p.parse(tokens)
def test_redefine_terminal(self):
""" Test correct behavior when a terminal is redefined """
g = Grammar()
g.add_terminals([EOF, 'b', 'c'])
g.add_production('goal', ['a'])
with self.assertRaises(ParserGenerationException):
g.add_production('b', ['c']) # Not allowed
g.add_production('a', ['c'])
g.start_symbol = 'goal'
LrParserBuilder(g).generate_parser()
def test_eps(self):
""" Test epsilon terminal """
g = Grammar()
g.add_terminals(['a', 'b'])
g.add_production('input', ['optional_a', 'b'])
g.add_production('optional_a', ['a'])
g.add_production('optional_a', [])
g.start_symbol = 'input'
p = LrParserBuilder(g).generate_parser()
tokens = gen_tokens(['b'])
p.parse(tokens)
def test_reduce_reduce_conflict(self):
""" Check if a reduce-reduce conflict is detected """
# Define a grammar with an obvious reduce-reduce conflict:
g = Grammar()
g.add_terminals(['id'])
g.add_production('goal', ['a'])
g.add_production('a', ['b'])
g.add_production('a', ['c'])
g.add_production('b', ['id'])
g.add_production('c', ['id'])
g.start_symbol = 'goal'
with self.assertRaises(ParserGenerationException):
LrParserBuilder(g).generate_parser()
def test_rewrite_epsilons(self):
""" Test grammar rewriting. This involves the removal of epsilon
rules. """
grammar = Grammar()
grammar.add_terminals(['a', 'b', 'c'])
grammar.add_production('X', [])
grammar.add_production('X', ['c'])
grammar.add_production('Y', ['X', 'a'])
self.assertFalse(grammar.is_normal)
self.assertEqual(1, len(grammar.productions_for_name('Y')))
grammar.rewrite_eps_productions()
self.assertEqual(2, len(grammar.productions_for_name('Y')))
self.assertTrue(grammar.is_normal)
def test_ambiguous_grammar(self):
"""
Test if ambiguous grammar is handled correctly by priorities
"""
# TODO: check that ambiguous grammars have different priorities!
grammar = Grammar()
grammar.add_terminals(['mov', 'num', '+'])
grammar.add_production('expr', ['num', '+', 'num'],
lambda rh1, _, rh3: rh1.val + rh3.val,
priority=3)
grammar.add_production('expr', ['num', '+', 'num'],
lambda rh1, _, rh3: rh1.val + rh3.val + 1,
priority=2)
grammar.add_production('ins', ['mov', 'expr'],
lambda _, rh2: rh2,
priority=2)
grammar.start_symbol = 'ins'
parser = EarleyParser(grammar)
result = parser.parse(gen_tokens(['mov', ('num', 1), '+', ('num', 1)]))
self.assertEqual(3, result)
def __init__(self, kws, sigfilter, opt_timescale):
toks = ['ID', 'NUMBER', 'STRING', 'TIMEUNIT', EPS, EOF] + kws
g = Grammar()
g.add_terminals(toks)
g.add_production('input', ['exp_list'])
g.add_one_or_more('exp', 'exp_list')
g.add_production('exp', ['$SCOPE', 'scopetype', 'ID', '$END'],
self.handle_start_module)
g.add_production('scopetype', ['MODULE'], lambda l: l)
g.add_production('scopetype', ['TASK'], lambda l: l)
g.add_production('scopetype', ['BEGIN'], lambda l: l)
g.add_production('exp', ['$UPSCOPE', '$END'], self.handle_end_module)
g.add_production('exp', ['$TIMESCALE', 'TIMEUNIT', '$END'],
self.handle_timescale)
g.add_production('exp',
['$VAR', 'type', 'NUMBER', 'code', 'name', '$END'],
self.handle_vardecl)
g.add_production('code', ['ID'], lambda l: l)
g.add_production('code', ['NUMBER'], self.handle_number)
g.add_production('name', ['ID'], lambda l: l)
g.add_production('name', ['ID', 'ID'], self.handle_index)
g.add_production('type', ['WIRE'], lambda l: l)
g.add_production('type', ['REG'], lambda l: l)
g.add_production('type', ['INTEGER'], lambda l: l)
g.add_production('type', ['TRIREG'], lambda l: l)
g.start_symbol = 'input'
self.p = LrParserBuilder(g).generate_parser()
self.sigprefix = []
self.sigfilter = sigfilter
self.timescale = opt_timescale
def test_simple_grammar(self):
# 1. define a simple grammar:
g = Grammar()
g.add_terminals(['identifier', '(', ')', '+', '*'])
g.add_production('input', ['expression'])
g.add_production('expression', ['term'])
g.add_production('expression', ['expression', '+', 'term'])
g.add_production('term', ['factor'])
g.add_production('term', ['term', '*', 'factor'])
g.add_production('factor', ['(', 'expression', ')'])
g.add_production('factor', ['identifier'])
g.start_symbol = 'input'
# 2. define input:
tokens = gen_tokens(
['identifier', '+', 'identifier', '+', 'identifier'])
# 3. build parser:
p = LrParserBuilder(g).generate_parser()
# 4. feed input:
p.parse(tokens)
def test_expression_grammar(self):
grammar = Grammar()
grammar.add_terminals(['EOF', 'identifier', '(', ')', '+', '*', 'num'])
grammar.add_production('input', ['expression'], lambda rhs: rhs)
grammar.add_production('expression', ['term'], lambda rhs: rhs)
grammar.add_production('expression', ['expression', '+', 'term'],
lambda rh1, rh2, rh3: rh1 + rh3)
grammar.add_production('term', ['factor'], lambda rhs: rhs)
grammar.add_production('term', ['term', '*', 'factor'],
lambda rh1, rh2, rh3: rh1 * rh3)
grammar.add_production('factor', ['(', 'expression', ')'],
lambda rh1, rh2, rh3: rh2)
grammar.add_production('factor', ['identifier'])
grammar.add_production('factor', ['num'], lambda rhs: rhs.val)
grammar.start_symbol = 'input'
parser = EarleyParser(grammar)
result = parser.parse(
gen_tokens([('num', 7), '*', ('num', 11), '+', ('num', 3)]))
self.assertEqual(80, result)
def setUp(self):
g = Grammar()
g.add_terminals(['EOF', 'identifier', '(', ')', '+', '*', 'num'])
g.add_production('input', ['expression'])
g.add_production('expression', ['term'])
g.add_production('expression', ['expression', '+', 'term'])
g.add_production('term', ['factor'])
g.add_production('term', ['term', '*', 'factor'])
g.add_production('factor', ['(', 'expression', ')'])
g.add_production('factor', ['identifier'])
g.add_production('factor', ['num'])
g.start_symbol = 'input'
self.g = g