def setUp(self):
self.arith = grammar.Grammar('S', ('+', '*', '(', ')', 'id'))
self.result = None
def get_result(v):
self.result = v
return v
def add(x, y):
t = x + y
return t
def mul(x, y):
t = x * y
return t
self.arith.add_rule('S', ['E', get_result])
self.arith.add_rule('E', ['E', '+', 'T', add])
self.arith.add_rule('E', ['T', lambda v: v])
self.arith.add_rule('T', ['T', '*', 'F', mul])
self.arith.add_rule('T', ['F', lambda v: v])
self.arith.add_rule('F', ['(', 'E', ')', lambda v: v])
self.arith.add_rule('F', ['id', lambda v: v])
self.action_table, self.goto_table, self.start_state = build_parsing_table(
self.arith, LR0(self.arith.START, 0, 0))
self.driver = Driver(self.action_table, self.goto_table,
self.start_state)
def test_kernel_collection_lalr(self):
collection = self.kernels_lalr
propagate_lookaheads(collection)
self.assertTrue(len(collection) == 7)
expecteds = [
(LR0(self.lrvalue.START, 0, 0), [self.lrvalue.EOF]),
(LR0(self.lrvalue.START, 0, 1), [self.lrvalue.EOF]),
(LR0('S', 0, 1), [self.lrvalue.EOF]),
(LR0('C', 0, 1), ['c', 'd', self.lrvalue.EOF]),
(LR0('C', 1, 1), ['c', 'd', self.lrvalue.EOF]),
(LR0('S', 0, 2), [self.lrvalue.EOF]),
(LR0('C', 0, 2), ['c', 'd', self.lrvalue.EOF]),
]
for kernels in collection:
self.assertTrue(len(kernels) == 1)
found = False
count = 0
for item_lookaheads in expecteds:
item, lookaheads = item_lookaheads
if item in kernels:
self.assertTrue(
set(kernels).pop().lookaheads == set(lookaheads))
found = True
del expecteds[count]
break
count += 1
self.assertTrue(found)
def setUp(self):
self.lrvalue = grammar.Grammar('S', ('c', 'd'))
self.lrvalue.add_rule('S', ['C', 'C'])
self.lrvalue.add_rule('C', ['c', 'C'])
self.lrvalue.add_rule('C', ['d'])
self.StartExtendedSymbol = grammar.Grammar.START
self.kernels_lalr, self.goto_table = generate_spontaneously_lookaheads(
self.lrvalue, LR0(self.lrvalue.START, 0, 0),
UserFriendlyMapping(False))
def test_build_tables_of_arithmetic_grammar(self):
action_table, goto_table, start_set = build_parsing_table(
self.arith, LR0(self.StartExtendedSymbol, 0, 0))
states = action_table.keys()
s5 = None #Shift to state number 5
s4 = None #Shift to state number 4
count = 0
for state in states:
if 'id' in action_table[state]:
s5 = action_table[state]['id'] if s5 == None else s5
s4 = action_table[state]['('] if s4 == None else s4
self.assertTrue(s5 == action_table[state]['id'])
self.assertTrue(s4 == action_table[state]['('])
count += 1
self.assertTrue(count == 4)
state_5, state_4 = s5._state_to_shift, s4._state_to_shift
self.assertTrue(action_table[state_5]['+'] == \
action_table[state_5]['*'] == \
action_table[state_5][')'] == \
action_table[state_5][self.arith.EOF])
self.assertTrue(len(action_table[state_5]) == 4)
count = 0
for state in states:
s = action_table[state]
try:
if s['+'] == s['*'] == s[')'] == s[self.arith.EOF]:
self.assertTrue(hasattr(s['+'], '_semantic_action'))
count += 1
except:
pass
self.assertTrue(count == 4)
count = 0
s7 = None
for state in states:
s = action_table[state]
try:
if s['+'] == s[')'] == s[self.arith.EOF]:
count += 1
if self.assertTrue(hasattr(s['*'], '_state_to_shift')):
s7 = s['*'] if s7 == None else s7
self.assertTrue(s7 == s['*'])
except:
pass
self.assertTrue(count == 6)
def setUp(self):
self.arith = grammar.Grammar(
'S', ('+', '*', '(', ')', 'id', 'var', '=', 'let'))
self.symbol_table = [dict()]
self.result = None
def get_result(x):
self.result = x
return x
def add(x, y):
t = x + y
return t
def mul(x, y):
t = x * y
return t
def set_var(lv, rv):
self.symbol_table[-1][lv] = rv
return rv
def get_var(rv):
for table in reversed(self.symbol_table):
if rv in table:
return table[rv]
raise KeyError(rv)
def push():
self.symbol_table.append(dict())
def pop(*others):
self.symbol_table.pop()
self.arith.add_rule('S', ['E', get_result])
self.arith.add_rule('E', ['E', '+', 'T', add])
self.arith.add_rule('E', ['T', lambda v: v])
self.arith.add_rule('T', ['T', '*', 'F', mul])
self.arith.add_rule('T', ['F', lambda v: v])
self.arith.add_rule('F', ['(', 'E', ')', lambda v: v])
self.arith.add_rule('F', ['id', lambda v: v])
self.arith.add_rule('F', ['var', '=', 'E', set_var])
self.arith.add_rule('F', ['var', get_var])
self.arith.add_rule('F',
['let', push, '(', 'E', ')', pop, lambda v: v])
self.action_table, self.goto_table, self.start_state = build_parsing_table(
self.arith, LR0(self.arith.START, 0, 0))
self.driver = Driver(self.action_table, self.goto_table,
self.start_state)
def test_equivalence_sets_lalr_and_sets_lr0(self):
states_lalr = frozenset([
frozenset([
LALR(self.StartExtendedSymbol, 0, 0),
]),
frozenset([
LALR(self.StartExtendedSymbol, 0, 1),
]),
frozenset([
LALR('S', 0, 1),
]),
frozenset([
LALR('C', 0, 1),
]),
frozenset([
LALR('C', 1, 1),
]),
frozenset([
LALR('S', 0, 2),
]),
frozenset([
LALR('C', 0, 2),
]),
])
states_lr0 = frozenset([
frozenset([
LR0(self.StartExtendedSymbol, 0, 0),
]),
frozenset([
LR0(self.StartExtendedSymbol, 0, 1),
]),
frozenset([
LR0('S', 0, 1),
]),
frozenset([
LR0('C', 0, 1),
]),
frozenset([
LR0('C', 1, 1),
]),
frozenset([
LR0('S', 0, 2),
]),
frozenset([
LR0('C', 0, 2),
]),
])
self.assertTrue(states_lr0 == states_lalr)
def setUp(self):
self.arith = grammar.Grammar('S', ('(', ')', 'id', 'let'))
self.symbol_table = [dict()]
def push(*args):
pass
def pop(*args):
pass
self.arith.add_rule('S', ['E'])
self.arith.add_rule('E', ['id'])
self.arith.add_rule(
'E', ['let', push, '(', 'E', ')', pop, lambda *args: args])
self.action_table, self.goto_table, self.start_state = build_parsing_table(
self.arith, LR0(self.arith.START, 0, 0), disable_mapping=True)
self.driver = Driver(self.action_table, dict(self.goto_table),
self.start_state)
def setUp(self):
syn = syntax.Syntax('input')
self.result = []
syn.terminal('NL', None)
syn.terminal('NUM', None)
syn.terminal('+', None)
syn.terminal('-', None)
syn.repeat(('line', ), 'input')
syn.choice((('NL', ), ('expr', 'NL', lambda x: self.result.append(x))),
'line')
syn.choice((('NUM', ), ('expr', 'expr', '+', lambda x, y: x + y),
('expr', 'expr', '-', lambda x, y: x - y)), 'expr')
self.grammar = syn.as_grammar()
self.start_item = LR0(self.grammar.START, 0, 0)
self.action_table, self.goto_table, self.start_state = build_parsing_table(
self.grammar, self.start_item, disable_mapping=True)
self.driver = Driver(self.action_table, self.goto_table,
self.start_state)
self.kernel_states = [
frozenset([
LR0(self.grammar.START, 0, 0),
]),
frozenset([
LR0(self.grammar.START, 0, 1),
]),
frozenset([
LR0('input', 0, 1),
LR0('input', 1, 1),
]),
frozenset([
LR0('line', 0, 1),
]),
frozenset([
LR0('line', 1, 1),
LR0('expr', 1, 1),
LR0('expr', 2, 1),
]),
frozenset([
LR0('expr', 0, 1),
]),
frozenset([
LR0('input', 0, 2),
]),
frozenset([
LR0('line', 1, 2),
]),
frozenset([
LR0('expr', 1, 1),
LR0('expr', 1, 2),
LR0('expr', 2, 1),
LR0('expr', 2, 2),
]),
frozenset([
LR0('expr', 1, 3),
]),
frozenset([
LR0('expr', 2, 3),
])
]
def test_closures_ofLRValue(self):
a1 = set([
LR0(self.StartExtendedSymbol, 0, 0),
LR0('S', 0, 0),
LR0('S', 1, 0),
LR0('R', 0, 0),
LR0('R', 1, 0),
LR0('L', 0, 0),
LR0('L', 1, 0)
])
a2 = set([
LR0('S', 0, 0),
LR0('S', 1, 0),
LR0('R', 0, 0),
LR0('R', 1, 0),
LR0('R', 1, 1),
LR0('L', 0, 0),
LR0('L', 0, 1),
LR0('L', 1, 0)
])
self.assertTrue(a1 == closure(
set([LR0(self.StartExtendedSymbol, 0, 0)]), self.lrvalue))
self.assertTrue(
a2 == closure(set([LR0('L', 0, 1), LR0('R', 1, 1)]), self.lrvalue))
def test_closures_ofArith(self):
a1 = set([
LR0(self.StartExtendedSymbol, 0, 0),
LR0('E', 0, 0),
LR0('E', 1, 0),
LR0('T', 0, 0),
LR0('T', 1, 0),
LR0('F', 0, 0),
LR0('F', 1, 0)
])
a2 = set([
LR0('F', 0, 1),
LR0('E', 0, 0),
LR0('E', 1, 0),
LR0('T', 0, 0),
LR0('T', 1, 0),
LR0('F', 0, 0),
LR0('F', 1, 0)
])
self.assertTrue(a1 == closure(
set([LR0(self.StartExtendedSymbol, 0, 0)]), self.arith))
self.assertTrue(a2 == closure(set([LR0('F', 0, 1)]), self.arith))
def test_closures_ofLRValue_with_actions(self):
a1 = set([
LR0(self.StartExtendedSymbol, 0, 0),
LR0('S', 0, 0),
LR0('S', 1, 0),
LR0('R', 0, 0),
LR0('R', 1, 0),
LR0('L', 0, 0),
LR0('L', 1, 0)
])
a2 = set([
LR0(self.lrvalue_with_actions.ACTION_INSIDE % (3, '<lambda>'), 0,
0),
LR0('R', 0, 0),
LR0('R', 1, 0),
LR0('R', 1, 1),
LR0('L', 0, 0),
LR0('L', 0, 1),
LR0('L', 1, 0)
])
self.assertTrue(
a1 == closure(set([LR0(self.StartExtendedSymbol, 0, 0)]),
self.lrvalue_with_actions))
self.assertTrue(a2 == closure(set([LR0(
'L', 0, 1), LR0('R', 1, 1)]), self.lrvalue_with_actions))
def test_closures_ofSome(self):
a1 = set([
LR0(self.StartExtendedSymbol, 0, 0),
LR0('S', 0, 0),
LR0('C', 0, 0),
LR0('C', 1, 0)
])
a2 = set(
[LR0('S', 0, 1),
LR0('C', 0, 0),
LR0('C', 1, 0),
LR0('C', 1, 1)])
self.assertTrue(a1 == closure(
set([LR0(self.StartExtendedSymbol, 0, 0)]), self.some))
self.assertTrue(
a2 == closure(set([LR0('S', 0, 1), LR0('C', 1, 1)]), self.some))
def test_build_tables(self):
action_table, goto_table, start_set = build_parsing_table(
self.arith, LR0(self.StartExtendedSymbol, 0, 0))
self.assertTrue(len(action_table) == 12)
def test_goto_arith(self):
a1 = set([
LR0('E', 0, 2),
LR0('T', 0, 0),
LR0('T', 1, 0),
LR0('F', 0, 0),
LR0('F', 1, 0),
])
a2 = set([
LR0('E', 0, 1),
])
a3 = set([
LR0('T', 0, 3),
])
self.assertTrue(a1 == goto(
set([LR0(self.StartExtendedSymbol, 0, 1),
LR0('E', 0, 1)]), '+', self.arith))
self.assertTrue(a2 == goto(set([LR0('E', 0, 0)]), 'E', self.arith))
self.assertTrue(
a3 == goto(set([
LR0('T', 0, 2),
LR0('F', 0, 0),
LR0('F', 1, 0),
]), 'F', self.arith))
def setUp(self):
self.lrvalue = grammar.Grammar(None, ('=', '*', ';', 'id', 'num'))
self.symbol_table = dict()
self.last_value = None
def set_var(lv, rv):
self.symbol_table[lv] = rv;
return self.symbol_table[lv]
def get_var(rv):
if rv in self.symbol_table:
return self.symbol_table[rv]
raise KeyError(rv)
def grab_last_value(v):
self.last_value = v if isinstance(v, int) else self.symbol_table[v]
return v
self.lrvalue.augment('S')
self.lrvalue.add_rule('S', ['E', ';', 'S', lambda e, s: s])
self.lrvalue.add_rule('S', ['E', ';', lambda v: v])
self.lrvalue.add_rule('E', ['L', '=', 'R', set_var])
self.lrvalue.add_rule('E', ['R', grab_last_value])
self.lrvalue.add_rule('L', ['*', 'R', get_var])
self.lrvalue.add_rule('L', ['id', lambda v: v])
self.lrvalue.add_rule('R', ['L', lambda v: v])
self.lrvalue.add_rule('R', ['num', lambda v: v])
self.action_table, self.goto_table, self.start_state = build_parsing_table_lalr(self.lrvalue, LR0(self.lrvalue.START, 0, 0), False)
self.driver = Driver(self.action_table, self.goto_table, self.start_state)
def test_reduce_reduce_conflict(self):
self.assertRaisesRegexp(ReduceReduce, "during process '#' terminal",
build_parsing_table, self.reduce_reduce,
LR0(self.StartExtendedSymbol, 0, 0), False)
def probe(*args):
print "--", len(args), "-", " ".join(map(str, args))
def prompt(*args):
print ">>> ",
grammar = from_string(rpcalc_grammar,
'input',
print_result=print_result,
probe=probe,
prompt=prompt,
up=lambda x: x,
add=lambda x, y: x + y,
sub=lambda x, y: x - y,
mul=lambda x, y: x * y,
div=lambda x, y: x / y,
mod=lambda x, y: x % y,
neg=lambda x: -x)
states, gotos, first = build_parsing_table(grammar, LR0(grammar.START, 0, 0))
driver = Driver(states, gotos, first)
print "Reverse polish calculator. Write expressions like "
print " >>> 2 + 3 (result in 2 + 3 = 5) or "
print " >>> 2 + 4 * 5 (result in 2 + (4 * 5) = 22)"
print
prompt()
driver.parse(CalcLexer(sys.stdin))
def test_action_table(self):
states_shifts_reduce_actions = [
(frozenset([
LR0(self.arith.START, 0, 0),
LR0('S', 0, 0),
LR0('E', 0, 0),
LR0('E', 1, 0),
]), (('id', 2), ('let', 4)), ()),
(frozenset([
LR0(self.arith.START, 0, 1),
]), (), ()),
(frozenset([
LR0('E', 0, 1),
]), (), ()),
(frozenset([
LR0('S', 0, 1),
]), (), ()),
(frozenset([
LR0('E', 1, 1),
LR0(self.arith.ACTION_INSIDE % (1, 'push'), 0, 0),
]), (), ()),
(frozenset([
LR0('E', 1, 2),
]), (('(', 9), ), ()),
(frozenset([
LR0('E', 1, 4),
]), ((')', 8), ), ()),
(frozenset([
LR0('E', 1, 6),
]), (), ()),
(frozenset([
LR0('E', 1, 5),
LR0(self.arith.ACTION_INSIDE % (2, 'pop'), 0, 0),
]), (), ()),
(frozenset([
LR0('E', 1, 3),
LR0('E', 0, 0),
LR0('E', 1, 0),
]), (('id', 2), ('let', 4)), ()),
]
self.assertTrue(
len(states_shifts_reduce_actions) == len(self.action_table.keys()))
for state, shifts, reduces in states_shifts_reduce_actions:
h = hash(state)
self.assertTrue(
len(shifts) == len(
filter(lambda action: "Shift" in str(action),
self.action_table[h].values())))
if not shifts:
continue
keys, ids = zip(*shifts)
found_ids = [
i for i in range(len(states_shifts_reduce_actions))
if ("Shift %s" % hash(states_shifts_reduce_actions[i][0])
) in map(lambda a: str(a), self.action_table[h].values())
]
found_keys = self.action_table[h].keys()
self.assertTrue(frozenset(ids) == frozenset(found_ids))
self.assertTrue(frozenset(keys) == frozenset(found_keys))
def test_canonical_collection_lr_value(self):
collection = canonical_collection(self.lrvalue,
LR0(self.StartExtendedSymbol, 0, 0))
self.assertTrue(len(collection) == 10)
states = frozenset([
frozenset([
LR0(self.StartExtendedSymbol, 0, 0),
LR0('S', 0, 0),
LR0('S', 1, 0),
LR0('L', 0, 0),
LR0('L', 1, 0),
LR0('R', 0, 0),
]),
frozenset([
LR0(self.StartExtendedSymbol, 0, 1),
]),
frozenset([
LR0('S', 0, 1),
LR0('R', 0, 1),
]),
frozenset([
LR0('S', 1, 1),
]),
frozenset([
LR0('L', 0, 1),
LR0('L', 0, 0),
LR0('L', 1, 0),
LR0('R', 0, 0),
]),
frozenset([
LR0('L', 1, 1),
]),
frozenset([
LR0('S', 0, 2),
LR0('L', 0, 0),
LR0('L', 1, 0),
LR0('R', 0, 0),
]),
frozenset([
LR0('L', 0, 2),
]),
frozenset([
LR0('R', 0, 1),
]),
frozenset([
LR0('S', 0, 3),
]),
])
self.assertTrue(states == collection)
def test_cannonical_collection(self):
collection = canonical_collection(self.arith,
LR0(self.arith.START, 0, 0))
states = frozenset([
frozenset([
LR0(self.arith.START, 0, 0),
LR0('S', 0, 0),
LR0('E', 0, 0),
LR0('E', 1, 0),
]),
frozenset([
LR0(self.arith.START, 0, 1),
]),
frozenset([
LR0('E', 0, 1),
]),
frozenset([
LR0('S', 0, 1),
]),
frozenset([
LR0('E', 1, 1),
LR0(self.arith.ACTION_INSIDE % (1, 'push'), 0, 0),
]),
frozenset([
LR0('E', 1, 2),
]),
frozenset([
LR0('E', 1, 4),
]),
frozenset([
LR0('E', 1, 6),
]),
frozenset([
LR0('E', 1, 5),
LR0(self.arith.ACTION_INSIDE % (2, 'pop'), 0, 0),
]),
frozenset([
LR0('E', 1, 3),
LR0('E', 0, 0),
LR0('E', 1, 0),
]),
])
self.assertTrue(states == collection)
def test_canonical_collection_arith(self):
collection = canonical_collection(self.arith,
LR0(self.StartExtendedSymbol, 0, 0))
self.assertTrue(len(collection) == 12)
states = frozenset([
frozenset([
LR0(self.StartExtendedSymbol, 0, 0),
LR0('E', 0, 0),
LR0('E', 1, 0),
LR0('T', 0, 0),
LR0('T', 1, 0),
LR0('F', 0, 0),
LR0('F', 1, 0),
]),
frozenset([
LR0(self.StartExtendedSymbol, 0, 1),
LR0('E', 0, 1),
]),
frozenset([
LR0('E', 1, 1),
LR0('T', 0, 1),
]),
frozenset([
LR0('F', 1, 1),
]),
frozenset([
LR0('F', 0, 1),
LR0('E', 0, 0),
LR0('E', 1, 0),
LR0('T', 0, 0),
LR0('T', 1, 0),
LR0('F', 0, 0),
LR0('F', 1, 0),
]),
frozenset([
LR0('T', 1, 1),
]),
frozenset([
LR0('E', 0, 2),
LR0('T', 0, 0),
LR0('T', 1, 0),
LR0('F', 0, 0),
LR0('F', 1, 0),
]),
frozenset([
LR0('T', 0, 2),
LR0('F', 0, 0),
LR0('F', 1, 0),
]),
frozenset([
LR0('E', 0, 1),
LR0('F', 0, 2),
]),
frozenset([
LR0('E', 0, 3),
LR0('T', 0, 1),
]),
frozenset([
LR0('T', 0, 3),
]),
frozenset([
LR0('F', 0, 3),
]),
])
self.assertTrue(states == collection)
def test_goto_table(self):
states_gotos = [
(frozenset([
LR0(self.arith.START, 0, 0),
LR0('S', 0, 0),
LR0('E', 0, 0),
LR0('E', 1, 0),
]), (('S', 1), ('id', 2), ('E', 3), ('let', 4))),
(frozenset([
LR0(self.arith.START, 0, 1),
]), ()),
(frozenset([
LR0('E', 0, 1),
]), ()),
(frozenset([
LR0('S', 0, 1),
]), ()),
(frozenset([
LR0('E', 1, 1),
LR0(self.arith.ACTION_INSIDE % (1, 'push'), 0, 0),
]), ((self.arith.ACTION_INSIDE % (1, 'push'), 5), )),
(frozenset([
LR0('E', 1, 2),
]), (('(', 9), )),
(frozenset([
LR0('E', 1, 4),
]), ((')', 8), )),
(frozenset([
LR0('E', 1, 6),
]), ()),
(frozenset([
LR0('E', 1, 5),
LR0(self.arith.ACTION_INSIDE % (2, 'pop'), 0, 0),
]), ((self.arith.ACTION_INSIDE % (2, 'pop'), 7), )),
(frozenset([
LR0('E', 1, 3),
LR0('E', 0, 0),
LR0('E', 1, 0),
]), (('E', 6), ('id', 2), ('let', 4))),
]
checked = 0
for state, gotos in states_gotos:
h = hash(state)
if not gotos:
self.assertTrue(h not in self.goto_table)
continue
checked += 1
expected_keys, expected_states_id = zip(*gotos)
found_gotos_keys = frozenset(self.goto_table[h].keys())
found_gotos_hashs = frozenset(self.goto_table[h].values())
self.assertTrue(frozenset(expected_keys) == found_gotos_keys)
self.assertTrue(
frozenset(
[hash(states_gotos[i][0])
for i in expected_states_id]) == found_gotos_hashs)
self.assertTrue(checked == len(self.goto_table.keys()))