def setUp(self): self.arith = grammar.Grammar('S', ('+', '*', '(', ')', 'id')) self.result = None def get_result(v): self.result = v return v def add(x, y): t = x + y return t def mul(x, y): t = x * y return t self.arith.add_rule('S', ['E', get_result]) self.arith.add_rule('E', ['E', '+', 'T', add]) self.arith.add_rule('E', ['T', lambda v: v]) self.arith.add_rule('T', ['T', '*', 'F', mul]) self.arith.add_rule('T', ['F', lambda v: v]) self.arith.add_rule('F', ['(', 'E', ')', lambda v: v]) self.arith.add_rule('F', ['id', lambda v: v]) self.action_table, self.goto_table, self.start_state = build_parsing_table( self.arith, LR0(self.arith.START, 0, 0)) self.driver = Driver(self.action_table, self.goto_table, self.start_state)
def test_kernel_collection_lalr(self): collection = self.kernels_lalr propagate_lookaheads(collection) self.assertTrue(len(collection) == 7) expecteds = [ (LR0(self.lrvalue.START, 0, 0), [self.lrvalue.EOF]), (LR0(self.lrvalue.START, 0, 1), [self.lrvalue.EOF]), (LR0('S', 0, 1), [self.lrvalue.EOF]), (LR0('C', 0, 1), ['c', 'd', self.lrvalue.EOF]), (LR0('C', 1, 1), ['c', 'd', self.lrvalue.EOF]), (LR0('S', 0, 2), [self.lrvalue.EOF]), (LR0('C', 0, 2), ['c', 'd', self.lrvalue.EOF]), ] for kernels in collection: self.assertTrue(len(kernels) == 1) found = False count = 0 for item_lookaheads in expecteds: item, lookaheads = item_lookaheads if item in kernels: self.assertTrue( set(kernels).pop().lookaheads == set(lookaheads)) found = True del expecteds[count] break count += 1 self.assertTrue(found)
def setUp(self): self.lrvalue = grammar.Grammar('S', ('c', 'd')) self.lrvalue.add_rule('S', ['C', 'C']) self.lrvalue.add_rule('C', ['c', 'C']) self.lrvalue.add_rule('C', ['d']) self.StartExtendedSymbol = grammar.Grammar.START self.kernels_lalr, self.goto_table = generate_spontaneously_lookaheads( self.lrvalue, LR0(self.lrvalue.START, 0, 0), UserFriendlyMapping(False))
def test_build_tables_of_arithmetic_grammar(self): action_table, goto_table, start_set = build_parsing_table( self.arith, LR0(self.StartExtendedSymbol, 0, 0)) states = action_table.keys() s5 = None #Shift to state number 5 s4 = None #Shift to state number 4 count = 0 for state in states: if 'id' in action_table[state]: s5 = action_table[state]['id'] if s5 == None else s5 s4 = action_table[state]['('] if s4 == None else s4 self.assertTrue(s5 == action_table[state]['id']) self.assertTrue(s4 == action_table[state]['(']) count += 1 self.assertTrue(count == 4) state_5, state_4 = s5._state_to_shift, s4._state_to_shift self.assertTrue(action_table[state_5]['+'] == \ action_table[state_5]['*'] == \ action_table[state_5][')'] == \ action_table[state_5][self.arith.EOF]) self.assertTrue(len(action_table[state_5]) == 4) count = 0 for state in states: s = action_table[state] try: if s['+'] == s['*'] == s[')'] == s[self.arith.EOF]: self.assertTrue(hasattr(s['+'], '_semantic_action')) count += 1 except: pass self.assertTrue(count == 4) count = 0 s7 = None for state in states: s = action_table[state] try: if s['+'] == s[')'] == s[self.arith.EOF]: count += 1 if self.assertTrue(hasattr(s['*'], '_state_to_shift')): s7 = s['*'] if s7 == None else s7 self.assertTrue(s7 == s['*']) except: pass self.assertTrue(count == 6)
def setUp(self): self.arith = grammar.Grammar( 'S', ('+', '*', '(', ')', 'id', 'var', '=', 'let')) self.symbol_table = [dict()] self.result = None def get_result(x): self.result = x return x def add(x, y): t = x + y return t def mul(x, y): t = x * y return t def set_var(lv, rv): self.symbol_table[-1][lv] = rv return rv def get_var(rv): for table in reversed(self.symbol_table): if rv in table: return table[rv] raise KeyError(rv) def push(): self.symbol_table.append(dict()) def pop(*others): self.symbol_table.pop() self.arith.add_rule('S', ['E', get_result]) self.arith.add_rule('E', ['E', '+', 'T', add]) self.arith.add_rule('E', ['T', lambda v: v]) self.arith.add_rule('T', ['T', '*', 'F', mul]) self.arith.add_rule('T', ['F', lambda v: v]) self.arith.add_rule('F', ['(', 'E', ')', lambda v: v]) self.arith.add_rule('F', ['id', lambda v: v]) self.arith.add_rule('F', ['var', '=', 'E', set_var]) self.arith.add_rule('F', ['var', get_var]) self.arith.add_rule('F', ['let', push, '(', 'E', ')', pop, lambda v: v]) self.action_table, self.goto_table, self.start_state = build_parsing_table( self.arith, LR0(self.arith.START, 0, 0)) self.driver = Driver(self.action_table, self.goto_table, self.start_state)
def test_equivalence_sets_lalr_and_sets_lr0(self): states_lalr = frozenset([ frozenset([ LALR(self.StartExtendedSymbol, 0, 0), ]), frozenset([ LALR(self.StartExtendedSymbol, 0, 1), ]), frozenset([ LALR('S', 0, 1), ]), frozenset([ LALR('C', 0, 1), ]), frozenset([ LALR('C', 1, 1), ]), frozenset([ LALR('S', 0, 2), ]), frozenset([ LALR('C', 0, 2), ]), ]) states_lr0 = frozenset([ frozenset([ LR0(self.StartExtendedSymbol, 0, 0), ]), frozenset([ LR0(self.StartExtendedSymbol, 0, 1), ]), frozenset([ LR0('S', 0, 1), ]), frozenset([ LR0('C', 0, 1), ]), frozenset([ LR0('C', 1, 1), ]), frozenset([ LR0('S', 0, 2), ]), frozenset([ LR0('C', 0, 2), ]), ]) self.assertTrue(states_lr0 == states_lalr)
def setUp(self): self.arith = grammar.Grammar('S', ('(', ')', 'id', 'let')) self.symbol_table = [dict()] def push(*args): pass def pop(*args): pass self.arith.add_rule('S', ['E']) self.arith.add_rule('E', ['id']) self.arith.add_rule( 'E', ['let', push, '(', 'E', ')', pop, lambda *args: args]) self.action_table, self.goto_table, self.start_state = build_parsing_table( self.arith, LR0(self.arith.START, 0, 0), disable_mapping=True) self.driver = Driver(self.action_table, dict(self.goto_table), self.start_state)
def setUp(self): syn = syntax.Syntax('input') self.result = [] syn.terminal('NL', None) syn.terminal('NUM', None) syn.terminal('+', None) syn.terminal('-', None) syn.repeat(('line', ), 'input') syn.choice((('NL', ), ('expr', 'NL', lambda x: self.result.append(x))), 'line') syn.choice((('NUM', ), ('expr', 'expr', '+', lambda x, y: x + y), ('expr', 'expr', '-', lambda x, y: x - y)), 'expr') self.grammar = syn.as_grammar() self.start_item = LR0(self.grammar.START, 0, 0) self.action_table, self.goto_table, self.start_state = build_parsing_table( self.grammar, self.start_item, disable_mapping=True) self.driver = Driver(self.action_table, self.goto_table, self.start_state) self.kernel_states = [ frozenset([ LR0(self.grammar.START, 0, 0), ]), frozenset([ LR0(self.grammar.START, 0, 1), ]), frozenset([ LR0('input', 0, 1), LR0('input', 1, 1), ]), frozenset([ LR0('line', 0, 1), ]), frozenset([ LR0('line', 1, 1), LR0('expr', 1, 1), LR0('expr', 2, 1), ]), frozenset([ LR0('expr', 0, 1), ]), frozenset([ LR0('input', 0, 2), ]), frozenset([ LR0('line', 1, 2), ]), frozenset([ LR0('expr', 1, 1), LR0('expr', 1, 2), LR0('expr', 2, 1), LR0('expr', 2, 2), ]), frozenset([ LR0('expr', 1, 3), ]), frozenset([ LR0('expr', 2, 3), ]) ]
def test_closures_ofLRValue(self): a1 = set([ LR0(self.StartExtendedSymbol, 0, 0), LR0('S', 0, 0), LR0('S', 1, 0), LR0('R', 0, 0), LR0('R', 1, 0), LR0('L', 0, 0), LR0('L', 1, 0) ]) a2 = set([ LR0('S', 0, 0), LR0('S', 1, 0), LR0('R', 0, 0), LR0('R', 1, 0), LR0('R', 1, 1), LR0('L', 0, 0), LR0('L', 0, 1), LR0('L', 1, 0) ]) self.assertTrue(a1 == closure( set([LR0(self.StartExtendedSymbol, 0, 0)]), self.lrvalue)) self.assertTrue( a2 == closure(set([LR0('L', 0, 1), LR0('R', 1, 1)]), self.lrvalue))
def test_closures_ofArith(self): a1 = set([ LR0(self.StartExtendedSymbol, 0, 0), LR0('E', 0, 0), LR0('E', 1, 0), LR0('T', 0, 0), LR0('T', 1, 0), LR0('F', 0, 0), LR0('F', 1, 0) ]) a2 = set([ LR0('F', 0, 1), LR0('E', 0, 0), LR0('E', 1, 0), LR0('T', 0, 0), LR0('T', 1, 0), LR0('F', 0, 0), LR0('F', 1, 0) ]) self.assertTrue(a1 == closure( set([LR0(self.StartExtendedSymbol, 0, 0)]), self.arith)) self.assertTrue(a2 == closure(set([LR0('F', 0, 1)]), self.arith))
def test_closures_ofLRValue_with_actions(self): a1 = set([ LR0(self.StartExtendedSymbol, 0, 0), LR0('S', 0, 0), LR0('S', 1, 0), LR0('R', 0, 0), LR0('R', 1, 0), LR0('L', 0, 0), LR0('L', 1, 0) ]) a2 = set([ LR0(self.lrvalue_with_actions.ACTION_INSIDE % (3, '<lambda>'), 0, 0), LR0('R', 0, 0), LR0('R', 1, 0), LR0('R', 1, 1), LR0('L', 0, 0), LR0('L', 0, 1), LR0('L', 1, 0) ]) self.assertTrue( a1 == closure(set([LR0(self.StartExtendedSymbol, 0, 0)]), self.lrvalue_with_actions)) self.assertTrue(a2 == closure(set([LR0( 'L', 0, 1), LR0('R', 1, 1)]), self.lrvalue_with_actions))
def test_closures_ofSome(self): a1 = set([ LR0(self.StartExtendedSymbol, 0, 0), LR0('S', 0, 0), LR0('C', 0, 0), LR0('C', 1, 0) ]) a2 = set( [LR0('S', 0, 1), LR0('C', 0, 0), LR0('C', 1, 0), LR0('C', 1, 1)]) self.assertTrue(a1 == closure( set([LR0(self.StartExtendedSymbol, 0, 0)]), self.some)) self.assertTrue( a2 == closure(set([LR0('S', 0, 1), LR0('C', 1, 1)]), self.some))
def test_build_tables(self): action_table, goto_table, start_set = build_parsing_table( self.arith, LR0(self.StartExtendedSymbol, 0, 0)) self.assertTrue(len(action_table) == 12)
def test_goto_arith(self): a1 = set([ LR0('E', 0, 2), LR0('T', 0, 0), LR0('T', 1, 0), LR0('F', 0, 0), LR0('F', 1, 0), ]) a2 = set([ LR0('E', 0, 1), ]) a3 = set([ LR0('T', 0, 3), ]) self.assertTrue(a1 == goto( set([LR0(self.StartExtendedSymbol, 0, 1), LR0('E', 0, 1)]), '+', self.arith)) self.assertTrue(a2 == goto(set([LR0('E', 0, 0)]), 'E', self.arith)) self.assertTrue( a3 == goto(set([ LR0('T', 0, 2), LR0('F', 0, 0), LR0('F', 1, 0), ]), 'F', self.arith))
def setUp(self): self.lrvalue = grammar.Grammar(None, ('=', '*', ';', 'id', 'num')) self.symbol_table = dict() self.last_value = None def set_var(lv, rv): self.symbol_table[lv] = rv; return self.symbol_table[lv] def get_var(rv): if rv in self.symbol_table: return self.symbol_table[rv] raise KeyError(rv) def grab_last_value(v): self.last_value = v if isinstance(v, int) else self.symbol_table[v] return v self.lrvalue.augment('S') self.lrvalue.add_rule('S', ['E', ';', 'S', lambda e, s: s]) self.lrvalue.add_rule('S', ['E', ';', lambda v: v]) self.lrvalue.add_rule('E', ['L', '=', 'R', set_var]) self.lrvalue.add_rule('E', ['R', grab_last_value]) self.lrvalue.add_rule('L', ['*', 'R', get_var]) self.lrvalue.add_rule('L', ['id', lambda v: v]) self.lrvalue.add_rule('R', ['L', lambda v: v]) self.lrvalue.add_rule('R', ['num', lambda v: v]) self.action_table, self.goto_table, self.start_state = build_parsing_table_lalr(self.lrvalue, LR0(self.lrvalue.START, 0, 0), False) self.driver = Driver(self.action_table, self.goto_table, self.start_state)
def test_reduce_reduce_conflict(self): self.assertRaisesRegexp(ReduceReduce, "during process '#' terminal", build_parsing_table, self.reduce_reduce, LR0(self.StartExtendedSymbol, 0, 0), False)
def probe(*args): print "--", len(args), "-", " ".join(map(str, args)) def prompt(*args): print ">>> ", grammar = from_string(rpcalc_grammar, 'input', print_result=print_result, probe=probe, prompt=prompt, up=lambda x: x, add=lambda x, y: x + y, sub=lambda x, y: x - y, mul=lambda x, y: x * y, div=lambda x, y: x / y, mod=lambda x, y: x % y, neg=lambda x: -x) states, gotos, first = build_parsing_table(grammar, LR0(grammar.START, 0, 0)) driver = Driver(states, gotos, first) print "Reverse polish calculator. Write expressions like " print " >>> 2 + 3 (result in 2 + 3 = 5) or " print " >>> 2 + 4 * 5 (result in 2 + (4 * 5) = 22)" print prompt() driver.parse(CalcLexer(sys.stdin))
def test_action_table(self): states_shifts_reduce_actions = [ (frozenset([ LR0(self.arith.START, 0, 0), LR0('S', 0, 0), LR0('E', 0, 0), LR0('E', 1, 0), ]), (('id', 2), ('let', 4)), ()), (frozenset([ LR0(self.arith.START, 0, 1), ]), (), ()), (frozenset([ LR0('E', 0, 1), ]), (), ()), (frozenset([ LR0('S', 0, 1), ]), (), ()), (frozenset([ LR0('E', 1, 1), LR0(self.arith.ACTION_INSIDE % (1, 'push'), 0, 0), ]), (), ()), (frozenset([ LR0('E', 1, 2), ]), (('(', 9), ), ()), (frozenset([ LR0('E', 1, 4), ]), ((')', 8), ), ()), (frozenset([ LR0('E', 1, 6), ]), (), ()), (frozenset([ LR0('E', 1, 5), LR0(self.arith.ACTION_INSIDE % (2, 'pop'), 0, 0), ]), (), ()), (frozenset([ LR0('E', 1, 3), LR0('E', 0, 0), LR0('E', 1, 0), ]), (('id', 2), ('let', 4)), ()), ] self.assertTrue( len(states_shifts_reduce_actions) == len(self.action_table.keys())) for state, shifts, reduces in states_shifts_reduce_actions: h = hash(state) self.assertTrue( len(shifts) == len( filter(lambda action: "Shift" in str(action), self.action_table[h].values()))) if not shifts: continue keys, ids = zip(*shifts) found_ids = [ i for i in range(len(states_shifts_reduce_actions)) if ("Shift %s" % hash(states_shifts_reduce_actions[i][0]) ) in map(lambda a: str(a), self.action_table[h].values()) ] found_keys = self.action_table[h].keys() self.assertTrue(frozenset(ids) == frozenset(found_ids)) self.assertTrue(frozenset(keys) == frozenset(found_keys))
def test_canonical_collection_lr_value(self): collection = canonical_collection(self.lrvalue, LR0(self.StartExtendedSymbol, 0, 0)) self.assertTrue(len(collection) == 10) states = frozenset([ frozenset([ LR0(self.StartExtendedSymbol, 0, 0), LR0('S', 0, 0), LR0('S', 1, 0), LR0('L', 0, 0), LR0('L', 1, 0), LR0('R', 0, 0), ]), frozenset([ LR0(self.StartExtendedSymbol, 0, 1), ]), frozenset([ LR0('S', 0, 1), LR0('R', 0, 1), ]), frozenset([ LR0('S', 1, 1), ]), frozenset([ LR0('L', 0, 1), LR0('L', 0, 0), LR0('L', 1, 0), LR0('R', 0, 0), ]), frozenset([ LR0('L', 1, 1), ]), frozenset([ LR0('S', 0, 2), LR0('L', 0, 0), LR0('L', 1, 0), LR0('R', 0, 0), ]), frozenset([ LR0('L', 0, 2), ]), frozenset([ LR0('R', 0, 1), ]), frozenset([ LR0('S', 0, 3), ]), ]) self.assertTrue(states == collection)
def test_cannonical_collection(self): collection = canonical_collection(self.arith, LR0(self.arith.START, 0, 0)) states = frozenset([ frozenset([ LR0(self.arith.START, 0, 0), LR0('S', 0, 0), LR0('E', 0, 0), LR0('E', 1, 0), ]), frozenset([ LR0(self.arith.START, 0, 1), ]), frozenset([ LR0('E', 0, 1), ]), frozenset([ LR0('S', 0, 1), ]), frozenset([ LR0('E', 1, 1), LR0(self.arith.ACTION_INSIDE % (1, 'push'), 0, 0), ]), frozenset([ LR0('E', 1, 2), ]), frozenset([ LR0('E', 1, 4), ]), frozenset([ LR0('E', 1, 6), ]), frozenset([ LR0('E', 1, 5), LR0(self.arith.ACTION_INSIDE % (2, 'pop'), 0, 0), ]), frozenset([ LR0('E', 1, 3), LR0('E', 0, 0), LR0('E', 1, 0), ]), ]) self.assertTrue(states == collection)
def test_canonical_collection_arith(self): collection = canonical_collection(self.arith, LR0(self.StartExtendedSymbol, 0, 0)) self.assertTrue(len(collection) == 12) states = frozenset([ frozenset([ LR0(self.StartExtendedSymbol, 0, 0), LR0('E', 0, 0), LR0('E', 1, 0), LR0('T', 0, 0), LR0('T', 1, 0), LR0('F', 0, 0), LR0('F', 1, 0), ]), frozenset([ LR0(self.StartExtendedSymbol, 0, 1), LR0('E', 0, 1), ]), frozenset([ LR0('E', 1, 1), LR0('T', 0, 1), ]), frozenset([ LR0('F', 1, 1), ]), frozenset([ LR0('F', 0, 1), LR0('E', 0, 0), LR0('E', 1, 0), LR0('T', 0, 0), LR0('T', 1, 0), LR0('F', 0, 0), LR0('F', 1, 0), ]), frozenset([ LR0('T', 1, 1), ]), frozenset([ LR0('E', 0, 2), LR0('T', 0, 0), LR0('T', 1, 0), LR0('F', 0, 0), LR0('F', 1, 0), ]), frozenset([ LR0('T', 0, 2), LR0('F', 0, 0), LR0('F', 1, 0), ]), frozenset([ LR0('E', 0, 1), LR0('F', 0, 2), ]), frozenset([ LR0('E', 0, 3), LR0('T', 0, 1), ]), frozenset([ LR0('T', 0, 3), ]), frozenset([ LR0('F', 0, 3), ]), ]) self.assertTrue(states == collection)
def test_goto_table(self): states_gotos = [ (frozenset([ LR0(self.arith.START, 0, 0), LR0('S', 0, 0), LR0('E', 0, 0), LR0('E', 1, 0), ]), (('S', 1), ('id', 2), ('E', 3), ('let', 4))), (frozenset([ LR0(self.arith.START, 0, 1), ]), ()), (frozenset([ LR0('E', 0, 1), ]), ()), (frozenset([ LR0('S', 0, 1), ]), ()), (frozenset([ LR0('E', 1, 1), LR0(self.arith.ACTION_INSIDE % (1, 'push'), 0, 0), ]), ((self.arith.ACTION_INSIDE % (1, 'push'), 5), )), (frozenset([ LR0('E', 1, 2), ]), (('(', 9), )), (frozenset([ LR0('E', 1, 4), ]), ((')', 8), )), (frozenset([ LR0('E', 1, 6), ]), ()), (frozenset([ LR0('E', 1, 5), LR0(self.arith.ACTION_INSIDE % (2, 'pop'), 0, 0), ]), ((self.arith.ACTION_INSIDE % (2, 'pop'), 7), )), (frozenset([ LR0('E', 1, 3), LR0('E', 0, 0), LR0('E', 1, 0), ]), (('E', 6), ('id', 2), ('let', 4))), ] checked = 0 for state, gotos in states_gotos: h = hash(state) if not gotos: self.assertTrue(h not in self.goto_table) continue checked += 1 expected_keys, expected_states_id = zip(*gotos) found_gotos_keys = frozenset(self.goto_table[h].keys()) found_gotos_hashs = frozenset(self.goto_table[h].values()) self.assertTrue(frozenset(expected_keys) == found_gotos_keys) self.assertTrue( frozenset( [hash(states_gotos[i][0]) for i in expected_states_id]) == found_gotos_hashs) self.assertTrue(checked == len(self.goto_table.keys()))