def item_closure(self, item_map):
closure = item_map.copy()
work_list = closure.copy()
while work_list:
item, la_set = work_list.popitem()
ntid, rule_id, pos = item
derives = self.nt_rules[ntid][rule_id] # fetch the original rule
if pos < len(derives):
next_symbol = derives[pos]
if next_symbol[0] == 'nt':
new_la = set()
for category in la_set:
new_la |= self.calc_first_set_seq(derives[pos + 1:] +
d(t(category)))
next_ntid = next_symbol[1]
for rule_id, rule in enumerate(self.nt_rules[next_ntid]):
loc = 0
while loc < len(rule) and rule[loc] == t(
epsilon): # skip over epsilons
loc += 1
key = (next_ntid, rule_id, loc)
if key in closure:
diff = new_la - closure[key]
if diff:
closure[key] |= diff
work_list[key] = diff
else:
closure[key] = new_la.copy()
work_list[key] = new_la.copy()
return closure
def test_item_partition_goto(self):
rule_set = LALRRuleSet() # example grammar on EC pp.120
goal, list_, pair = rule_set.new_nt(3)
rule_set.add_rule(goal, d(nt(list_)))
rule_set.add_rule(list_, d(nt(list_), nt(pair)))
rule_set.add_rule(list_, d(nt(pair)))
rule_set.add_rule(pair, d(t('('), nt(pair), t(')')))
rule_set.add_rule(pair, d(t('('), t(')')))
rule_set.mark_goal(goal)
cc3 = {
(pair, 0, 1): {eof, '('},
(pair, 1, 1): {eof, '('},
(pair, 0, 0): {')'},
(pair, 1, 0): {')'}
}
partitions = rule_set.item_partition_goto(cc3)
self.assertEqual(
partitions, {
('nt', pair): {
(pair, 0, 1): {eof, '('}
},
('t', '('): {
(pair, 0, 0): {')'},
(pair, 1, 0): {')'}
},
('t', ')'): {
(pair, 1, 1): {eof, '('}
}
})
def test_item_closure_with_eps(self):
rule_set = LALRRuleSet()
S, M = rule_set.new_nt(2)
rule_set.add_rule(S, d(t('if'), t('B'), nt(M), nt(S)))
rule_set.add_rule(S, d(t('A')))
rule_set.add_rule(M, d(t(epsilon)))
item_set = {(S, 0, 2): {eof}}
result = rule_set.item_closure(item_set)
self.assertEqual(result, {(S, 0, 2): {eof}, (M, 0, 1): {'if', 'A'}})
def build_test_rule():
rule_set = LALRRuleSet() # example grammar on EC pp.120
goal, list_, pair = rule_set.new_nt(3)
rule_set.add_rule(goal, d(nt(list_)))
rule_set.add_rule(list_, d(nt(list_), nt(pair)))
rule_set.add_rule(list_, d(nt(pair)))
rule_set.add_rule(pair, d(t('('), nt(pair), t(')')))
rule_set.add_rule(pair, d(t('('), t(')')))
rule_set.mark_goal(goal)
return rule_set
def test_calc_ambiguous_table_2(self):
rule_set = LALRRuleSet() # example grammar on EC pp.136
stmt = rule_set.new_nt(1)[0]
rule_set.add_rule(stmt, d(t('if'), nt(stmt)))
rule_set.add_rule(stmt, d(t('if'), nt(stmt), t('else'), nt(stmt)))
rule_set.add_rule(stmt, d(t('assign')))
rule_set.mark_goal(stmt)
with self.assertRaises(LALRTableBuildError):
action, goto = rule_set.calc_parse_table()
def test_calc_parse_table(self):
rule_set = LALRRuleSet() # example grammar on EC pp.120
goal, list_, pair = rule_set.new_nt(3)
rule_set.add_rule(goal, d(nt(list_)))
rule_set.add_rule(list_, d(nt(list_), nt(pair)))
rule_set.add_rule(list_, d(nt(pair)))
rule_set.add_rule(pair, d(t('('), nt(pair), t(')')))
rule_set.add_rule(pair, d(t('('), t(')')))
rule_set.mark_goal(goal)
cc, action, goto = rule_set.calc_parse_table()
correct_action = [{
'(': (Action.shift, )
}, {
'(': (Action.shift, ),
eof: (Action.accept, 0, 0)
}, {
'(': (Action.reduce, 1, 1),
eof: (Action.reduce, 1, 1)
}, {
'(': (Action.shift, ),
')': (Action.shift, )
}, {
'(': (Action.reduce, 1, 0),
eof: (Action.reduce, 1, 0)
}, {
')': (Action.shift, )
}, {
'(': (Action.reduce, 2, 1),
eof: (Action.reduce, 2, 1),
')': (Action.reduce, 2, 1)
}, {
'(': (Action.reduce, 2, 0),
eof: (Action.reduce, 2, 0),
')': (Action.reduce, 2, 0)
}]
correct_goto = [{
('nt', 1): 1,
('nt', 2): 2,
('t', '('): 3
}, {
('nt', 2): 4,
('t', '('): 3
}, {}, {
('nt', 2): 5,
('t', '('): 3,
('t', ')'): 6
}, {}, {
('t', ')'): 7
}, {}, {}]
self.assertIsomorphic(action, goto, correct_action, correct_goto)
def test_item_advance(self):
rule_set = LALRRuleSet() # example grammar on EC pp.120
goal, list_, pair = rule_set.new_nt(3)
rule_set.add_rule(goal, d(nt(list_)))
rule_set.add_rule(list_, d(nt(list_), nt(pair)))
rule_set.add_rule(list_, d(nt(pair)))
rule_set.add_rule(pair, d(t('('), nt(pair), t(')')))
rule_set.add_rule(pair, d(t('('), t(')')))
rule_set.mark_goal(goal)
item_set = {(pair, 0, 0): {')'}, (pair, 1, 0): {')'}}
result = rule_set.item_advance(item_set)
self.assertEqual(result, {(pair, 0, 1): {')'}, (pair, 1, 1): {')'}})
def test_calc_LALR_conflict_table(self):
rule_set = LALRRuleSet() # example grammar on stanford handout pp.3
S, B, C = rule_set.new_nt(3)
rule_set.add_rule(S, d(t('a'), nt(B), t('c')))
rule_set.add_rule(S, d(t('b'), nt(C), t('c')))
rule_set.add_rule(S, d(t('a'), nt(C), t('d')))
rule_set.add_rule(S, d(t('b'), nt(B), t('d')))
rule_set.add_rule(B, d(t('e')))
rule_set.add_rule(C, d(t('e')))
rule_set.mark_goal(S)
with self.assertRaises(LALRTableBuildError):
action, goto = rule_set.calc_parse_table()
def test_calc_initial(self):
rule_set = LALRRuleSet() # example grammar on stanford handout pp.1
S, X = rule_set.new_nt(2)
rule_set.add_rule(S, d(nt(X), nt(X)))
rule_set.add_rule(X, d(t('a'), nt(X)))
rule_set.add_rule(X, d(t('b')))
rule_set.mark_goal(S)
initial = rule_set.calc_initial()
self.assertEqual(initial, {
(S, 0, 0): {eof},
(X, 0, 0): {'a', 'b'},
(X, 1, 0): {'a', 'b'}
})
def test_calc_parse_table_2(self):
rule_set = LALRRuleSet() # example grammar on stanford handout pp.1
S, X = rule_set.new_nt(2)
rule_set.add_rule(S, d(nt(X), nt(X)))
rule_set.add_rule(X, d(t('a'), nt(X)))
rule_set.add_rule(X, d(t('b')))
rule_set.mark_goal(S)
cc, action, goto = rule_set.calc_parse_table()
correct_action = [{
'a': (Action.shift, ),
'b': (Action.shift, )
}, {
'a': (Action.shift, ),
'b': (Action.shift, )
}, {
'a': (Action.shift, ),
'b': (Action.shift, )
}, {
'a': (Action.reduce, 1, 1),
'b': (Action.reduce, 1, 1),
eof: (Action.reduce, 1, 1)
}, {
eof: (Action.accept, 0, 0)
}, {
'a': (Action.reduce, 1, 0),
'b': (Action.reduce, 1, 0),
eof: (Action.reduce, 1, 0)
}]
correct_goto = [{
('nt', X): 1,
('t', 'a'): 2,
('t', 'b'): 3
}, {
('nt', X): 4,
('t', 'a'): 2,
('t', 'b'): 3
}, {
('nt', X): 5,
('t', 'a'): 2,
('t', 'b'): 3
}, {}, {}, {}]
self.assertIsomorphic(action, goto, correct_action, correct_goto)
def calc_initial(self):
initial = {}
for rule_id, rule in enumerate(self.nt_rules[self.goal]):
loc = 0
while loc < len(rule) and rule[loc] == t(
epsilon): # skip over epsilons
loc += 1
initial[(self.goal, rule_id, loc)] = {eof}
# initial = {(self.goal, i, 0): {eof} for i in range(len(self.nt_rules[self.goal]))}
initial = self.item_closure(initial)
return initial
def test_item_closure(self):
rule_set = LALRRuleSet() # example grammar on EC pp.120
goal, list_, pair = rule_set.new_nt(3)
rule_set.add_rule(goal, d(nt(list_)))
rule_set.add_rule(list_, d(nt(list_), nt(pair)))
rule_set.add_rule(list_, d(nt(pair)))
rule_set.add_rule(pair, d(t('('), nt(pair), t(')')))
rule_set.add_rule(pair, d(t('('), t(')')))
rule_set.mark_goal(goal)
item_set = {(goal, 0, 0): {eof}}
closure = rule_set.item_closure(item_set)
self.assertEqual(
closure, {
(goal, 0, 0): {eof},
(list_, 0, 0): {eof, '('},
(list_, 1, 0): {eof, '('},
(pair, 0, 0): {eof, '('},
(pair, 1, 0): {eof, '('}
})
item_set = {(pair, 0, 1): {eof, '('}, (pair, 1, 1): {eof, '('}}
closure = rule_set.item_closure(item_set)
self.assertEqual(
closure, {
(pair, 0, 1): {eof, '('},
(pair, 1, 1): {eof, '('},
(pair, 0, 0): {')'},
(pair, 1, 0): {')'}
}) # CC3
item_set = {(pair, 0, 1): {')'}, (pair, 1, 1): {')'}}
closure = rule_set.item_closure(item_set)
self.assertEqual(
closure, {
(pair, 0, 0): {')'},
(pair, 0, 1): {')'},
(pair, 1, 0): {')'},
(pair, 1, 1): {')'}
}) # CC6
def parse(self, tokens, repo=None):
tokens = iter(tokens)
data_stack = [None]
state_stack = [0]
token = next(tokens)
while True:
state = state_stack[-1]
if token[0] not in self.action[state]:
raise ParseError(self.action[state].keys(), token[0])
else:
action = self.action[state][token[0]]
if action[0] == Action.accept:
ntid, rule_id = action[1:]
derives = self.rule_set.nt_rules[ntid][rule_id]
split_point = -len(derives)
data_list = data_stack[split_point:]
if self.rule_actions[(ntid, rule_id)] is not None:
data = self.rule_actions[(ntid, rule_id)](data_list, repo)
else:
data = None
return data
elif action[0] == Action.shift:
data_stack.append(token[1])
state_stack.append(self.goto[state][t(token[0])])
token = next(tokens)
elif action[0] == Action.reduce:
ntid, rule_id = action[1:]
derives = self.rule_set.nt_rules[ntid][rule_id]
split_point = -pop_len(derives)
if split_point == 0:
data_list = []
else:
data_list = data_stack[split_point:]
data_stack = data_stack[:split_point]
state_stack = state_stack[:split_point]
# perform actions
if self.rule_actions[(ntid, rule_id)] is not None:
data = self.rule_actions[(ntid, rule_id)](data_list, repo)
else:
data = None
data_stack.append(data)
state = state_stack[-1]
state_stack.append(self.goto[state][nt(ntid)])
eq = re_utils.atom('=', EQ)
alt_delim = re_utils.atom('|', ALT_DELIM)
nfa = re_utils.alt([name, blank, eq, alt_delim])
bnf_scanner = scanner_builder(nfa, category_info)
rule_set = LALRRuleSet()
bnf, alt, dlist = rule_set.new_nt(3)
rule_set.mark_goal(bnf)
rule_actions = {}
rule_set.add_rule(bnf, d(t(NAME), t(EQ), nt(alt)))
def rule_bnf(data_list, repo):
return data_list[0], data_list[2]
rule_actions[(bnf, 0)] = rule_bnf
rule_set.add_rule(alt, d(nt(alt), t(ALT_DELIM), nt(dlist)))
def rule_alt(data_list, repo):
return data_list[0] + [data_list[2]]
rule_actions[(alt, 0)] = rule_alt
def pop_len(derives):
i = 0
for x in derives:
if x != t(epsilon):
i += 1
return i
