def test_iter(self):
d = IdentityDict()
x = []
y = []
d[x] = 1
d[y] = 2
assert sorted(d.items(), key=itemgetter(1)) == [(x, 1), (y, 2)]
def __init__(self, grammar):
self.grammar = grammar
self.lr_action = {}
self.lr_goto = {}
self._lr_goto_cache = IdentityDict()
self._lr_other_goto_cache = {}
self.lr0_cidhash = IdentityDict()
self._add_count = 0
self.sr_conflicts = []
self.rr_conflicts = []
self.build_table()
def test_delitem(self):
d = IdentityDict()
x = []
d[x] = "hello"
del d[x]
with py.test.raises(KeyError):
d[x]
def lr0_goto(cls, I, x, add_count, goto_cache):
s = goto_cache.setdefault(x, IdentityDict())
gs = []
for p in I:
n = p.lr_next
if n and n.lr_before == x:
s1 = s.get(n)
if not s1:
s1 = {}
s[n] = s1
gs.append(n)
s = s1
g = s.get("$end")
if not g:
if gs:
g = cls.lr0_closure(gs, add_count)
s["$end"] = g
else:
s["$end"] = gs
return g
def from_grammar(cls, grammar):
cidhash = IdentityDict()
goto_cache = {}
add_count = Counter()
C = cls.lr0_items(grammar, add_count, cidhash, goto_cache)
cls.add_lalr_lookaheads(grammar, C, add_count, cidhash, goto_cache)
lr_action = [None] * len(C)
lr_goto = [None] * len(C)
sr_conflicts = []
rr_conflicts = []
for st, I in enumerate(C):
st_action = {}
st_actionp = {}
st_goto = {}
for p in I:
if p.getlength() == p.lr_index + 1:
if p.name == "S'":
# Start symbol. Accept!
st_action["$end"] = 0
st_actionp["$end"] = p
else:
laheads = p.lookaheads[st]
for a in laheads:
if a in st_action:
r = st_action[a]
if r > 0:
sprec, slevel = grammar.productions[st_actionp[a].number].prec
rprec, rlevel = grammar.precedence.get(a, ("right", 0))
if (slevel < rlevel) or (slevel == rlevel and rprec == "left"):
st_action[a] = -p.number
st_actionp[a] = p
if not slevel and not rlevel:
sr_conflicts.append((st, repr(a), "reduce"))
grammar.productions[p.number].reduced += 1
elif not (slevel == rlevel and rprec == "nonassoc"):
if not rlevel:
sr_conflicts.append((st, repr(a), "shift"))
elif r < 0:
oldp = grammar.productions[-r]
pp = grammar.productions[p.number]
if oldp.number > pp.number:
st_action[a] = -p.number
st_actionp[a] = p
chosenp, rejectp = pp, oldp
grammar.productions[p.number].reduced += 1
grammar.productions[oldp.number].reduced -= 1
else:
chosenp, rejectp = oldp, pp
rr_conflicts.append((st, repr(chosenp), repr(rejectp)))
else:
raise ParserGeneratorError("Unknown conflict in state %d" % st)
else:
st_action[a] = -p.number
st_actionp[a] = p
grammar.productions[p.number].reduced += 1
else:
i = p.lr_index
a = p.prod[i + 1]
if a in grammar.terminals:
g = cls.lr0_goto(I, a, add_count, goto_cache)
j = cidhash.get(g, -1)
if j >= 0:
if a in st_action:
r = st_action[a]
if r > 0:
if r != j:
raise ParserGeneratorError("Shift/shift conflict in state %d" % st)
elif r < 0:
rprec, rlevel = grammar.productions[st_actionp[a].number].prec
sprec, slevel = grammar.precedence.get(a, ("right", 0))
if (slevel > rlevel) or (slevel == rlevel and rprec == "right"):
grammar.productions[st_actionp[a].number].reduced -= 1
st_action[a] = j
st_actionp[a] = p
if not rlevel:
sr_conflicts.append((st, repr(a), "shift"))
elif not (slevel == rlevel and rprec == "nonassoc"):
if not slevel and not rlevel:
sr_conflicts.append((st, repr(a), "reduce"))
else:
raise ParserGeneratorError("Unknown conflict in state %d" % st)
else:
st_action[a] = j
st_actionp[a] = p
nkeys = set()
for ii in I:
for s in ii.unique_syms:
if s in grammar.nonterminals:
nkeys.add(s)
for n in nkeys:
g = cls.lr0_goto(I, n, add_count, goto_cache)
j = cidhash.get(g, -1)
if j >= 0:
st_goto[n] = j
lr_action[st] = st_action
lr_goto[st] = st_goto
default_reductions = [0] * len(lr_action)
for state, actions in enumerate(lr_action):
actions = set(itervalues(actions))
if len(actions) == 1 and next(iter(actions)) < 0:
default_reductions[state] = next(iter(actions))
return LRTable(grammar, lr_action, lr_goto, default_reductions, sr_conflicts, rr_conflicts)
def from_grammar(cls, grammar):
cidhash = IdentityDict()
goto_cache = {}
add_count = Counter()
C = cls.lr0_items(grammar, add_count, cidhash, goto_cache)
cls.add_lalr_lookaheads(grammar, C, add_count, cidhash, goto_cache)
lr_action = [None] * len(C)
lr_goto = [None] * len(C)
sr_conflicts = []
rr_conflicts = []
for st, I in enumerate(C):
st_action = {}
st_actionp = {}
st_goto = {}
for p in I:
if p.getlength() == p.lr_index + 1:
if p.name == "S'":
# Start symbol. Accept!
st_action["$end"] = 0
st_actionp["$end"] = p
else:
laheads = p.lookaheads[st]
for a in laheads:
if a in st_action:
r = st_action[a]
if r > 0:
sprec, slevel = grammar.productions[st_actionp[a].number].prec
rprec, rlevel = grammar.precedence.get(a, ("right", 0))
if (slevel < rlevel) or (slevel == rlevel and rprec == "left"):
st_action[a] = -p.number
st_actionp[a] = p
if not slevel and not rlevel:
sr_conflicts.append((st, repr(a), "reduce"))
grammar.productions[p.number].reduced += 1
elif not (slevel == rlevel and rprec == "nonassoc"):
if not rlevel:
sr_conflicts.append((st, repr(a), "shift"))
elif r < 0:
oldp = grammar.productions[-r]
pp = grammar.productions[p.number]
if oldp.number > pp.number:
st_action[a] = -p.number
st_actionp[a] = p
chosenp, rejectp = pp, oldp
grammar.productions[p.number].reduced += 1
grammar.productions[oldp.number].reduced -= 1
else:
chosenp, rejectp = oldp, pp
rr_conflicts.append((st, repr(chosenp), repr(rejectp)))
else:
raise LALRError("Unknown conflict in state %d" % st)
else:
st_action[a] = -p.number
st_actionp[a] = p
grammar.productions[p.number].reduced += 1
else:
i = p.lr_index
a = p.prod[i + 1]
if a in grammar.terminals:
g = cls.lr0_goto(I, a, add_count, goto_cache)
j = cidhash.get(g, -1)
if j >= 0:
if a in st_action:
r = st_action[a]
if r > 0:
if r != j:
raise LALRError("Shift/shift conflict in state %d" % st)
elif r < 0:
rprec, rlevel = grammar.productions[st_actionp[a].number].prec
sprec, slevel = grammar.precedence.get(a, ("right", 0))
if (slevel > rlevel) or (slevel == rlevel and rprec == "right"):
grammar.productions[st_actionp[a].number].reduced -= 1
st_action[a] = j
st_actionp[a] = p
if not rlevel:
sr_conflicts.append((st, repr(a), "shift"))
elif not (slevel == rlevel and rprec == "nonassoc"):
if not slevel and not rlevel:
sr_conflicts.append((st, repr(a), "reduce"))
else:
raise LALRError("Unknown conflict in state %d" % st)
else:
st_action[a] = j
st_actionp[a] = p
nkeys = set()
for ii in I:
for s in ii.unique_syms:
if s in grammar.nonterminals:
nkeys.add(s)
for n in nkeys:
g = cls.lr0_goto(I, n, add_count, goto_cache)
j = cidhash.get(g, -1)
if j >= 0:
st_goto[n] = j
lr_action[st] = st_action
lr_goto[st] = st_goto
default_reductions = [0] * len(lr_action)
for state, actions in enumerate(lr_action):
actions = set(itervalues(actions))
if len(actions) == 1 and next(iter(actions)) < 0:
default_reductions[state] = next(iter(actions))
return LRTable(grammar, lr_action, lr_goto, default_reductions, sr_conflicts, rr_conflicts)
class LRTable(object):
def __init__(self, grammar):
self.grammar = grammar
self.lr_action = {}
self.lr_goto = {}
self._lr_goto_cache = IdentityDict()
self._lr_other_goto_cache = {}
self.lr0_cidhash = IdentityDict()
self._add_count = 0
self.sr_conflicts = []
self.rr_conflicts = []
self.build_table()
def lr0_items(self):
C = [self.lr0_closure([self.grammar.productions[0].lr_next])]
for i, I in enumerate(C):
self.lr0_cidhash[I] = i
i = 0
while i < len(C):
I = C[i]
i += 1
asyms = set()
for ii in I:
asyms.update(ii.unique_syms)
for x in asyms:
g = self.lr0_goto(I, x)
if not g:
continue
if g in self.lr0_cidhash:
continue
self.lr0_cidhash[g] = len(C)
C.append(g)
return C
def lr0_closure(self, I):
self._add_count += 1
J = I[:]
added = True
while added:
added = False
for j in J:
for x in j.lr_after:
if x.lr0_added == self._add_count:
continue
J.append(x.lr_next)
x.lr0_added = self._add_count
added = True
return J
def lr0_goto(self, I, x):
if I in self._lr_goto_cache and x in self._lr_goto_cache[I]:
return self._lr_goto_cache[I][x]
s = self._lr_other_goto_cache.setdefault(x, IdentityDict())
gs = []
for p in I:
n = p.lr_next
if n and n.lr_before == x:
s1 = s.get(n)
if not s1:
s1 = {}
s[n] = s1
gs.append(n)
s = s1
g = s.get("$end")
if not g:
if gs:
g = self.lr0_closure(gs)
s["$end"] = g
else:
s["$end"] = gs
self._lr_goto_cache.setdefault(I, {})[x] = g
return g
def add_lalr_lookaheads(self, C):
nullable = self.compute_nullable_nonterminals()
trans = self.find_nonterminal_transitions(C)
readsets = self.compute_read_sets(C, trans, nullable)
lookd, included = self.compute_lookback_includes(C, trans, nullable)
followsets = self.compute_follow_sets(trans, readsets, included)
self.add_lookaheads(lookd, followsets)
def compute_nullable_nonterminals(self):
nullable = set()
num_nullable = 0
while True:
for p in self.grammar.productions[1:]:
if p.getlength() == 0:
nullable.add(p.name)
continue
for t in p.prod:
if t not in nullable:
break
else:
nullable.add(p.name)
if len(nullable) == num_nullable:
break
num_nullable = len(nullable)
return nullable
def find_nonterminal_transitions(self, C):
trans = []
for idx, state in enumerate(C):
for p in state:
if p.lr_index < p.getlength() - 1:
t = (idx, p.prod[p.lr_index + 1])
if t[1] in self.grammar.nonterminals and t not in trans:
trans.append(t)
return trans
def compute_read_sets(self, C, ntrans, nullable):
FP = lambda x: self.dr_relation(C, x, nullable)
R = lambda x: self.reads_relation(C, x, nullable)
return digraph(ntrans, R, FP)
def compute_follow_sets(self, ntrans, readsets, includesets):
FP = lambda x: readsets[x]
R = lambda x: includesets.get(x, [])
return digraph(ntrans, R, FP)
def dr_relation(self, C, trans, nullable):
state, N = trans
terms = []
g = self.lr0_goto(C[state], N)
for p in g:
if p.lr_index < p.getlength() - 1:
a = p.prod[p.lr_index + 1]
if a in self.grammar.terminals and a not in terms:
terms.append(a)
if state == 0 and N == self.grammar.productions[0].prod[0]:
terms.append("$end")
return terms
def reads_relation(self, C, trans, empty):
rel = []
state, N = trans
g = self.lr0_goto(C[state], N)
j = self.lr0_cidhash.get(g, -1)
for p in g:
if p.lr_index < p.getlength() - 1:
a = p.prod[p.lr_index + 1]
if a in empty:
rel.append((j, a))
return rel
def compute_lookback_includes(self, C, trans, nullable):
lookdict = {}
includedict = {}
dtrans = dict.fromkeys(trans, 1)
for state, N in trans:
lookb = []
includes = []
for p in C[state]:
if p.name != N:
continue
lr_index = p.lr_index
j = state
while lr_index < p.getlength() - 1:
lr_index += 1
t = p.prod[lr_index]
if (j, t) in dtrans:
li = lr_index + 1
while li < p.getlength():
if p.prod[li] in self.grammar.terminals:
break
if p.prod[li] not in nullable:
break
li += 1
else:
includes.append((j, t))
g = self.lr0_goto(C[j], t)
j = self.lr0_cidhash.get(g, -1)
for r in C[j]:
if r.name != p.name:
continue
if r.getlength() != p.getlength():
continue
i = 0
while i < r.lr_index:
if r.prod[i] != p.prod[i + 1]:
break
i += 1
else:
lookb.append((j, r))
for i in includes:
includedict.setdefault(i, []).append((state, N))
lookdict[state, N] = lookb
return lookdict, includedict
def add_lookaheads(self, lookbacks, followset):
for trans, lb in iteritems(lookbacks):
for state, p in lb:
f = followset.get(trans, [])
laheads = p.lookaheads.setdefault(state, [])
for a in f:
if a not in laheads:
laheads.append(a)
def build_table(self):
C = self.lr0_items()
self.add_lalr_lookaheads(C)
st = 0
for I in C:
st_action = {}
st_actionp = {}
st_goto = {}
for p in I:
if p.getlength() == p.lr_index + 1:
if p.name == "S'":
# Start symbol. Accept!
st_action["$end"] = 0
st_actionp["$end"] = p
else:
laheads = p.lookaheads[st]
for a in laheads:
if a in st_action:
r = st_action[a]
if r > 0:
sprec, slevel = self.grammar.productions[st_actionp[a].number].prec
rprec, rlevel = self.grammar.precedence.get(a, ("right", 0))
if (slevel < rlevel) or (slevel == rlevel and rprec == "left"):
st_action[a] = -p.number
st_actionp[a] = p
if not slevel and not rlevel:
self.sr_conflicts.append((st, a, "reduce"))
self.grammar.productions[p.number].reduced += 1
elif slevel == rlevel and rprec == "nonassoc":
st_action[a] = None
else:
if not rlevel:
self.sr_conflicts.append((st, a, "shift"))
elif r < 0:
oldp = self.grammar.productions[-r]
pp = self.grammar.productions[p.number]
if oldp.number > pp.number:
st_action[a] = -p.number
st_actionp[a] = p
chosenp, rejectp = pp, oldp
self.grammar.productions[p.number].reduced += 1
self.grammar.productions[oldp.number].reduced -= 1
else:
chosenp, rejectp = oldp, pp
self.rr_conflicts.append((st, chosenp, rejectp))
else:
raise LALRError("Unknown conflict in state %d" % st)
else:
st_action[a] = -p.number
st_actionp[a] = p
self.grammar.productions[p.number].reduced += 1
else:
i = p.lr_index
a = p.prod[i + 1]
if a in self.grammar.terminals:
g = self.lr0_goto(I, a)
j = self.lr0_cidhash.get(g, -1)
if j >= 0:
if a in st_action:
r = st_action[a]
if r > 0:
if r != j:
raise LALRError("Shift/shift conflict in state %d" % st)
elif r < 0:
rprec, rlevel = self.grammar.productions[st_actionp[a].number].prec
sprec, slevel = self.grammar.precedence.get(a, ("right", 0))
if (slevel > rlevel) or (slevel == rlevel and rprec == "right"):
self.grammar.productions[st_actionp[a].number].reduced -= 1
st_action[a] = j
st_actionp[a] = p
if not rlevel:
self.sr_conflicts.append((st, a, "shift"))
elif slevel == rlevel and rprec == "nonassoc":
st_action[a] = None
else:
if not slevel and not rlevel:
self.sr_conflicts.append((st, a, "reduce"))
else:
raise LALRError("Unknown conflict in state %d" % st)
else:
st_action[a] = j
st_actionp[a] = p
nkeys = set()
for ii in I:
for s in ii.unique_syms:
if s in self.grammar.nonterminals:
nkeys.add(s)
for n in nkeys:
g = self.lr0_goto(I, n)
j = self.lr0_cidhash.get(g, -1)
if j >= 0:
st_goto[n] = j
self.lr_action[st] = st_action
self.lr_goto[st] = st_goto
st += 1
def test_create(self):
IdentityDict()
def test_len(self):
d = IdentityDict()
d[[]] = 3
d[3] = 5
assert len(d) == 2
def test_get_set_item(self):
d = IdentityDict()
x = []
d[x] = "test"
assert d[x] == "test"
