def compute_firsts(G):
firsts = {}
change = True
# init First(Vt)
for terminal in G.terminals:
firsts[terminal] = ContainerSet(terminal)
# init First(Vn)
for nonterminal in G.nonTerminals:
firsts[nonterminal] = ContainerSet()
while change:
change = False
# P: X -> alpha
for production in G.Productions:
X = production.Left
alpha = production.Right
# get current First(X)
first_X = firsts[X]
# init First(alpha)
try:
first_alpha = firsts[alpha]
except KeyError:
first_alpha = firsts[alpha] = ContainerSet()
# CurrentFirst(alpha)???
local_first = compute_local_first(firsts, alpha)
# update First(X) and First(alpha) from CurrentFirst(alpha)
change |= first_alpha.hard_update(local_first)
change |= first_X.hard_update(local_first)
# First(Vt) + First(Vt) + First(RightSides)
return firsts
def build_larl1_automaton(G, firsts=None):
assert len(G.startSymbol.productions) == 1, 'Grammar must be augmented'
if not firsts:
firsts = compute_firsts(G)
firsts[G.EOF] = ContainerSet(G.EOF)
start_production = G.startSymbol.productions[0]
start_item = Item(start_production, 0, lookaheads=ContainerSet(G.EOF))
start = frozenset([start_item.Center()])
closure = closure_lr1([start_item], firsts)
automaton = State(frozenset(closure), True)
pending = [start]
visited = {start: automaton}
while pending:
current = pending.pop()
current_state = visited[current]
current_closure = current_state.state
for symbol in G.terminals + G.nonTerminals:
goto = goto_lr1(current_closure, symbol, just_kernel=True)
closure = closure_lr1(goto, firsts)
center = frozenset(item.Center() for item in goto)
if center == frozenset():
continue
try:
next_state = visited[center]
centers = {item.Center(): item for item in next_state.state}
centers = {
item.Center(): (centers[item.Center()], item)
for item in closure
}
updated_items = set()
for c, (itemA, itemB) in centers.items():
item = Item(c.production, c.pos,
itemA.lookaheads | itemB.lookaheads)
updated_items.add(item)
updated_items = frozenset(updated_items)
if next_state.state != updated_items:
pending.append(center)
next_state.state = updated_items
except KeyError:
visited[center] = next_state = State(frozenset(closure), True)
pending.append(center)
if current_state[symbol.Name] is None:
current_state.add_transition(symbol.Name, next_state)
else:
assert current_state.get(
symbol.Name) is next_state, 'Bad build!!!'
automaton.set_formatter(multiline_formatter)
return automaton
def compute_follows(G, firsts):
follows = {}
change = True
local_firsts = {}
for nonterminal in G.nonTerminals:
follows[nonterminal] = ContainerSet()
follows[G.startSymbol] = ContainerSet(G.EOF)
while change:
change = False
for production in G.Productions:
X = production.Left
alpha = production.Right
follow_X = follows[X]
for i, Y in enumerate(alpha):
if Y.IsNonTerminal:
try:
beta_f = local_firsts[alpha, i]
except KeyError:
beta_f = local_firsts[alpha, i] = compute_local_first(
firsts, islice(alpha, i + 1, None))
change |= follows[Y].update(beta_f)
if beta_f.contains_epsilon:
change |= follows[Y].update(follow_X)
return follows
def compute_firsts(G):
firsts = {}
change = True
for terminal in G.terminals:
firsts[terminal] = ContainerSet(terminal)
for nonterminal in G.nonTerminals:
firsts[nonterminal] = ContainerSet()
while change:
change = False
# P: X -> alpha
for production in G.Productions:
X, alpha = production
first_X = firsts[X]
try:
first_alpha = firsts[alpha]
except KeyError:
first_alpha = firsts[alpha] = ContainerSet()
local_first = compute_local_first(firsts, alpha)
change |= first_alpha.hard_update(local_first)
change |= first_X.hard_update(local_first)
return firsts
def compute_follows(G, firsts):
follows = {}
change = True
# init Follow(Vn)
for nonterminal in G.nonTerminals:
follows[nonterminal] = ContainerSet()
follows[G.startSymbol] = ContainerSet(G.EOF)
while change:
change = False
# P: X -> alpha
for production in G.Productions:
X = production.Left
alpha = production.Right
follow_X = follows[X]
for i, symbol in enumerate(alpha):
if symbol.IsNonTerminal:
beta = alpha[i + 1:]
first_beta = _compute_local_first(firsts, beta)
change |= follows[symbol].update(first_beta)
if first_beta.contains_epsilon or len(beta) == 0:
change |= follows[symbol].update(follow_X)
# Follow(Vn)
return follows
def closure_lr1(items, firsts):
closure = ContainerSet(*items)
changed = True
while changed:
new_items = ContainerSet()
for item in closure:
new_items.extend(expand(item, firsts))
changed = closure.update(new_items)
return compress(closure)
def Compact_Automata(self, automata):
new_states = {}
for state in automata:
new_states[state] = state
states_to_compress = []
for state1 in automata:
if not new_states[state1] == state1:
continue
states_to_compress = [state1]
for state2 in automata:
if state1 == state2 or not new_states[state2] == state2:
continue
node1 = state1.state
node2 = state2.state
are_equals = False
if len(node1) == len(node2):
for item1 in node1:
are_equals = False
for item2 in node2:
if item1.Center() == item2.Center():
are_equals = True
if not are_equals:
break
if are_equals:
states_to_compress.append(state2)
compress_set = ContainerSet()
for state in states_to_compress:
node = state.state
compress_set.update(ContainerSet(*node))
new_node = self.compress(compress_set)
new_state = State(frozenset(new_node), True)
for state in states_to_compress:
new_states[state] = new_state
new_automata = new_states[automata]
for state in automata:
for key in state.transitions:
for to_state in state.transitions[key]:
try:
assert new_states[to_state] in new_states[state].transitions[key]
except:
new_states[state].add_transition(key, new_states[to_state])
return new_automata
def closure_lr1(self, items, firsts):
closure = ContainerSet(*items)
changed = True
while changed:
changed = False
new_items = ContainerSet()
for item in closure:
new_items.update(ContainerSet(*self.expand(item, firsts)))
changed = closure.update(new_items)
return self.compress(closure)
def ClosureLR1(items, firsts):
closure = ContainerSet(*items)
changed = True
while changed:
changed = False
new_items = ContainerSet()
for item in closure:
new_items.extend(LR1Parser.Expand(item, firsts))
changed = closure.update(new_items)
return LR1Parser.Compress(closure)
def closure_lr1(items, firsts):
closure = ContainerSet(*items)
changed = True
while changed:
changed = False
new_items = ContainerSet()
# Your code here!!!
for x in closure:
new_items.extend(expand(x, firsts))
changed = closure.update(new_items)
return compress(closure)
def build_LR1_automaton(self, G):
assert len(G.startSymbol.productions) == 1, 'Grammar must be augmented'
firsts = compute_firsts(G)
firsts[G.EOF] = ContainerSet(G.EOF)
start_production = G.startSymbol.productions[0]
start_item = Item(start_production, 0, lookaheads=(G.EOF,))
start = frozenset([start_item])
closure = self.closure_lr1(start, firsts)
automaton = State(frozenset(closure), True)
pending = [ start ]
visited = { start: automaton }
while pending:
current = pending.pop()
current_state = visited[current]
for symbol in G.terminals + G.nonTerminals:
closure_current = self.closure_lr1(current, firsts)
goto = self.goto_lr1(closure_current, symbol, just_kernel=True)
if len(goto) == 0:
continue
try:
next_state = visited[goto]
except KeyError:
next_closure = self.closure_lr1(goto, firsts)
visited[goto] = next_state = State(frozenset(next_closure), True)
pending.append(goto)
current_state.add_transition(symbol.Name, next_state)
automaton.set_formatter(multiline_formatter)
return automaton
def nfa_to_dfa(automaton):
transitions = {}
start = epsilon_closure(automaton, [automaton.start])
start.id = 0
start.is_final = any(s in automaton.finals for s in start)
states = [start]
pending = [start]
while pending:
state = pending.pop()
for symbol in automaton.vocabulary:
e_closure = ContainerSet(*move(automaton, state, symbol))
e_closure.update(epsilon_closure(automaton, e_closure.set))
if (len(e_closure) == 0):
continue
try:
transitions[state.id,
symbol] = states[states.index(e_closure)].id
except ValueError:
e_closure.id = len(states)
e_closure.is_final = any(s in automaton.finals
for s in e_closure)
states.append(e_closure)
pending.append(e_closure)
transitions[state.id, symbol] = e_closure.id
finals = [state.id for state in states if state.is_final]
dfa = DFA(len(states), finals, transitions)
return dfa
def compute_local_first(firsts, alpha):
"""
Computes First(alpha), given First(Vt) and First(Vn)
alpha in (Vt U Vn)*
"""
first_alpha = ContainerSet()
try:
alpha_is_epsilon = alpha.IsEpsilon
except:
alpha_is_epsilon = False
# alpha == epsilon ? First(alpha) = { epsilon }
if alpha_is_epsilon:
first_alpha.set_epsilon()
# alpha = X1 ... XN
# First(Xi) subset of First(alpha)
# epsilon in First(X1)...First(Xi) ? First(Xi+1) subset of First(X) & First(alpha)
# epsilon in First(X1)...First(XN) ? epsilon in First(X) & First(alpha)
else:
for symbol in alpha:
first_symbol = firsts[symbol]
first_alpha.update(first_symbol)
if not first_symbol.contains_epsilon:
break
else:
first_alpha.set_epsilon()
return first_alpha
def compute_local_first_queue(firsts, alpha):
first_alpha = ContainerSet()
temp = []
try:
alpha_is_epsilon = alpha.IsEpsilon
except:
alpha_is_epsilon = False
if alpha_is_epsilon:
first_alpha.set_epsilon(True)
return first_alpha
breaked = False
while len(alpha) > 0:
symbol = alpha.pop()
temp.append(symbol)
first_alpha.update(firsts[symbol])
if not firsts[symbol].contains_epsilon:
breaked = True
break
if not breaked:
first_alpha.set_epsilon(True)
while len(temp) > 0:
alpha.append(temp.pop())
return first_alpha
def ComputeLocalFirst(firsts: dict, alpha) -> ContainerSet:
"""
Computa el conjunto First de la cadena alpha, esta cadena puede
tener tanto terminales como non-terminales.
"""
first_alpha = ContainerSet()
try:
alpha_is_epsilon = alpha.IsEpsilon
except:
alpha_is_epsilon = False
# alpha == epsilon ? First(alpha) = { epsilon }
if alpha_is_epsilon:
first_alpha.set_epsilon()
# alpha = X1 ... XN
# First(Xi) subset of First(alpha)
# epsilon in First(X1)...First(Xi) ? First(Xi+1) subset of First(X) & First(alpha)
# epsilon in First(X1)...First(XN) ? epsilon in First(X) & First(alpha)
else:
for symbol in alpha:
first_symbol = firsts[symbol]
first_alpha.update(first_symbol)
if not first_symbol.contains_epsilon:
break
else:
first_alpha.set_epsilon()
return first_alpha
def closure_lr1(items, firsts, with_lookaheads=True):
closure = ContainerSet(*items)
changed = True
while changed:
changed = False
new_items = ContainerSet()
# Your code here!!!
for item in closure:
new_items.extend(expand(item, firsts))
changed = closure.update(new_items)
if with_lookaheads:
return compress(closure)
else:
return closure
def compute_follows(G, firsts):
follows = {}
change = True
local_firsts = {}
# init Follow(Vn)
for nonterminal in G.nonTerminals:
follows[nonterminal] = ContainerSet()
follows[G.startSymbol] = ContainerSet(G.EOF)
while change:
change = False
# P: X -> alpha
for production in G.Productions:
X = production.Left
alpha = production.Right
follow_X = follows[X]
###################################################
# X -> zeta Y beta
# First(beta) - { epsilon } subset of Follow(Y)
# beta ->* epsilon or X -> zeta Y ? Follow(X) subset of Follow(Y)
###################################################
# <CODE_HERE> #
###################################################
##
for it, val in enumerate(alpha):
if val.IsNonTerminal:
follows_val = follows[val]
try:
first = local_firsts[alpha, it]
except:
first = local_firsts[alpha, it] = compute_local_first(
firsts, islice(alpha, it + 1, None))
change |= follows_val.update(first)
if first.contains_epsilon:
change |= follows_val.update(follow_X)
return follows
def compute_follows(G, firsts):
follows = {}
change = True
local_firsts = {}
# init Follow(Vn)
for nonterminal in G.nonTerminals:
follows[nonterminal] = ContainerSet()
follows[G.startSymbol] = ContainerSet(G.EOF)
while change:
change = False
# P: X -> alpha
for production in G.Productions:
X = production.Left
alpha = production.Right
follow_X = follows[X]
# X -> zeta Y beta
# First(beta) - { epsilon } subset of Follow(Y)
# beta ->* epsilon or X -> zeta Y ? Follow(X) subset of Follow(Y)
for i, symbol in enumerate(alpha):
if symbol.IsNonTerminal:
follow_Y = follows[symbol]
try:
first_beta = local_firsts[alpha, i]
except KeyError:
first_beta = local_firsts[alpha,
i] = compute_local_first(
firsts,
islice(
alpha, i + 1, None))
change |= follow_Y.update(first_beta)
if first_beta.contains_epsilon:
change |= follow_Y.update(follow_X)
# Follow(Vn)
return follows
def follows(self):
G = self.G
return {
G['E']:
ContainerSet(G[')'], G.EOF, contains_epsilon=False),
G['T']:
ContainerSet(G['|'], G[')'], G.EOF, contains_epsilon=False),
G['F']:
ContainerSet(G['symbol'],
G['|'],
G['('],
G[')'],
G.EOF,
contains_epsilon=False),
G['A']:
ContainerSet(G['symbol'],
G.EOF,
G['|'],
G['*'],
G['('],
G[')'],
contains_epsilon=False),
G['X']:
ContainerSet(G[')'], G.EOF, contains_epsilon=False),
G['Y']:
ContainerSet(G['|'], G[')'], G.EOF, contains_epsilon=False),
G['Z']:
ContainerSet(G['symbol'],
G.EOF,
G['|'],
G['('],
G[')'],
contains_epsilon=False)
}
def compute_follows(G, firsts):
follows = {}
change = True
local_firsts = {}
# init Follow(Vn)
for nonterminal in G.nonTerminals:
follows[nonterminal] = ContainerSet()
follows[G.startSymbol] = ContainerSet(G.EOF)
while change:
change = False
# P: X -> alpha
for production in G.Productions:
x = production.Left
alpha = production.Right
follow_x = follows[x]
for i, Y in enumerate(alpha):
if not Y.IsTerminal:
follow_y = follows[Y]
try:
first_beta = local_firsts[alpha, i]
except KeyError:
first_beta = local_firsts[alpha,
i] = compute_local_first(
firsts,
islice(
alpha, i + 1, None))
change |= follow_y.update(first_beta)
if i == len(alpha) - 1 or first_beta.contains_epsilon:
change |= follow_y.update(follow_x)
# Follow(Vn)
return follows
def ComputeFirsts(G: Grammar) -> dict:
"""
Calcula el conjunto First de los terminales, los no-terminales
y las partes derechas de la gramatica.
"""
firsts = {}
change = True
# init First(Vt)
for terminal in G.terminals:
firsts[terminal] = ContainerSet(terminal)
# init First(Vn)
for nonterminal in G.nonTerminals:
firsts[nonterminal] = ContainerSet()
while change:
change = False
# P: X -> alpha
for production in G.Productions:
X = production.Left
alpha = production.Right
# get current First(X)
first_X = firsts[X]
# init First(alpha)
try:
first_alpha = firsts[alpha]
except:
first_alpha = firsts[alpha] = ContainerSet()
# CurrentFirst(alpha)???
local_first = UtilsParsers.ComputeLocalFirst(firsts, alpha)
# update First(X) and First(alpha) from CurrentFirst(alpha)
change |= first_alpha.hard_update(local_first)
change |= first_X.hard_update(local_first)
# First(Vt) + First(Vt) + First(RightSides)
return firsts
def ComputeFollows(G: Grammar, firsts: dict) -> dict:
"""
Calcula el conjunto Follow de todos los no terminales de la
gramatica.
"""
follows = {}
change = True
local_firsts = {}
# init Follow(Vn)
for nonterminal in G.nonTerminals:
follows[nonterminal] = ContainerSet()
follows[G.startSymbol] = ContainerSet(G.EOF)
while change:
change = False
# P: X -> alpha
for production in G.Productions:
X = production.Left
alpha = production.Right
follow_X = follows[X]
# X -> zeta Y beta
# First(beta) - { epsilon } subset of Follow(Y)
# beta ->* epsilon or X -> zeta Y ? Follow(X) subset of Follow(Y)
for i, symbol in enumerate(alpha):
if symbol.IsNonTerminal:
follow_symbol = follows[symbol]
beta = alpha[i + 1:]
try:
first_beta = local_firsts[beta]
except KeyError:
first_beta = local_firsts[
beta] = UtilsParsers.ComputeLocalFirst(
firsts, beta)
change |= follow_symbol.update(first_beta)
if first_beta.contains_epsilon or len(beta) == 0:
change |= follow_symbol.update(follow_X)
return follows
def expand(item, firsts):
next_symbol = item.NextSymbol
if next_symbol is None or not next_symbol.IsNonTerminal:
return []
lookaheads = ContainerSet()
for preview in item.Preview():
lookaheads.hard_update(compute_local_first(firsts, preview))
assert not lookaheads.contains_epsilon
return [Item(prod, 0, lookaheads) for prod in next_symbol.productions]
def epsilon_closure(automaton, states):
Y = [s for s in states]
x = {s for s in states}
while Y:
z = Y.pop()
epsilon_transitions = automaton.epsilon_transitions(z)
for G in epsilon_transitions:
if G not in x:
x.add(G)
Y.append(G)
return ContainerSet(*x)
def epsilon_closure(automaton, states):
pending = [s for s in states]
closure = {s for s in states}
while pending:
state = pending.pop()
# Your code here
for new_state in automaton.epsilon_transitions(state):
closure.add(new_state)
pending.append(new_state)
return ContainerSet(*closure)
def epsilon_closure(automaton, states):
pending = [ s for s in states ] # equivalente a list(states) pero me gusta así :p
closure = { s for s in states } # equivalente a set(states) pero me gusta así :p
while pending:
state = pending.pop()
eps = automaton.epsilon_transitions(state)
for var in eps:
pending.append(var)
closure.add(var)
return ContainerSet(*closure)
def follows(self):
G = self.G
return {
G['E']:
ContainerSet(G[')'], G.EOF, contains_epsilon=False),
G['T']:
ContainerSet(G[')'], G['-'], G.EOF, G['+'],
contains_epsilon=False),
G['F']:
ContainerSet(G['-'],
G.EOF,
G['*'],
G['/'],
G[')'],
G['+'],
contains_epsilon=False),
G['X']:
ContainerSet(G[')'], G.EOF, contains_epsilon=False),
G['Y']:
ContainerSet(G[')'], G['-'], G.EOF, G['+'], contains_epsilon=False)
}
def _build_parsing_table(self):
G = self.G.AugmentedGrammar(True)
automaton = build_LR1_automaton(G)
centers = {}
for i, node in enumerate(automaton):
node.idx = i
try:
same_center = centers[node_centers(node)]
centers[node_centers(node)] = State(
compress(node.state.union(same_center.state)), True)
except KeyError:
centers[node_centers(node)] = node
centers[node_centers(node)].idx = i
self.automaton = automaton
self.centers = centers
Goto = {}
firsts = compute_firsts(G)
firsts[G.EOF] = ContainerSet(G.EOF)
for node in centers.values():
for symbol in G.terminals + G.nonTerminals:
temp = goto_lr1(node.state, symbol, firsts)
if not temp:
continue
Goto[(node.idx,
symbol)] = centers[node_centers(State(temp, True))]
for node in centers.values():
idx = node.idx
for item in node.state:
# Your code here!!!
# - Fill `self.Action` and `self.Goto` according to `item`)
# - Feel free to use `self._register(...)`)
if item.IsReduceItem:
production = item.production
if production.Left == G.startSymbol:
self._register(self.action, (idx, G.EOF), ("OK", None))
else:
for lookahead in item.lookaheads:
self._register(self.action, (idx, lookahead),
("REDUCE", production))
else:
next_symbol = item.NextSymbol
if next_symbol.IsTerminal:
self._register(
self.action, (idx, next_symbol),
("SHIFT", Goto[(node.idx, next_symbol)].idx))
else:
self._register(self.goto, (idx, next_symbol),
Goto[(node.idx, next_symbol)].idx)
def expand(item, firsts):
next_symbol = item.NextSymbol
if next_symbol is None or not next_symbol.IsNonTerminal:
return []
lookaheads = ContainerSet()
# Your code here!!! (Compute lookahead for child items)
for string in item.Preview():
lookaheads.update(compute_local_first(firsts, string))
assert not lookaheads.contains_epsilon
# Your code here!!! (Build and return child items)
return [Item(prod, 0, lookaheads) for prod in next_symbol.productions]
def compute_follows(G, firsts):
follows = { }
change = True
# init Follow(Vn)
for nonterminal in G.nonTerminals:
follows[nonterminal] = ContainerSet()
follows[G.startSymbol] = ContainerSet(G.EOF)
while change:
change = False
# P: X -> alpha
for production in G.Productions:
X = production.Left
alpha = production.Right
follow_X = follows[X]
###################################################
# X -> zeta Y beta
# First(beta) - { epsilon } subset of Follow(Y)
# beta ->* epsilon or X -> zeta Y ? Follow(X) subset of Follow(Y)
###################################################
for i in range(len(alpha)):
if alpha[i].IsTerminal:
continue
follow_y = follows[alpha[i]]
first_beta = compute_local_first(firsts,alpha[i+1:])
change|=follow_y.update(first_beta)
if(first_beta).contains_epsilon or len(alpha[i+1:]) == 0:
change |= follow_y.update(follow_X)
###################################################
# Follow(Vn)
return follows