def test_epsilon_remove():
# epsilon test #1
grammar = Grammar()
S = grammar.add_nonterminal("S", True)
A, B, C = grammar.add_nonterminals("A B C")
a, b = grammar.add_terminals("a b")
S %= A + B
S %= C
A %= b + A + b
A %= grammar.epsilon
B %= b
C %= a
C %= b
new_grammar = __remove_epsilon_productions(grammar)
_, new_grammar = grammar_to_graph(new_grammar)
_graph = {}
_graph["S"] = [["A", "B"], ["C"], ["B"]]
_graph["A"] = [["b", "A", "b"], ["b", "b"]]
_graph["B"] = [["b"]]
_graph["C"] = [["a"], ["b"]]
assert (new_grammar == _graph)
# epsilon test #2
grammar = Grammar()
S = grammar.add_nonterminal("S", True)
A, B, C = grammar.add_nonterminals("A B C")
a, b = grammar.add_terminals("a b")
S %= A + B
S %= C
A %= b + A + b
A %= grammar.epsilon
B %= b
B %= grammar.epsilon
C %= a
C %= b
new_grammar = __remove_epsilon_productions(grammar)
_, new_grammar = grammar_to_graph(new_grammar)
_graph = {}
_graph["S"] = [["A", "B"], ["C"], ["B"], ["A"], []]
_graph["A"] = [["b", "A", "b"], ["b", "b"]]
_graph["B"] = [["b"]]
_graph["C"] = [["a"], ["b"]]
assert (new_grammar == _graph)
def __remove_epsilon_productions(grammar):
S, d = grammar_to_graph(grammar)
nonterminals = [t.name for t in grammar.nonterminals]
nullable = {}
nullable = __find_nullable_nonterminals(d, nullable, S, nonterminals)
for key, value in d.items():
new_value = [v for v in value]
for sentence in value:
if sentence == []:
new_value.remove(sentence)
for i in range(0, len(sentence)):
if sentence[i] in nonterminals and nullable[sentence[i]]:
new_sentence = sentence[0:i] + sentence[i +
1:len(sentence)]
if not new_sentence in new_value:
new_value.append(new_sentence)
d[key] = new_value
if nullable[S]:
d[S].append([])
return graph_to_grammar(S, d)
def remove_left_recursion(grammar):
new_grammar = __remove_epsilon_productions(grammar)
new_grammar = remove_unit_prods(new_grammar)
nonterminals = [t.name for t in new_grammar.nonterminals]
S, d = grammar_to_graph(new_grammar)
for i in range(0, len(nonterminals)):
for j in range(0, i):
for sentence in d[nonterminals[i]]:
if sentence[0] == nonterminals[j]:
d[nonterminals[i]].remove(sentence)
remove_first = sentence[1:len(sentence)]
for sentence in d[nonterminals[j]]:
new_sentence = []
for item in sentence:
new_sentence.append(item)
for item in remove_first:
new_sentence.append(item)
d[nonterminals[i]].append(new_sentence)
d = __remove_inmediate_left_recursion(d)
return graph_to_grammar(S, d)
def test_remove_unreachable_prods():
grammar = Grammar()
S = grammar.add_nonterminal("S", True)
A, B, C = grammar.add_nonterminals("A B C")
a, b = grammar.add_terminals("a b")
S %= a
A %= B + a + b
A %= B + a + a
A %= B
new_grammar = remove_common_prefixes(grammar)
S, new_grammar = grammar_to_graph(new_grammar)
_graph = {}
_graph["S"] = [["a"]]
_graph["A"] = [["B", "A''"]]
_graph["A'"] = [["b"], ["a"]]
_graph["A''"] = [[], ["a", "A'"]]
print(new_grammar)
print(_graph)
assert new_grammar == _graph
def test_general_recursion_remove():
grammar = Grammar()
S = grammar.add_nonterminal("S", True)
A, B, C = grammar.add_nonterminals("A B C")
a, b = grammar.add_terminals("a b")
S %= A + b
S %= C
A %= B + a
B %= S + b
C %= b
new_grammar = remove_left_recursion(grammar)
_, new_grammar = grammar_to_graph(new_grammar)
_graph = {}
_graph["S"] = [["b"], ["A", "b"]]
_graph["A"] = [["B", "a"]]
_graph["B"] = [["b", "b", "B'"]]
_graph["B'"] = [["a", "b", "b", "B'"], []]
_graph["C"] = [["b"]]
print(_graph)
print(new_grammar)
assert (new_grammar == _graph)
def test_direct_recursion_remove():
grammar = Grammar()
S = grammar.add_nonterminal("S", True)
A, B, C = grammar.add_nonterminals("A B C")
a, b = grammar.add_terminals("a b")
S %= A + B
S %= C
A %= A + b
A %= a
B %= b
C %= a
C %= b
_, G = grammar_to_graph(grammar)
new_grammar = __remove_inmediate_left_recursion(G)
_graph = {}
_graph["S"] = [["A", "B"], ["C"]]
_graph["A"] = [["a", "A'"]]
_graph["A'"] = [["b", "A'"], []]
_graph["B"] = [["b"]]
_graph["C"] = [["a"], ["b"]]
assert (new_grammar == _graph)
def test_grammar_to_graph():
grammar = Grammar()
S = grammar.add_nonterminal("S", True)
A, B, C, X, Y = grammar.add_nonterminals("A B C X Y")
a, b, d, e = grammar.add_terminals("a b d e")
S %= A + B
S %= C
A %= C
A %= d
B %= Y
C %= a
C %= b
C %= X
X %= d
X %= e
Y %= e
_S, graph = grammar_to_graph(grammar)
_graph = {}
_graph["S"] = [["A", "B"], ["C"]]
_graph["A"] = [["C"], ["d"]]
_graph["B"] = [["Y"]]
_graph["C"] = [["a"], ["b"], ["X"]]
_graph["X"] = [["d"], ["e"]]
_graph["Y"] = [["e"]]
assert (graph == _graph)
def test_remove_left_recursion_2():
grammar = Grammar()
A = grammar.add_nonterminal("A", True)
B, C, D, E, F = grammar.add_nonterminals("B C D E F")
a, b, c, d = grammar.add_terminals("a b c d")
A %= b + B
A %= c + C
A %= d + D
B %= c + C
B %= grammar.epsilon
C %= c + c + c
C %= A
C %= a
C %= b
C %= grammar.epsilon
D %= d
D %= b
D %= E
D %= grammar.epsilon
E %= F
E %= C
F %= D
new_grammar = remove_left_recursion(grammar)
_, new_grammar = grammar_to_graph(new_grammar)
_graph = {}
_graph["S"] = [["b"], ["A", "b"]]
_graph["A"] = [["B", "a"]]
_graph["B"] = [["b", "b", "B'"]]
_graph["B'"] = [["a", "b", "b", "B'"], []]
_graph["C"] = [["b"]]
print(_graph)
print(new_grammar)
assert True
# A -> b B | c C | d D
# B -> c C | eps
# C -> c c c | A | a | b | eps
# D -> d | b | E | eps
# E -> F | C
# F -> D
def remove_unit_prods(G: Grammar):
S, d = grammar_to_graph(G)
nonterminals = [t.name for t in G.nonterminals]
new_d = {}
u = __find_unitary_pairs(d, nonterminals)
for pair in u:
for sentence in d[pair[1]]:
if not (len(sentence) == 1 and sentence[0] in nonterminals):
try:
if not sentence in new_d[pair[0]]:
new_d[pair[0]].append(sentence)
except KeyError:
new_d[pair[0]] = [sentence]
return graph_to_grammar(S, new_d)
def remove_unreachable_prods(G: Grammar):
S, d = grammar_to_graph(G)
nonterminals = [t.name for t in G.nonterminals]
mark = {}
for p in d.keys():
mark[p[0]] = False
__overlook(d, mark, nonterminals, S)
for t in nonterminals:
if not mark[t]:
_ = d.pop(t)
return graph_to_grammar(S, d)
def remove_common_prefixes(grammar: Grammar):
S, d = grammar_to_graph(grammar)
nonterminals = [nt.name for nt in grammar.nonterminals]
for A in nonterminals:
try:
_ = d[A]
except KeyError:
continue
count = 1
trie = Trie((A, d[A]))
prefix_nodes = [n for n in trie.prefix_nodes]
prefix_nodes.sort(key=lambda x: x.depth, reverse=True)
for (n) in (
prefix_nodes
): # get the longest common prefix among the productions be the prefix α
productions = trie.get_node_productions(
n) # get all the productions with that prefix
n.children.clear()
# A -> α ω1 | α ω2 | ... | α ωΝ
# replace those productions with
# A -> αA'
# A' -> ω1 | ω2 | ... | ωΝ
A_new = A + ("'" * count)
count += 1
d[A] = [productions[0][0:n.depth + 1] + [A_new]]
n.productions = [d[A][-1]]
for p in productions:
if len(p) > n.depth + 1:
try:
d[A_new].append(p[n.depth + 1:])
except KeyError:
d[A_new] = [p[n.depth + 1:]]
else:
try:
d[A_new].append([])
except KeyError:
d[A_new] = [[]]
print(d)
return graph_to_grammar(S, d)
def test_remove_unitary_prods_2():
grammar = Grammar()
A = grammar.add_nonterminal("A", True)
B, C, D, E, F = grammar.add_nonterminals("B C D E F")
a, b, c, d = grammar.add_terminals("a b c d")
A %= b + B
A %= c + C
A %= d + D
B %= c + C
B %= grammar.epsilon
C %= c + c + c
C %= A
C %= a
C %= b
C %= grammar.epsilon
D %= d
D %= b
D %= E
D %= grammar.epsilon
E %= F
E %= C
F %= D
new_grammar = remove_unit_prods(grammar)
_, new_grammar = grammar_to_graph(new_grammar)
_graph = {}
_graph["S"] = [["a"]]
_graph["A"] = [["B", "A''"]]
_graph["A'"] = [["b"], ["a"]]
_graph["A''"] = [[], ["a", "A'"]]
assert True
def test_remove_unreachable_prods():
grammar = Grammar()
S = grammar.add_nonterminal("S", True)
A, B, C = grammar.add_nonterminals("A B C")
a, b = grammar.add_terminals("a b")
S %= A
S %= b
A %= B
B %= a
C %= a
new_grammar = remove_unreachable_prods(grammar)
_, new_grammar = grammar_to_graph(new_grammar)
_graph = {}
_graph["S"] = [["A"], ["b"]]
_graph["A"] = [["B"]]
_graph["B"] = [["a"]]
assert new_grammar == _graph
grammar = Grammar()
S = grammar.add_nonterminal("S", True)
A, B, C = grammar.add_nonterminals("A B C")
a, b = grammar.add_terminals("a b")
S %= A + b
S %= C
A %= B + a
B %= S + b
C %= b
new_grammar = remove_unreachable_prods(grammar)
_, new_grammar = grammar_to_graph(new_grammar)
_graph = {}
_graph["S"] = [["A", "b"], ["C"]]
_graph["A"] = [["B", "a"]]
_graph["B"] = [["S", "b"]]
_graph["C"] = [["b"]]
assert new_grammar == _graph
grammar = Grammar()
S = grammar.add_nonterminal("S", True)
A, B, C = grammar.add_nonterminals("A B C")
a, b = grammar.add_terminals("a b")
S %= A + B
S %= b
S %= a
A %= B + a
B %= S + b
C %= b
C %= a
new_grammar = remove_unreachable_prods(grammar)
_, new_grammar = grammar_to_graph(new_grammar)
_graph = {}
_graph["S"] = [["A", "B"], ["b"], ["a"]]
_graph["A"] = [["B", "a"]]
_graph["B"] = [["S", "b"]]
assert new_grammar == _graph
# _graph = {}
# _graph["S"] = [["a"], ["b"], ["C", "b"]]
# _graph["A"] = [["a"], ["C", "b"]]
# _graph["B"] = [["a"], ["C", "b"]]
# _graph["C"] = [["a"]]
grammar = Grammar()
S = grammar.add_nonterminal("S", True)
A, B, C = grammar.add_nonterminals("A B C")
a, b = grammar.add_terminals("a b")
S %= a
S %= b
S %= C + b
A %= a
A %= C + b
B %= a
B %= C + b
C %= a
new_grammar = remove_unreachable_prods(grammar)
_, new_grammar = grammar_to_graph(new_grammar)
_graph = {}
_graph["S"] = [["a"], ["b"], ["C", "b"]]
_graph["C"] = [["a"]]
print(_graph)
print(new_grammar)
assert new_grammar == _graph
def test_remove_unit_prods():
grammar = Grammar()
S = grammar.add_nonterminal("S", True)
A, B, C = grammar.add_nonterminals("A B C")
a, b = grammar.add_terminals("a b")
S %= A
S %= b
A %= B
A %= a
B %= a
B %= C + b
C %= a
new_grammar = remove_unnecesary_productions(grammar)
_, new_grammar = grammar_to_graph(new_grammar)
_graph = {}
_graph["S"] = [["a"], ["b"], ["C", "b"]]
_graph["C"] = [["a"]]
assert new_grammar == _graph
grammar = Grammar()
S = grammar.add_nonterminal("S", True)
A, B, C = grammar.add_nonterminals("A B C")
a, b = grammar.add_terminals("a b")
S %= A + B
S %= C
A %= A + b
A %= a
B %= b
C %= a
C %= b
new_grammar = remove_unnecesary_productions(grammar)
_, new_grammar = grammar_to_graph(new_grammar)
# _graph = {}
# _graph["S"] = [["b"], ["a"], ["A", "B"]]
# _graph["A"] = [["A", "b"], ["a"]]
# _graph["B"] = [["b"]]
# print(_graph)
# print(new_grammar)
# assert (new_grammar == _graph)
grammar = Grammar()
S = grammar.add_nonterminal("S", True)
A, B, C = grammar.add_nonterminals("A B C")
a, b = grammar.add_terminals("a b")
S %= A
S %= b
A %= B
A %= a
B %= a
B %= C + b
C %= a
new_grammar = remove_unnecesary_productions(grammar)
_, new_grammar = grammar_to_graph(new_grammar)
_graph = {}
_graph["S"] = [["a"], ["b"], ["C", "b"]]
_graph["C"] = [["a"]]
assert new_grammar == _graph
grammar = Grammar()
S = grammar.add_nonterminal("S", True)
X, Y = grammar.add_nonterminals("X Y")
a, b = grammar.add_terminals("a b")
S %= X + a
S %= Y
X %= b
Y %= a
new_grammar = remove_unnecesary_productions(grammar)
_, new_grammar = grammar_to_graph(new_grammar)
_graph = {}
_graph["S"] = [["X", "a"], ["b"]]
_graph["X"] = [["b"]]
