def test_epsilon_remove(): # epsilon test #1 grammar = Grammar() S = grammar.add_nonterminal("S", True) A, B, C = grammar.add_nonterminals("A B C") a, b = grammar.add_terminals("a b") S %= A + B S %= C A %= b + A + b A %= grammar.epsilon B %= b C %= a C %= b new_grammar = __remove_epsilon_productions(grammar) _, new_grammar = grammar_to_graph(new_grammar) _graph = {} _graph["S"] = [["A", "B"], ["C"], ["B"]] _graph["A"] = [["b", "A", "b"], ["b", "b"]] _graph["B"] = [["b"]] _graph["C"] = [["a"], ["b"]] assert (new_grammar == _graph) # epsilon test #2 grammar = Grammar() S = grammar.add_nonterminal("S", True) A, B, C = grammar.add_nonterminals("A B C") a, b = grammar.add_terminals("a b") S %= A + B S %= C A %= b + A + b A %= grammar.epsilon B %= b B %= grammar.epsilon C %= a C %= b new_grammar = __remove_epsilon_productions(grammar) _, new_grammar = grammar_to_graph(new_grammar) _graph = {} _graph["S"] = [["A", "B"], ["C"], ["B"], ["A"], []] _graph["A"] = [["b", "A", "b"], ["b", "b"]] _graph["B"] = [["b"]] _graph["C"] = [["a"], ["b"]] assert (new_grammar == _graph)
def __remove_epsilon_productions(grammar): S, d = grammar_to_graph(grammar) nonterminals = [t.name for t in grammar.nonterminals] nullable = {} nullable = __find_nullable_nonterminals(d, nullable, S, nonterminals) for key, value in d.items(): new_value = [v for v in value] for sentence in value: if sentence == []: new_value.remove(sentence) for i in range(0, len(sentence)): if sentence[i] in nonterminals and nullable[sentence[i]]: new_sentence = sentence[0:i] + sentence[i + 1:len(sentence)] if not new_sentence in new_value: new_value.append(new_sentence) d[key] = new_value if nullable[S]: d[S].append([]) return graph_to_grammar(S, d)
def remove_left_recursion(grammar): new_grammar = __remove_epsilon_productions(grammar) new_grammar = remove_unit_prods(new_grammar) nonterminals = [t.name for t in new_grammar.nonterminals] S, d = grammar_to_graph(new_grammar) for i in range(0, len(nonterminals)): for j in range(0, i): for sentence in d[nonterminals[i]]: if sentence[0] == nonterminals[j]: d[nonterminals[i]].remove(sentence) remove_first = sentence[1:len(sentence)] for sentence in d[nonterminals[j]]: new_sentence = [] for item in sentence: new_sentence.append(item) for item in remove_first: new_sentence.append(item) d[nonterminals[i]].append(new_sentence) d = __remove_inmediate_left_recursion(d) return graph_to_grammar(S, d)
def test_remove_unreachable_prods(): grammar = Grammar() S = grammar.add_nonterminal("S", True) A, B, C = grammar.add_nonterminals("A B C") a, b = grammar.add_terminals("a b") S %= a A %= B + a + b A %= B + a + a A %= B new_grammar = remove_common_prefixes(grammar) S, new_grammar = grammar_to_graph(new_grammar) _graph = {} _graph["S"] = [["a"]] _graph["A"] = [["B", "A''"]] _graph["A'"] = [["b"], ["a"]] _graph["A''"] = [[], ["a", "A'"]] print(new_grammar) print(_graph) assert new_grammar == _graph
def test_general_recursion_remove(): grammar = Grammar() S = grammar.add_nonterminal("S", True) A, B, C = grammar.add_nonterminals("A B C") a, b = grammar.add_terminals("a b") S %= A + b S %= C A %= B + a B %= S + b C %= b new_grammar = remove_left_recursion(grammar) _, new_grammar = grammar_to_graph(new_grammar) _graph = {} _graph["S"] = [["b"], ["A", "b"]] _graph["A"] = [["B", "a"]] _graph["B"] = [["b", "b", "B'"]] _graph["B'"] = [["a", "b", "b", "B'"], []] _graph["C"] = [["b"]] print(_graph) print(new_grammar) assert (new_grammar == _graph)
def test_direct_recursion_remove(): grammar = Grammar() S = grammar.add_nonterminal("S", True) A, B, C = grammar.add_nonterminals("A B C") a, b = grammar.add_terminals("a b") S %= A + B S %= C A %= A + b A %= a B %= b C %= a C %= b _, G = grammar_to_graph(grammar) new_grammar = __remove_inmediate_left_recursion(G) _graph = {} _graph["S"] = [["A", "B"], ["C"]] _graph["A"] = [["a", "A'"]] _graph["A'"] = [["b", "A'"], []] _graph["B"] = [["b"]] _graph["C"] = [["a"], ["b"]] assert (new_grammar == _graph)
def test_grammar_to_graph(): grammar = Grammar() S = grammar.add_nonterminal("S", True) A, B, C, X, Y = grammar.add_nonterminals("A B C X Y") a, b, d, e = grammar.add_terminals("a b d e") S %= A + B S %= C A %= C A %= d B %= Y C %= a C %= b C %= X X %= d X %= e Y %= e _S, graph = grammar_to_graph(grammar) _graph = {} _graph["S"] = [["A", "B"], ["C"]] _graph["A"] = [["C"], ["d"]] _graph["B"] = [["Y"]] _graph["C"] = [["a"], ["b"], ["X"]] _graph["X"] = [["d"], ["e"]] _graph["Y"] = [["e"]] assert (graph == _graph)
def test_remove_left_recursion_2(): grammar = Grammar() A = grammar.add_nonterminal("A", True) B, C, D, E, F = grammar.add_nonterminals("B C D E F") a, b, c, d = grammar.add_terminals("a b c d") A %= b + B A %= c + C A %= d + D B %= c + C B %= grammar.epsilon C %= c + c + c C %= A C %= a C %= b C %= grammar.epsilon D %= d D %= b D %= E D %= grammar.epsilon E %= F E %= C F %= D new_grammar = remove_left_recursion(grammar) _, new_grammar = grammar_to_graph(new_grammar) _graph = {} _graph["S"] = [["b"], ["A", "b"]] _graph["A"] = [["B", "a"]] _graph["B"] = [["b", "b", "B'"]] _graph["B'"] = [["a", "b", "b", "B'"], []] _graph["C"] = [["b"]] print(_graph) print(new_grammar) assert True # A -> b B | c C | d D # B -> c C | eps # C -> c c c | A | a | b | eps # D -> d | b | E | eps # E -> F | C # F -> D
def remove_unit_prods(G: Grammar): S, d = grammar_to_graph(G) nonterminals = [t.name for t in G.nonterminals] new_d = {} u = __find_unitary_pairs(d, nonterminals) for pair in u: for sentence in d[pair[1]]: if not (len(sentence) == 1 and sentence[0] in nonterminals): try: if not sentence in new_d[pair[0]]: new_d[pair[0]].append(sentence) except KeyError: new_d[pair[0]] = [sentence] return graph_to_grammar(S, new_d)
def remove_unreachable_prods(G: Grammar): S, d = grammar_to_graph(G) nonterminals = [t.name for t in G.nonterminals] mark = {} for p in d.keys(): mark[p[0]] = False __overlook(d, mark, nonterminals, S) for t in nonterminals: if not mark[t]: _ = d.pop(t) return graph_to_grammar(S, d)
def remove_common_prefixes(grammar: Grammar): S, d = grammar_to_graph(grammar) nonterminals = [nt.name for nt in grammar.nonterminals] for A in nonterminals: try: _ = d[A] except KeyError: continue count = 1 trie = Trie((A, d[A])) prefix_nodes = [n for n in trie.prefix_nodes] prefix_nodes.sort(key=lambda x: x.depth, reverse=True) for (n) in ( prefix_nodes ): # get the longest common prefix among the productions be the prefix α productions = trie.get_node_productions( n) # get all the productions with that prefix n.children.clear() # A -> α ω1 | α ω2 | ... | α ωΝ # replace those productions with # A -> αA' # A' -> ω1 | ω2 | ... | ωΝ A_new = A + ("'" * count) count += 1 d[A] = [productions[0][0:n.depth + 1] + [A_new]] n.productions = [d[A][-1]] for p in productions: if len(p) > n.depth + 1: try: d[A_new].append(p[n.depth + 1:]) except KeyError: d[A_new] = [p[n.depth + 1:]] else: try: d[A_new].append([]) except KeyError: d[A_new] = [[]] print(d) return graph_to_grammar(S, d)
def test_remove_unitary_prods_2(): grammar = Grammar() A = grammar.add_nonterminal("A", True) B, C, D, E, F = grammar.add_nonterminals("B C D E F") a, b, c, d = grammar.add_terminals("a b c d") A %= b + B A %= c + C A %= d + D B %= c + C B %= grammar.epsilon C %= c + c + c C %= A C %= a C %= b C %= grammar.epsilon D %= d D %= b D %= E D %= grammar.epsilon E %= F E %= C F %= D new_grammar = remove_unit_prods(grammar) _, new_grammar = grammar_to_graph(new_grammar) _graph = {} _graph["S"] = [["a"]] _graph["A"] = [["B", "A''"]] _graph["A'"] = [["b"], ["a"]] _graph["A''"] = [[], ["a", "A'"]] assert True
def test_remove_unreachable_prods(): grammar = Grammar() S = grammar.add_nonterminal("S", True) A, B, C = grammar.add_nonterminals("A B C") a, b = grammar.add_terminals("a b") S %= A S %= b A %= B B %= a C %= a new_grammar = remove_unreachable_prods(grammar) _, new_grammar = grammar_to_graph(new_grammar) _graph = {} _graph["S"] = [["A"], ["b"]] _graph["A"] = [["B"]] _graph["B"] = [["a"]] assert new_grammar == _graph grammar = Grammar() S = grammar.add_nonterminal("S", True) A, B, C = grammar.add_nonterminals("A B C") a, b = grammar.add_terminals("a b") S %= A + b S %= C A %= B + a B %= S + b C %= b new_grammar = remove_unreachable_prods(grammar) _, new_grammar = grammar_to_graph(new_grammar) _graph = {} _graph["S"] = [["A", "b"], ["C"]] _graph["A"] = [["B", "a"]] _graph["B"] = [["S", "b"]] _graph["C"] = [["b"]] assert new_grammar == _graph grammar = Grammar() S = grammar.add_nonterminal("S", True) A, B, C = grammar.add_nonterminals("A B C") a, b = grammar.add_terminals("a b") S %= A + B S %= b S %= a A %= B + a B %= S + b C %= b C %= a new_grammar = remove_unreachable_prods(grammar) _, new_grammar = grammar_to_graph(new_grammar) _graph = {} _graph["S"] = [["A", "B"], ["b"], ["a"]] _graph["A"] = [["B", "a"]] _graph["B"] = [["S", "b"]] assert new_grammar == _graph # _graph = {} # _graph["S"] = [["a"], ["b"], ["C", "b"]] # _graph["A"] = [["a"], ["C", "b"]] # _graph["B"] = [["a"], ["C", "b"]] # _graph["C"] = [["a"]] grammar = Grammar() S = grammar.add_nonterminal("S", True) A, B, C = grammar.add_nonterminals("A B C") a, b = grammar.add_terminals("a b") S %= a S %= b S %= C + b A %= a A %= C + b B %= a B %= C + b C %= a new_grammar = remove_unreachable_prods(grammar) _, new_grammar = grammar_to_graph(new_grammar) _graph = {} _graph["S"] = [["a"], ["b"], ["C", "b"]] _graph["C"] = [["a"]] print(_graph) print(new_grammar) assert new_grammar == _graph
def test_remove_unit_prods(): grammar = Grammar() S = grammar.add_nonterminal("S", True) A, B, C = grammar.add_nonterminals("A B C") a, b = grammar.add_terminals("a b") S %= A S %= b A %= B A %= a B %= a B %= C + b C %= a new_grammar = remove_unnecesary_productions(grammar) _, new_grammar = grammar_to_graph(new_grammar) _graph = {} _graph["S"] = [["a"], ["b"], ["C", "b"]] _graph["C"] = [["a"]] assert new_grammar == _graph grammar = Grammar() S = grammar.add_nonterminal("S", True) A, B, C = grammar.add_nonterminals("A B C") a, b = grammar.add_terminals("a b") S %= A + B S %= C A %= A + b A %= a B %= b C %= a C %= b new_grammar = remove_unnecesary_productions(grammar) _, new_grammar = grammar_to_graph(new_grammar) # _graph = {} # _graph["S"] = [["b"], ["a"], ["A", "B"]] # _graph["A"] = [["A", "b"], ["a"]] # _graph["B"] = [["b"]] # print(_graph) # print(new_grammar) # assert (new_grammar == _graph) grammar = Grammar() S = grammar.add_nonterminal("S", True) A, B, C = grammar.add_nonterminals("A B C") a, b = grammar.add_terminals("a b") S %= A S %= b A %= B A %= a B %= a B %= C + b C %= a new_grammar = remove_unnecesary_productions(grammar) _, new_grammar = grammar_to_graph(new_grammar) _graph = {} _graph["S"] = [["a"], ["b"], ["C", "b"]] _graph["C"] = [["a"]] assert new_grammar == _graph grammar = Grammar() S = grammar.add_nonterminal("S", True) X, Y = grammar.add_nonterminals("X Y") a, b = grammar.add_terminals("a b") S %= X + a S %= Y X %= b Y %= a new_grammar = remove_unnecesary_productions(grammar) _, new_grammar = grammar_to_graph(new_grammar) _graph = {} _graph["S"] = [["X", "a"], ["b"]] _graph["X"] = [["b"]]