def exp_probabilities(expansions, nonterminal="<symbol>"): probabilities = [exp_prob(expansion) for expansion in expansions] prob_dist = prob_distribution(probabilities, nonterminal) prob_mapping = {} for i in range(len(expansions)): expansion = exp_string(expansions[i]) prob_mapping[expansion] = prob_dist[i] return prob_mapping
def find_expansion(self, tree): symbol, children = tree applied_expansion = \ "".join([child_symbol for child_symbol, _ in children]) for expansion in self.grammar[symbol]: if exp_string(expansion) == applied_expansion: return expansion raise KeyError(symbol + ": did not find expansion " + repr(applied_expansion))
def expansion_to_children(expansion): # print("Converting " + repr(expansion)) # strings contains all substrings -- both terminals and nonterminals such # that ''.join(strings) == expansion expansion = exp_string(expansion) assert isinstance(expansion, str) if expansion == "": # Special case: epsilon expansion return [("", [])] strings = re.split(RE_NONTERMINAL, expansion) return [(s, None) if is_nonterminal(s) else (s, []) for s in strings if len(s) > 0]
def replace_symbol(grammar, old_symbol, new_symbol): """Return a grammar in which all occurrences of `old_symbol` are replaced by `new_symbol`""" new_grammar = {} for symbol in grammar: new_expansions = [] for expansion in grammar[symbol]: new_expansion_string = exp_string(expansion).replace(old_symbol, new_symbol) if len(exp_opts(expansion)) > 0: new_expansion = (new_expansion_string, exp_opts(expansion)) else: new_expansion = new_expansion_string new_expansions.append(new_expansion) new_grammar[symbol] = new_expansions # Remove unused parts for nonterminal in unreachable_nonterminals(new_grammar): del new_grammar[nonterminal] return new_grammar
"<digit>": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"], } if __name__ == "__main__": assert is_valid_grammar(PROBABILISTIC_EXPR_GRAMMAR, supported_opts={'prob'}) if __name__ == "__main__": PROBABILISTIC_EXPR_GRAMMAR["<leaddigit>"] if __name__ == "__main__": leaddigit_expansion = PROBABILISTIC_EXPR_GRAMMAR["<leaddigit>"][0] leaddigit_expansion if __name__ == "__main__": exp_string(leaddigit_expansion) def exp_prob(expansion): """Return the options of an expansion""" return exp_opt(expansion, 'prob') if __name__ == "__main__": exp_prob(leaddigit_expansion) if __name__ == "__main__": f = GrammarFuzzer(PROBABILISTIC_EXPR_GRAMMAR) f.fuzz() if __package__ is None or __package__ == "":