def add_tree(self, tree):
(symbol, children) = tree
if not is_nonterminal(symbol):
return
direct_children = [(symbol, None) if is_nonterminal(symbol) else
(symbol, []) for symbol, c in children]
self.add_coverage(symbol, direct_children)
for c in children:
self.add_tree(c)
def insert_into_tree(self, my_tree, pair):
var, values, my_scope = my_tree
(nt_var, nt_seq), (v, v_scope) = pair
applied = False
for i, value_ in enumerate(values):
key, arr, scope = value_
self.log(2, "-> [%d] %s" % (i, repr(value_)))
if is_nonterminal(key):
applied = self.insert_into_tree(value_, pair)
if applied:
break
else:
if v_scope != scope:
if nt_seq > scope:
continue
if not v or v not in key:
continue
prefix_k_suffix = [
(nt_var, [(v, [], nt_seq)],
scope) if i == 1 else (e, [], scope)
for i, e in enumerate(key.partition(v))
if e
]
del values[i]
for j, rep in enumerate(prefix_k_suffix):
values.insert(j + i, rep)
applied = True
self.log(2, " > %s" % (repr([i[0] for i in prefix_k_suffix])))
break
return applied
def insert_into_tree(self, my_tree, pair):
var, values = my_tree
k, v = pair
self.log(1, "- Node: %s\t\t? (%s:%s)" % (var, k, repr(v)))
applied = False
for i, value_ in enumerate(values):
value, arr = value_
self.log(2, "-> [%d] %s" % (i, repr(value)))
if is_nonterminal(value):
applied = self.insert_into_tree(value_, pair)
if applied:
break
elif v in value:
prefix_k_suffix = [
(k, [[v, []]]) if i == 1 else (e, [])
for i, e in enumerate(value.partition(v))
if e
]
del values[i]
for j, rep in enumerate(prefix_k_suffix):
values.insert(j + i, rep)
applied = True
self.log(2, " > %s" % (repr([i[0] for i in prefix_k_suffix])))
break
else:
continue
return applied
def apply_result(self, result, children):
if isinstance(result, str):
children = [(result, [])]
elif isinstance(result, list):
symbol_indexes = [
i for i, c in enumerate(children) if is_nonterminal(c[0])
]
for index, value in enumerate(result):
if value is not None:
child_index = symbol_indexes[index]
if not isinstance(value, str):
value = repr(value)
if self.log:
print("Replacing",
all_terminals(children[child_index]), "by",
value)
# children[child_index] = (value, [])
child_symbol, _ = children[child_index]
children[child_index] = (child_symbol, [(value, [])])
elif result is None:
pass
elif isinstance(result, bool):
pass
else:
if self.log:
print("Replacing",
"".join([all_terminals(c) for c in children]), "by",
result)
children = [(repr(result), [])]
return children
def tree_to_grammar(self, tree):
node, children = tree
one_alt = [ck for ck, gc in children]
hsh = {node: [one_alt] if one_alt else []}
for child in children:
if not is_nonterminal(child[0]):
continue
chsh = self.tree_to_grammar(child)
for k in chsh:
if k not in hsh:
hsh[k] = chsh[k]
else:
hsh[k].extend(chsh[k])
return hsh
def expansion_to_children(expansion):
# print("Converting " + repr(expansion))
# strings contains all substrings -- both terminals and nonterminals such
# that ''.join(strings) == expansion
expansion = exp_string(expansion)
assert isinstance(expansion, str)
if expansion == "": # Special case: epsilon expansion
return [("", [])]
strings = re.split(RE_NONTERMINAL, expansion)
return [(s, None) if is_nonterminal(s) else (s, []) for s in strings
if len(s) > 0]
def tree_to_grammar(self, tree):
key, children, scope = tree
one_alt = [ckey for ckey, gchildren, cscope in children if ckey != key]
hsh = {key: [one_alt] if one_alt else []}
for child in children:
(ckey, _gc, _cscope) = child
if not is_nonterminal(ckey):
continue
chsh = self.tree_to_grammar(child)
for k in chsh:
if k not in hsh:
hsh[k] = chsh[k]
else:
hsh[k].extend(chsh[k])
return hsh
def expansion_to_children(expansion):
# print("Converting " + repr(expansion))
# strings contains all substrings -- both terminals and nonterminals such
# that ''.join(strings) == expansion
# See nonterminals() in Grammars.py
if isinstance(expansion, tuple):
expansion = expansion[0]
if expansion == "": # Special case: empty expansion
return [("", [])]
strings = re.split(RE_NONTERMINAL, expansion)
return [(s, None) if is_nonterminal(s) else (s, [])
for s in strings if len(s) > 0]
def get_replacements(self, grammar):
replacements = {}
for k in grammar:
if k == START_SYMBOL:
continue
alts = grammar[k]
if len(set([str(i) for i in alts])) != 1:
continue
rule = alts[0]
if len(rule) != 1:
continue
tok = rule[0]
if not is_nonterminal(tok):
continue
replacements[k] = tok
return replacements
def nt_var(self, var):
return var if is_nonterminal(var) else to_nonterminal(var)
def fsm_last_state_symbol(self, tree):
"""Return current (expected) state symbol"""
for state in reversed(self.fsm_path(tree)):
if is_nonterminal(state):
return state
assert False
def tree_to_string(tree):
symbol, children, *_ = tree
if children:
return ''.join(tree_to_string(c) for c in children)
else:
return '' if is_nonterminal(symbol) else symbol