def apply_result(self, result, children):
if isinstance(result, str):
children = [(result, [])]
elif isinstance(result, list):
symbol_indexes = [
i for i, c in enumerate(children) if is_nonterminal(c[0])
]
for index, value in enumerate(result):
if value is not None:
child_index = symbol_indexes[index]
if not isinstance(value, str):
value = repr(value)
if self.log:
print("Replacing",
all_terminals(children[child_index]), "by",
value)
# children[child_index] = (value, [])
child_symbol, _ = children[child_index]
children[child_index] = (child_symbol, [(value, [])])
elif result is None:
pass
elif isinstance(result, bool):
pass
else:
if self.log:
print("Replacing",
"".join([all_terminals(c) for c in children]), "by",
result)
children = [(repr(result), [])]
return children
def run_post_functions(self, tree, depth=float("inf")):
symbol, children = tree
if children == []:
return True, children # Terminal symbol
try:
expansion = self.find_expansion(tree)
except KeyError:
# Expansion (no longer) found - ignore
return True, children
result = True
function = exp_post_expansion_function(expansion)
if function is not None:
result = self.eval_function(tree, function)
if isinstance(result, bool) and not result:
if self.log:
print(all_terminals(tree), "did not satisfy", symbol,
"constraint")
return False, children
children = self.apply_result(result, children)
if depth > 0:
for c in children:
result, _ = self.run_post_functions(c, depth - 1)
if isinstance(result, bool) and not result:
return False, children
return result, children
def run_post_functions_locally(self, new_tree):
symbol, _ = new_tree
result, children = self.run_post_functions(new_tree, depth=0)
if not isinstance(result, bool) or result:
# No constraints, or constraint satisfied
# children = self.apply_result(result, children)
new_tree = (symbol, children)
return new_tree
# Replace tree by unexpanded symbol and try again
if self.log:
print(
all_terminals(new_tree),
"did not satisfy",
symbol,
"constraint")
if self.replacement_attempts_counter > 0:
if self.log:
print("Trying another expansion")
self.replacement_attempts_counter -= 1
return (symbol, None)
if self.log:
print("Starting from scratch")
raise RestartExpansionException
def expansion_key(symbol, expansion):
"""Convert (symbol, children) into a key. `children` can be an expansion string or a derivation tree."""
if isinstance(expansion, tuple):
expansion = expansion[0]
if not isinstance(expansion, str):
children = expansion
expansion = all_terminals((symbol, children))
return symbol + " -> " + expansion
def reduce_tree(self, tree):
# Find possible reductions
smallest_tree = tree
tree_reductions = self.reductions(tree)
print("Alternatives: " + queue_to_string(tree_reductions))
while len(tree_reductions) > 0:
t = tree_reductions[0]
tree_reductions = tree_reductions[1:]
s = all_terminals(t)
if self.test(s) == Runner.FAIL:
# Found new smallest tree; try to reduce that one further
smallest_tree = t
tree_reductions = self.reductions(t)
tree_reductions.sort(key=lambda tree: -number_of_nodes(tree))
print("New smallest tree: " + all_terminals(smallest_tree))
return smallest_tree
def eval_function(self, tree, function):
symbol, children = tree
assert callable(function)
args = []
for (symbol, exp) in children:
if exp != [] and exp is not None:
symbol_value = all_terminals((symbol, exp))
args.append(symbol_value)
result = function(*args)
if self.log:
print(repr(function) + repr(tuple(args)), "=", repr(result))
return result
def choose_node_expansion(self, node, possible_children):
(symbol, tree) = node
expansions = self.grammar[symbol]
probabilities = exp_probabilities(expansions)
weights = []
for child in possible_children:
expansion = all_terminals((node, child))
child_weight = probabilities[expansion]
if self.log:
print(repr(expansion), "p =", child_weight)
weights.append(child_weight)
if sum(weights) == 0:
# No alternative (probably expanding at minimum cost)
weights = None
return random.choices(range(len(possible_children)),
weights=weights)[0]
def reduce_subtree(self, tree, subtree, depth=-1):
symbol, children = subtree
if len(children) == 0:
return False
if self.log_reduce:
print("Reducing", all_terminals(subtree), "with depth", depth)
reduced = False
while True:
reduced_child = False
for i, child in enumerate(children):
(child_symbol, _) = child
for reduction in self.symbol_reductions(
child, child_symbol, depth):
if number_of_nodes(reduction) >= number_of_nodes(child):
continue
# Try this reduction
if self.log_reduce:
print(
"Replacing",
all_terminals(
children[i]),
"by",
all_terminals(reduction))
children[i] = reduction
if self.test(all_terminals(tree)) == Runner.FAIL:
# Success
if self.log_reduce:
print("New tree:", all_terminals(tree))
reduced = reduced_child = True
break
else:
# Didn't work out - restore
children[i] = child
if not reduced_child:
if self.log_reduce:
print("Tried all alternatives for", all_terminals(subtree))
break
# Run recursively
for c in children:
if self.reduce_subtree(tree, c, depth):
reduced = True
return reduced
def shrink_tree(tree):
name, children = tree
if name in VAR_TOKENS:
return (name, [(all_terminals(tree), [])])
else:
return (name, [shrink_tree(c) for c in children])
mystring = '1+2'
A1_GRAMMAR = {
"<start>": ["<expr>"],
"<expr>": ["<expr>+<expr>", "<expr>-<expr>", "<integer>"],
"<integer>": ["<digit><integer>", "<digit>"],
"<digit>": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
}
if __name__ == "__main__":
tree = ('<start>', [
('<expr>', [('<expr>', [('<integer>', [('<digit>', [('1', [])])])]),
('+', []),
('<expr>', [('<integer>', [('<digit>', [('2', [])])])])])
])
assert mystring == all_terminals(tree)
display_tree(tree)
A2_GRAMMAR = {
"<start>": ["<expr>"],
"<expr>": ["<integer><expr_>"],
"<expr_>": ["+<expr>", "-<expr>", ""],
"<integer>": ["<digit><integer_>"],
"<integer_>": ["<integer>", ""],
"<digit>": ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
}
if __name__ == "__main__":
tree = ('<start>', [('<expr>', [
('<integer>', [('<digit>', [('1', [])]), ('<integer_>', [('', [])])]),
('<expr_>', [('+', []),
def reduce(self, inp):
tree = self.parse(inp)
self.reduce_tree(tree)
return all_terminals(tree)
def parse(self, inp):
tree, *_ = self.parser.parse(inp)
if self.log_reduce:
print(all_terminals(tree))
return tree
def tree_list_to_string(q):
return "[" + ", ".join([all_terminals(tree) for tree in q]) + "]"
import copy
if __name__ == "__main__":
new_derivation_tree = copy.deepcopy(derivation_tree)
# We really should have some query language
sub_expr_tree = new_derivation_tree[1][0][1][2]
display_tree(sub_expr_tree)
if __name__ == "__main__":
new_derivation_tree[1][0] = sub_expr_tree
display_tree(new_derivation_tree)
if __name__ == "__main__":
all_terminals(new_derivation_tree)
# ### Simplifying by Alternative Expansions
if __name__ == "__main__":
print('\n### Simplifying by Alternative Expansions')
if __name__ == "__main__":
term_tree = new_derivation_tree[1][0][1][0][1][0][1][1][1][0]
display_tree(term_tree)
["<expr>"],
"<expr>":
["<expr>+<expr>", "<expr>-<expr>", "<integer>"],
"<integer>":
["<digit><integer>", "<digit>"],
"<digit>":
["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]
}
if __name__ == "__main__":
tree = ('<start>',[
('<expr>',[
('<expr>',[('<integer>',[('<digit>',[('1',[])])])]),
('+',[]),
('<expr>',[('<integer>',[('<digit>',[('2',[])])])])])])
assert mystring == all_terminals(tree)
display_tree(tree)
A2_GRAMMAR = {
"<start>":
["<expr>"],
"<expr>":
["<integer><expr_>"],
"<expr_>":
["+<expr>", "-<expr>", ""],
"<integer>":
["<digit><integer_>"],
"<integer_>":
["<integer>", ""],
"<digit>":
def parse(self, inp):
tree = self.parser.parse(inp)[0]
print(all_terminals(tree))
return tree
def derivation_reductions(self, tree):
(symbol, children) = tree
if len(children) == 0:
return [] # Terminal symbol
print("Trying alternative expansions for " + symbol)
# Possible expansions for this symbol
expansions = self.grammar[symbol]
print("Expansions: " + repr(expansions))
alternatives = \
[expansion_to_children(expansion) for expansion in expansions]
reductions = []
for alternative in alternatives:
if len(alternative) > len(children):
continue # New alternative has more children
match = True
new_children_reductions = []
# print("Trying alternative expansion " + queue_to_string(alternative))
for alt_child in alternative:
(alt_symbol, _) = alt_child
child_reductions = subtrees_with_symbol(alt_symbol, tree)
if len(child_reductions) == 0:
# Child not found; cannot apply rule
match = False
break
# print("Found alternatives " + queue_to_string(child_reductions))
new_children_reductions.append(child_reductions)
if not match:
continue # Try next alternative
# Go through the possible combinations
for new_children in possible_combinations(new_children_reductions):
new_tree = (symbol, new_children)
if number_of_nodes(new_tree) >= number_of_nodes(tree):
continue # No reduction
reductions.append(new_tree)
# Apply this recursively
if children is not None:
for i in range(0, len(children)):
child = children[i]
child_reductions = self.derivation_reductions(child)
for reduced_child in child_reductions:
new_children = (children[:i] + [reduced_child] +
children[i + 1:])
reductions.append((symbol, new_children))
# Filter duplicates
unique_reductions = []
for r in reductions:
if r not in unique_reductions:
unique_reductions.append(r)
reductions = unique_reductions
if len(reductions) > 0:
# We have a new expansion
print("Can reduce " + symbol + " " + all_terminals(tree) +
" to reduced subtrees " + queue_to_string(reductions))
return reductions