def _generate_tree(tree, output, selected_production, unexpanded):
productions = params['BNF_GRAMMAR'].rules[tree.root]
if selected_production == -1:
# if len(productions['choices']) == 1:
# selected_production = 0
# else:
unexpanded.append(tree)
return output, unexpanded
chosen_prod = productions['choices'][int(selected_production)]
tree.children = []
# print(chosen_prod)
for symbol in chosen_prod['choice']:
# Iterate over all symbols in the chosen production.
if symbol["type"] == "T":
# The symbol is a terminal. Append new node to children.
tree.children.append(Tree(symbol["symbol"], tree, type=symbol["type"]))
# Append the terminal to the output list.
output.append(symbol["symbol"])
elif symbol["type"] == "NT":
# The symbol is a non-terminal. Append new node to children.
tree.children.append(Tree(symbol["symbol"], tree, type=symbol["type"]))
output, unexpanded = _generate_tree(tree.children[-1], output, -1, unexpanded)
return output, unexpanded
def pi_random_init(depth):
tree = Tree(str(params['BNF_GRAMMAR'].start_rule[0]),
None, max_depth=depth, depth_limit=depth)
genome = tree.pi_random_derivation(0, max_depth=depth)
if tree.check_expansion(params['BNF_GRAMMAR'].non_terminals.keys()):
print("tree.pi_random_init generated an Invalid")
quit()
return tree.get_output(), genome, tree, False
def generate_tree_from_genome(genome):
""" Returns a tree given an input of a genome. Faster than normal genome
initialisation as less information is returned. To be used when a tree
needs to be built quickly from a given genome."""
new_tree = Tree((str(params['BNF_GRAMMAR'].start_rule[0]), ),
None,
depth_limit=params['MAX_TREE_DEPTH'])
new_tree.fast_genome_derivation(genome)
return new_tree
def pi_grow_init(depth):
tree = Tree((str(params['BNF_GRAMMAR'].start_rule[0]), ),
None,
max_depth=depth,
depth_limit=depth)
genome = tree.pi_grow(0, max_depth=depth)
if tree.check_expansion():
print("tree.pi_grow_init generated an Invalid")
quit()
return tree.get_output(), genome, tree, False
def genome_tree_derivation(ind_tree,
genome,
index,
depth,
max_depth,
nodes,
invalid=False):
""" Builds a tree using production choices from a given genome. Not
guaranteed to terminate.
"""
if not invalid and index < len(genome) and\
max_depth <= params['MAX_TREE_DEPTH']:
nodes += 1
depth += 1
ind_tree.id, ind_tree.depth = nodes, depth
productions = params['BNF_GRAMMAR'].rules[ind_tree.root]
ind_tree.codon = genome[index % len(genome)]
selection = ind_tree.codon % len(productions)
chosen_prod = productions[selection]
index += 1
ind_tree.children = []
for i in range(len(chosen_prod)):
symbol = chosen_prod[i]
if symbol[1] == "T":
ind_tree.children.append(Tree((symbol[0], ), ind_tree))
elif symbol[1] == "NT":
ind_tree.children.append(Tree((symbol[0], ), ind_tree))
index, nodes, d, max_depth, invalid = \
genome_tree_derivation(ind_tree.children[-1], genome,
index, depth, max_depth, nodes,
invalid)
else:
# Mapping incomplete
return index, nodes, depth, max_depth, True
NT_kids = [
kid for kid in ind_tree.children
if kid.root in params['BNF_GRAMMAR'].non_terminals
]
if not NT_kids:
# Then the branch terminates here
depth += 1
nodes += 1
if not invalid:
if (depth > max_depth):
max_depth = depth
if max_depth > params['MAX_TREE_DEPTH']:
invalid = True
return index, nodes, depth, max_depth, invalid
def genome_tree_map(genome):
tree = Tree((str(params['BNF_GRAMMAR'].start_rule[0]), ),
None,
depth_limit=params['MAX_TREE_DEPTH'])
used_codons, nodes, depth, max_depth, invalid = \
genome_tree_derivation(tree, genome, 0, 0, 0, 0)
if invalid:
return None, genome, tree, nodes, invalid, max_depth, \
used_codons
else:
return tree.get_output(), genome, tree, nodes, invalid, max_depth, \
used_codons
def generate_PI_ind_tree(max_depth):
"""
Generate an individual using a given Position Independent subtree
initialisation method.
:param max_depth: The maximum depth for the initialised subtree.
:return: A fully built individual.
"""
# Initialise an instance of the tree class
ind_tree = Tree(str(params['BNF_GRAMMAR'].start_rule["symbol"]), None)
# Generate a tree
genome, output, nodes, depth = pi_grow(ind_tree, max_depth)
# Get remaining individual information
phenotype, invalid, used_cod = "".join(output), False, len(genome)
if params['BNF_GRAMMAR'].python_mode:
# Grammar contains python code
phenotype = python_filter(phenotype)
# Initialise individual
ind = individual.Individual(genome, ind_tree, map_ind=False)
# Set individual parameters
ind.phenotype, ind.nodes = phenotype, nodes
ind.depth, ind.used_codons, ind.invalid = depth, used_cod, invalid
# Generate random tail for genome.
ind.genome = genome + [randint(0, params['CODON_SIZE']) for
_ in range(int(ind.used_codons / 2))]
return ind
def map_tree_from_genome(genome):
"""
Maps a full tree from a given genome.
:param genome: A genome to be mapped.
:return: All components necessary for a fully mapped individual.
"""
# Initialise an instance of the tree class
tree = Tree(str(params['BNF_GRAMMAR'].start_rule["symbol"]), None)
# Map tree from the given genome
output, used_codons, nodes, depth, max_depth, invalid = \
genome_tree_map(tree, genome, [], 0, 0, 0, 0)
# Build phenotype.
phenotype = "".join(output)
if invalid:
# Return "None" phenotype if invalid
return None, genome, tree, nodes, invalid, max_depth, \
used_codons
else:
return phenotype, genome, tree, nodes, invalid, max_depth, \
used_codons
def map_tree_from_genome(genome):
"""
Maps a full tree from a given genome.
:param genome: A genome to be mapped.
:return: All components necessary for a fully mapped individual.
"""
# Initialise an instance of the tree class
tree = Tree(str(params['BNF_GRAMMAR'].start_rule[0]),
None,
depth_limit=params['MAX_TREE_DEPTH'])
# Map tree from the given genome
output, used_codons, nodes, depth, max_depth, invalid = \
genome_tree_map(tree, genome, [], 0, 0, 0, 0)
# Build phenotype.
phenotype = "".join(output)
if params['BNF_GRAMMAR'].python_mode:
# Grammar contains python code
phenotype = python_filter(phenotype)
if invalid:
# Return "None" phenotype if invalid
return None, genome, tree, nodes, invalid, max_depth, \
used_codons
else:
return phenotype, genome, tree, nodes, invalid, max_depth, \
used_codons
def tree_derivation(ind_tree, genome, method, nodes, depth, max_depth,
depth_limit):
""" Derive a tree using a given method """
nodes += 1
depth += 1
ind_tree.id, ind_tree.depth = nodes, depth
productions = params['BNF_GRAMMAR'].rules[ind_tree.root]
available = ind_tree.legal_productions(method, depth_limit, productions)
chosen_prod = choice(available)
prod_choice = productions.index(chosen_prod)
codon = randrange(len(productions), params['BNF_GRAMMAR'].codon_size,
len(productions)) + prod_choice
ind_tree.codon = codon
genome.append(codon)
ind_tree.children = []
for symbol in chosen_prod:
if symbol[1] == params['BNF_GRAMMAR'].T:
# if the right hand side is a terminal
ind_tree.children.append(Tree((symbol[0], ), ind_tree))
elif symbol[1] == params['BNF_GRAMMAR'].NT:
# if the right hand side is a non-terminal
ind_tree.children.append(Tree((symbol[0], ), ind_tree))
genome, nodes, d, max_depth = \
tree_derivation(ind_tree.children[-1], genome, method, nodes,
depth, max_depth, depth_limit - 1)
NT_kids = [
kid for kid in ind_tree.children
if kid.root in params['BNF_GRAMMAR'].non_terminals
]
if not NT_kids:
# Then the branch terminates here
depth += 1
nodes += 1
if depth > max_depth:
max_depth = depth
return genome, nodes, depth, max_depth
def pi_random_derivation(tree, index, max_depth=20):
""" Randomly builds a tree from a given root node up to a maximum
given depth. Uses position independent stuff.
"""
queue = [[
tree, params['BNF_GRAMMAR'].non_terminals[tree.root]['recursive']
]]
while queue:
num = len(queue)
chosen = randint(0, num - 1)
all_node = queue.pop(chosen)
node = all_node[0]
n, depth = tree.get_tree_info(tree)
depth += 1
if depth < max_depth:
productions = params['BNF_GRAMMAR'].rules[node.root]
available = []
remaining_depth = max_depth - depth
if remaining_depth > params['BNF_GRAMMAR'].max_arity:
available = productions
else:
for prod in productions:
depth = 0
for item in prod:
if (item[1] == params['BNF_GRAMMAR'].NT) and \
(item[2] > depth):
depth = item[2]
if depth < remaining_depth:
available.append(prod)
chosen_prod = choice(available)
if len(productions) > 1:
prod_choice = productions.index(chosen_prod)
codon = randrange(0, params['BNF_GRAMMAR'].codon_size,
len(productions)) + prod_choice
node.codon = codon
node.id = index
index += 1
node.children = []
for i in range(len(chosen_prod)):
symbol = chosen_prod[i]
child = Tree(symbol[0], node)
node.children.append(child)
if symbol[1] == params['BNF_GRAMMAR'].NT:
# if the right hand side is a non-terminal
queue.insert(chosen + i, [
child, params['BNF_GRAMMAR'].non_terminals[child.root]
['recursive']
])
genome = tree.build_genome([])
return genome
def tree_init(depth, method):
"""
Initialise a tree to a given depth using a specified method for
initialisation.
"""
tree = Tree((str(params['BNF_GRAMMAR'].start_rule[0]), ),
None,
max_depth=depth - 1,
depth_limit=depth - 1)
genome, nodes, d, max_depth = mapper.tree_derivation(
tree, [], method, 0, 0, 0, depth - 1)
if tree.check_expansion():
print("tree.init generated an Invalid")
quit()
return tree.get_output(), genome, tree, nodes, \
False, max_depth, len(genome)
def generate_output(state):
# print(state)
grm = params['BNF_GRAMMAR']
ind_tree = Tree(str(grm.start_rule["symbol"]), None)
unexpanded = []
output = []
tree = ind_tree
s = copy.copy(state)
idx = 0
while True:
output, unexpanded = _generate_tree(tree, output, s[idx], unexpanded)
idx += 1
tree = unexpanded.pop(0) if unexpanded else None
if tree is None or s[idx] == -1:
break
return walk_tree(ind_tree, []), ind_tree
def generate_new_genome_and_phenotype():
writeLog('Creating new individual values')
depths = range(params['BNF_GRAMMAR'].min_ramp + 1,
params['MAX_INIT_TREE_DEPTH'] + 1)
size = params['POPULATION_SIZE']
if size < len(depths):
depths = depths[:int(size)]
max_depth = depths[int(len(depths) / 2)]
# Initialise an instance of the tree class
ind_tree = Tree(str(params['BNF_GRAMMAR'].start_rule["symbol"]), None)
# Generate a tree
genome, output, nodes, depth = pi_grow(ind_tree, max_depth)
# Get remaining individual information
phenotype, invalid, used_cod = "".join(output), False, len(genome)
return phenotype, nodes, genome, depth, used_cod, invalid
def generate_ind_tree(max_depth, method):
"""
Generate an individual using a given subtree initialisation method.
:param max_depth: The maximum depth for the initialised subtree.
:param method: The method of subtree initialisation required.
:return:
"""
# Initialise an instance of the tree class
ind_tree = Tree(str(params['BNF_GRAMMAR'].start_rule[0]),
None,
depth_limit=max_depth - 1)
# Generate a tree
genome, output, nodes, _, depth = generate_tree(ind_tree, [], [], method,
0, 0, 0, max_depth - 1)
# Get remaining individual information
phenotype, invalid, used_cod = "".join(output), False, len(genome)
if params['BNF_GRAMMAR'].python_mode:
# Grammar contains python code
phenotype = python_filter(phenotype)
# Initialise individual
ind = individual.Individual(genome, ind_tree, map_ind=False)
# Set individual parameters
ind.phenotype, ind.nodes = phenotype, nodes
ind.depth, ind.used_codons, ind.invalid = depth, used_cod, invalid
# Generate random tail for genome.
ind.genome = genome + [
randint(0, params['CODON_SIZE'])
for _ in range(int(ind.used_codons / 2))
]
return ind
def _generate_tree(self, pos, tree, output, selected_production,
undecided_trees):
productions = params['BNF_GRAMMAR'].rules[tree.root]
if selected_production == -1:
# if len(productions['choices']) == 1:
# selected_production = 0
# else:
undecided_trees.append(tree)
return output, undecided_trees
chosen_prod = productions['choices'][selected_production]
tree.children = []
pos.state = apply_action(pos.state,
(pos.num_nodes, selected_production))
pos.num_nodes += 1
for symbol in chosen_prod['choice']:
# Iterate over all symbols in the chosen production.
if symbol["type"] == "T":
# The symbol is a terminal. Append new node to children.
tr = Tree(symbol["symbol"], tree)
tree.children.append(tr)
tr.depth = tree.depth + 1
# Append the terminal to the output list.
output.append(symbol["symbol"])
elif symbol["type"] == "NT":
# The symbol is a non-terminal. Append new node to children.
tr = Tree(symbol["symbol"], tree)
tree.children.append(tr)
tr.depth = tree.depth + 1
output, undecided_trees = self._generate_tree(
pos, tree.children[-1], output, -1, undecided_trees)
return output, undecided_trees
def reduce(solution):
"""
Takes a list of all matching subtrees found in the target string and
iteratively combines and reduces subtrees to generate larger matching
subtrees. This process continues until the list of matching subtrees has
been completely reduced into a target string.
:param solution: A list of all snippets (i.e. matching subtrees found in
the target string.
:return: Nothing.
"""
# Find all non-terminals in the grammar that can be used to concatenate
# subtrees to new/larger subtrees.
reduce_NTs = params['BNF_GRAMMAR'].concat_NTs
# Pre-load the target string.
target = params['REVERSE_MAPPING_TARGET']
for idx, snippet_info in enumerate(solution):
# Get current snippet.
snippet = snippet_info[2]
# Find current snippet info.
NT = snippet_info[1]
# Get indexes of the current snippet
indexes = snippet_info[0]
start, end = indexes[0], indexes[1]
# Find if the snippet root (NT) exists anywhere in the
# reduction NTs.
if NT in reduce_NTs:
for reduce in reduce_NTs[NT]:
# Now we're searching for a specific subset of keys in the
# snippets dictionary.
# Generate list of only the desired Non Terminals.
NTs = reduce[2]
if len(NTs) == 1:
# This choice leads directly to the parent, check if parent
# snippet already exists.
# Child is current snippet.
child = [[snippet, trackers.snippets[snippet]]]
# Get key for new snippet.
key, start, end = generate_key_and_check(
start, end, reduce, child)
# Create a new node for the solution list.
new_entry = [indexes, reduce[1], key]
# Insert the current node into the solution.
if new_entry not in solution:
solution.insert(idx + 1, new_entry)
else:
# Find the index of the snippet root in the current
# reduction production choice.
NT_locs = [i for i, x in enumerate(NTs) if x[0] == NT]
for loc in NT_locs:
# We want to check each possible reduction option.
# Set where the original snippet starts and ends on
# the target string.
if loc == 0:
# The current snippet is at the start of the
# reduction attempt.
pre, aft = None, end
elif start == 0 and loc != 0:
# The current snippet is at the start of the target
# string, but we are trying to reduce_trees it with
# something before it.
break
elif end == len(params['TARGET']) and loc != \
NT_locs[-1]:
# The current snippet is at the end of the target
# string, but we are trying to reduce_trees it with
# something after it.
break
elif loc == len(NTs):
# The current snippet is at the end of the
# reduction attempt.
pre, aft = start, None
else:
# The current snippet is in the middle of the
# reduction attempt.
pre, aft = start, end
alt_cs = list(range(len(NTs)))
# Initialise a list of children to be reduced.
children = [[] for _ in range(len(NTs))]
# Set original snippet into children.
children[loc] = [snippet, trackers.snippets[snippet]]
curr_idx = solution.index(snippet_info)
# Step 1: reduce everything before the current loc.
for item in reversed(alt_cs[:loc]):
if NTs[item][1] == "T":
# This is a terminal, decrement by length of T.
# Check output of target string.
check = target[pre - len(NTs[item][0]):pre]
if check == NTs[item][0]:
# We have a match.
# Generate fake key for snippets dict.
key = str([pre - len(NTs[item][0]), pre])
# Create new tree from this terminal.
T_tree = Tree(check, None)
# Add to children.
children[item] = [key, T_tree]
# Decrement target string index.
pre -= len(NTs[item][0])
else:
# No match.
break
else:
# This is a NT. Check solution list for
# matching node.
available = [
sol for sol in solution[:curr_idx] if
sol[1] == NTs[item][0] and sol[0][1] == pre
]
for check in available:
# We have a match.
# Set the correct child in our
# children.
children[item] = [
check[2], trackers.snippets[check[2]]
]
# Decrement target string index.
child_len = get_num_from_str(check[2])
pre -= child_len[1] - child_len[0]
break
# Step cython_backup: reduce everything after the loc.
for i, item in enumerate(alt_cs[loc + 1:]):
if NTs[item][1] == "T":
# This is a terminal, decrement by length of T.
# Check output of target string.
check = target[aft:aft + len(NTs[item][0])]
if check == NTs[item][0]:
# We have a match.
# Generate fake key for snippets dict.
key = str([aft, aft + len(NTs[item][0])])
# Create new tree from this terminal.
T_tree = Tree(check, None)
# Add to children.
children[item] = [key, T_tree]
# Increment target string index.
aft += len(NTs[item][0])
else:
# No match.
break
else:
# We haven't looked ahead in the string,
# we can't add things we don't know yet.
break
if all([child != [] for child in children]):
# We have expanded all children and can collapse
# a node.
key, pre, aft = generate_key_and_check(
pre, aft, reduce, children)
# Create a new node for the solution list.
new_entry = [[pre, aft], reduce[1], key]
# Add the new reduced entry to the solution.
if new_entry not in solution:
solution.insert(idx + 1, new_entry)
def parse_terminals(target):
"""
Given a target string, build up a list of terminals which match certain
portions of the target string.
:return: A list of terminals in order of appearance in the target string.
"""
if params['VERBOSE']:
print("Target:\n", target)
# Pre-load all terminals and non-terminal rules from the grammar.
terms, rules = params['BNF_GRAMMAR'].terminals, params['BNF_GRAMMAR'].rules
# Initialise dict for storing the snippets for compiling a complete
# solution. The key for each entry is the portion of the target string on
# which the output matches, along with the root node of the subtree. The
# value is the subtree itself.
trackers.snippets = {}
# Initialise dict for deleted snippets, to ensure they aren't generated
# again.
trackers.deleted_snippets = []
for T in sorted(terms.keys()):
# Iterate over all Terminals.
# Find all occurances of this terminal in the target string.
occurrances = []
index = 0
while index < len(target):
index = target.find(T, index)
if index not in occurrances and index != -1:
occurrances.append(index)
index += len(T)
else:
break
for idx in occurrances:
# Check each occurrence of this terminal in the target string.
for NT in terms[T]:
if any([[T] == i
for i in [[sym['symbol'] for sym in choice['choice']]
for choice in rules[NT]['choices']]]):
# Check if current T is the entire production choice of any
# particular rule.
# Generate a key for the snippets repository.
key = " ".join([str([idx, idx + len(T)]), NT])
# Get index of production choice.
index = [[sym['symbol'] for sym in choice['choice']]
for choice in rules[NT]['choices']].index([T])
# Get production choice.
choice = rules[NT]['choices'][index]['choice']
# Generate a tree for this choice.
parent = Tree(NT, None)
# Generate a codon for this choice.
parent.codon = generate_codon(NT, choice)
# Set the snippet key for the parent.
parent.snippet = key
# Create child for terminal.
child = Tree(T, parent)
# Add child to parent.
parent.children.append(child)
# Add snippet to snippets repository.
trackers.snippets[key] = parent
def pi_random_derivation(tree, max_depth):
"""
Randomly builds a tree from a given root node up to a maximum given
depth. Uses position independent methods to derive non-terminal nodes.
Final tree is not guaranteed to reach the specified max_depth limit.
:param tree: An instance of the representation.tree.Tree class.
:param max_depth: The maximum depth to which to derive a tree.
:return: The fully derived tree.
"""
# Initialise derivation queue.
queue = [[
tree,
ret_true(params['BNF_GRAMMAR'].non_terminals[tree.root]['recursive'])
]]
# Initialise empty genome. With PI operators we can't use a depth-first
# traversal of the tree to build the genome, we need to build it as we
# encounter each node.
genome = []
while queue:
# Loop until no items remain in the queue.
# Pick a random item from the queue.
chosen = randint(0, len(queue) - 1)
# Pop the next item from the queue.
all_node = queue.pop(chosen)
node = all_node[0]
# Get depth current node.
if node.parent is not None:
node.depth = node.parent.depth + 1
# Find the productions possible from the current root.
productions = params['BNF_GRAMMAR'].rules[node.root]
# Set remaining depth.
remaining_depth = max_depth - node.depth
# Find which productions can be used based on the derivation method.
available = legal_productions("random", remaining_depth, node.root,
productions['choices'])
# Randomly pick a production choice.
chosen_prod = choice(available)
# Find the index of the chosen production and set a matching codon
# based on that index.
prod_index = productions['choices'].index(chosen_prod)
codon = randrange(productions['no_choices'],
params['BNF_GRAMMAR'].codon_size,
productions['no_choices']) + prod_index
# Set the codon for the current node and append codon to the genome.
node.codon = codon
# Insert codon into the genome.
genome.append(codon)
# Initialise empty list of children for current node.
node.children = []
for i, symbol in enumerate(chosen_prod['choice']):
# Iterate over all symbols in the chosen production.
# Create new child.
child = Tree(symbol["symbol"], node)
# Append new node to children.
node.children.append(child)
if symbol["type"] == "NT":
# The symbol is a non-terminal.
# Check whether child is recursive
recur_child = ret_true(params['BNF_GRAMMAR'].non_terminals[
child.root]['recursive'])
# Insert new child into the correct position in the queue.
queue.insert(chosen + i, [child, recur_child])
# genome, output, invalid, depth, and nodes can all be Generated by
# recursing through the tree once.
_, output, invalid, depth, \
nodes = tree.get_tree_info(params['BNF_GRAMMAR'].non_terminals.keys(),
[], [])
return genome, output, nodes, depth
from representation.position import PonyGEPositionFactory
from representation.recurrent import RecurrentModelFactory
from cross.utilities import PositionFactory, ModelFactory
from cross.selfplay import play_init, play_select_move
from cross.dual_net import Network
from cross.deepxor import num_actions
FLAGS = tf.flags.FLAGS
method = "random"
max_depth = 300
PositionFactory.set_factory('pony',
PonyGEPositionFactory(method=method, max_depth=7))
ModelFactory.set_factory('recurrent', RecurrentModelFactory(num_actions))
grm = Grammar(FLAGS.grammar)
params['BNF_GRAMMAR'] = grm
ind_tree = Tree(str(grm.start_rule["symbol"]), None)
play_init(Network(), tree=ind_tree)
for i in range(0, max_depth):
move = play_select_move()
print(move)
#print(ind_tree)
#genome, output, nodes, _, depth = generate_tree(ind_tree, [], [], method, 0, 0, 0, max_depth)
#ind = Individual(genome, ind_tree, map_ind=False)
#ind.phenotype = "".join(output)
#print(genome)
#print(ind.phenotype)
def generate_tree(tree, genome, output, method, nodes, depth, max_depth,
depth_limit):
"""
Recursive function to derive a tree using a given method.
:param tree: An instance of the Tree class.
:param genome: The list of all codons in a tree.
:param output: The list of all terminal nodes in a subtree. This is
joined to become the phenotype.
:param method: A string of the desired tree derivation method,
e.g. "full" or "random".
:param nodes: The total number of nodes in the tree.
:param depth: The depth of the current node.
:param max_depth: The maximum depth of any node in the tree.
:param depth_limit: The maximum depth the tree can expand to.
:return: genome, output, nodes, depth, max_depth.
"""
# Increment nodes and depth, set depth of current node.
nodes += 1
depth += 1
tree.depth = depth
# Find the productions possible from the current root.
productions = params['BNF_GRAMMAR'].rules[tree.root]
if depth_limit:
# Set remaining depth.
remaining_depth = depth_limit - depth
else:
remaining_depth = depth_limit
# Find which productions can be used based on the derivation method.
available = legal_productions(method, remaining_depth, tree.root,
productions['choices'])
# Randomly pick a production choice.
chosen_prod = choice(available)
# Find the index of the chosen production and set a matching codon based
# on that index.
prod_index = productions['choices'].index(chosen_prod)
codon = randrange(productions['no_choices'],
params['BNF_GRAMMAR'].codon_size,
productions['no_choices']) + prod_index
# Set the codon for the current node and append codon to the genome.
tree.codon = codon
genome.append(codon)
# Initialise empty list of children for current node.
tree.children = []
for symbol in chosen_prod['choice']:
# Iterate over all symbols in the chosen production.
if symbol["type"] == "T":
# The symbol is a terminal. Append new node to children.
tree.children.append(Tree(symbol["symbol"], tree))
# Append the terminal to the output list.
output.append(symbol["symbol"])
elif symbol["type"] == "NT":
# The symbol is a non-terminal. Append new node to children.
tree.children.append(Tree(symbol["symbol"], tree))
# recurse on the new node.
genome, output, nodes, d, max_depth = \
generate_tree(tree.children[-1], genome, output, method,
nodes, depth, max_depth, depth_limit)
NT_kids = [kid for kid in tree.children if kid.root in
params['BNF_GRAMMAR'].non_terminals]
if not NT_kids:
# Then the branch terminates here
depth += 1
nodes += 1
if depth > max_depth:
# Set new maximum depth
max_depth = depth
return genome, output, nodes, depth, max_depth
def pi_grow(tree, max_depth):
"""
Grows a tree until a single branch reaches a specified depth. Does this
by only using recursive production choices until a single branch of the
tree has reached the specified maximum depth. After that any choices are
allowed.
:param tree: An instance of the representation.tree.Tree class.
:param max_depth: The maximum depth to which to derive a tree.
:return: The fully derived tree.
"""
# Initialise derivation queue.
queue = [[tree, ret_true(params['BNF_GRAMMAR'].non_terminals[
tree.root]['recursive'])]]
# Initialise empty genome. With PI operators we can't use a depth-first
# traversal of the tree to build the genome, we need to build it as we
# encounter each node.
genome = []
while queue:
# Loop until no items remain in the queue.
# Pick a random item from the queue.
chosen = randint(0, len(queue) - 1)
# Pop the next item from the queue.
all_node = queue.pop(chosen)
node, recursive = all_node[0], all_node[0]
# Get depth of current node.
if node.parent is not None:
node.depth = node.parent.depth + 1
# Get maximum depth of overall tree.
_, overall_depth = get_nodes_and_depth(tree)
# Find the productions possible from the current root.
productions = params['BNF_GRAMMAR'].rules[node.root]
# Set remaining depth.
remaining_depth = max_depth - node.depth
if (overall_depth < max_depth) or \
(recursive and (not any([item[1] for item in queue]))):
# We want to prevent the tree from creating terminals until a
# single branch has reached the full depth. Only select recursive
# choices.
# Find which productions can be used based on the derivation method.
available = legal_productions("full", remaining_depth, node.root,
productions['choices'])
else:
# Any production choices can be made.
# Find which productions can be used based on the derivation method.
available = legal_productions("random", remaining_depth, node.root,
productions['choices'])
# Randomly pick a production choice.
chosen_prod = choice(available)
# Find the index of the chosen production and set a matching codon
# based on that index.
prod_index = productions['choices'].index(chosen_prod)
codon = randrange(productions['no_choices'],
params['BNF_GRAMMAR'].codon_size,
productions['no_choices']) + prod_index
# Set the codon for the current node and append codon to the genome.
node.codon = codon
# Insert codon into the genome.
genome.append(codon)
# Initialise empty list of children for current node.
node.children = []
for i, symbol in enumerate(chosen_prod['choice']):
# Iterate over all symbols in the chosen production.
# Create new child.
child = Tree(symbol["symbol"], node)
# Append new node to children.
node.children.append(child)
if symbol["type"] == "NT":
# The symbol is a non-terminal.
# Check whether child is recursive
recur_child = ret_true(params['BNF_GRAMMAR'].non_terminals
[child.root]['recursive'])
# Insert new child into the correct position in the queue.
queue.insert(chosen + i, [child, recur_child])
# genome, output, invalid, depth, and nodes can all be generated by
# recursing through the tree once.
_, output, invalid, depth, \
nodes = tree.get_tree_info(params['BNF_GRAMMAR'].non_terminals.keys(),
[], [])
return genome, output, nodes, depth
def genome_tree_map(tree,
genome,
output,
index,
depth,
max_depth,
nodes,
invalid=False):
"""
Recursive function which builds a tree using production choices from a
given genome. Not guaranteed to terminate.
:param tree: An instance of the representation.tree.Tree class.
:param genome: A full genome.
:param output: The list of all terminal nodes in a subtree. This is
joined to become the phenotype.
:param index: The index of the current location on the genome.
:param depth: The current depth in the tree.
:param max_depth: The maximum overall depth in the tree so far.
:param nodes: The total number of nodes in the tree thus far.
:param invalid: A boolean flag indicating whether or not the individual
is invalid.
:return: index, the index of the current location on the genome,
nodes, the total number of nodes in the tree thus far,
depth, the current depth in the tree,
max_depth, the maximum overall depth in the tree,
invalid, a boolean flag indicating whether or not the
individual is invalid.
"""
if not invalid and index < len(genome) * (params['MAX_WRAPS'] + 1):
# If the solution is not invalid thus far, and if we still have
# remaining codons in the genome, then we can continue to map the tree.
if params['MAX_TREE_DEPTH'] and (max_depth > params['MAX_TREE_DEPTH']):
# We have breached our maximum tree depth limit.
invalid = True
# Increment and set number of nodes and current depth.
nodes += 1
depth += 1
tree.id, tree.depth = nodes, depth
# Find all production choices and the number of those production
# choices that can be made by the current root non-terminal.
productions = params['BNF_GRAMMAR'].rules[tree.root]['choices']
no_choices = params['BNF_GRAMMAR'].rules[tree.root]['no_choices']
# Set the current codon value from the genome.
tree.codon = genome[index % len(genome)]
# Select the index of the correct production from the list.
selection = tree.codon % no_choices
# Set the chosen production
chosen_prod = productions[selection]
# Increment the index
index += 1
# Initialise an empty list of children.
tree.children = []
for symbol in chosen_prod['choice']:
# Add children to the derivation tree by creating a new instance
# of the representation.tree.Tree class for each child.
if symbol["type"] == "T":
# Append the child to the parent node. Child is a terminal, do
# not recurse.
tree.children.append(Tree(symbol["symbol"], tree))
output.append(symbol["symbol"])
elif symbol["type"] == "NT":
# Append the child to the parent node.
tree.children.append(Tree(symbol["symbol"], tree))
# Recurse by calling the function again to map the next
# non-terminal from the genome.
output, index, nodes, d, max_depth, invalid = \
genome_tree_map(tree.children[-1], genome, output,
index, depth, max_depth, nodes,
invalid=invalid)
else:
# Mapping incomplete, solution is invalid.
return output, index, nodes, depth, max_depth, True
# Find all non-terminals in the chosen production choice.
NT_kids = [
kid for kid in tree.children
if kid.root in params['BNF_GRAMMAR'].non_terminals
]
if not NT_kids:
# There are no non-terminals in the chosen production choice, the
# branch terminates here.
depth += 1
nodes += 1
if not invalid:
# The solution is valid thus far.
if depth > max_depth:
# Set the new maximum depth.
max_depth = depth
if params['MAX_TREE_DEPTH'] and (max_depth > params['MAX_TREE_DEPTH']):
# If our maximum depth exceeds the limit, the solution is invalid.
invalid = True
return output, index, nodes, depth, max_depth, invalid
def pi_grow(tree, index, max_depth=20):
""" Grows a tree until a single branch reaches a specified depth. Does
this by only using recursive production choices until a single
branch of the tree has reached the specified maximum depth. After
that any choices are allowed
"""
queue = [[
tree, params['BNF_GRAMMAR'].non_terminals[tree.root]['recursive']
]]
while queue:
num = len(queue)
chosen = randint(0, num - 1)
all_node = queue.pop(chosen)
node = all_node[0]
n, depth = tree.get_tree_info(tree)
depth += 1
if depth < max_depth:
productions = params['BNF_GRAMMAR'].rules[node.root]
available = []
remaining_depth = max_depth - depth
if (tree.get_max_tree_depth(tree) < max_depth - 1) or \
(node.parent is None) or \
(all_node[1] and (not any([item[1] for item in queue]))):
# We want to prevent the tree from creating terminals
# until a single branch has reached the full depth
if remaining_depth > params['BNF_GRAMMAR'].max_arity:
for production in productions:
if any(sym[3] for sym in production):
available.append(production)
if not available:
for production in productions:
if not all(sym[3] for sym in production):
available.append(production)
else:
for prod in productions:
depth = 0
for item in prod:
if (item[1] == params['BNF_GRAMMAR'].NT) and \
(item[2] > depth):
depth = item[2]
if depth < remaining_depth:
available.append(prod)
else:
if remaining_depth > params['BNF_GRAMMAR'].max_arity:
available = productions
else:
for prod in productions:
depth = 0
for item in prod:
if (item[1] == params['BNF_GRAMMAR'].NT) and \
(item[2] > depth):
depth = item[2]
if depth < remaining_depth:
available.append(prod)
chosen_prod = choice(available)
if len(productions) > 1:
prod_choice = productions.index(chosen_prod)
codon = randrange(0, params['BNF_GRAMMAR'].codon_size,
len(productions)) + prod_choice
node.codon = codon
node.id = index
index += 1
node.children = []
for i in range(len(chosen_prod)):
symbol = chosen_prod[i]
child = Tree(symbol[0], node)
node.children.append(child)
if symbol[1] == params['BNF_GRAMMAR'].NT:
# if the right hand side is a non-terminal
queue.insert(chosen + i, [
child, params['BNF_GRAMMAR'].non_terminals[child.root]
['recursive']
])
genome = tree.build_genome([])
return genome
def parse_terminals(target):
"""
Given a target string, build up a list of terminals which match certain
portions of the target string.
:return: A list of terminals in order of appearance in the target string.
"""
if params['VERBOSE']:
print("Target:\n", target)
# Pre-load all terminals and non-terminal rules from the grammar.
terms, rules = params['BNF_GRAMMAR'].terminals, params['BNF_GRAMMAR'].rules
# Initialise dict for storing the snippets for compiling a complete
# solution. The key for each entry is the portion of the target string on
# which the output matches, along with the root node of the subtree. The
# value is the subtree itself.
trackers.snippets = {}
# Initialise dict for deleted snippets, to ensure they aren't generated
# again.
trackers.deleted_snippets = []
for T in sorted(terms.keys()):
# Iterate over all Terminals.
# Find all occurances of this terminal in the target string.
occurrances = []
index = 0
while index < len(target):
index = target.find(T, index)
if index not in occurrances and index != -1:
occurrances.append(index)
index += len(T)
else:
break
for idx in occurrances:
# Check each occurrence of this terminal in the target string.
for NT in terms[T]:
if any([[T] == i for i in [[sym['symbol'] for sym in
choice['choice']] for choice in
rules[NT]['choices']]]):
# Check if current T is the entire production choice of any
# particular rule.
# Generate a key for the snippets repository.
key = " ".join([str([idx, idx+len(T)]), NT])
# Get index of production choice.
index = [[sym['symbol'] for sym in choice['choice']] for
choice in rules[NT]['choices']].index([T])
# Get production choice.
choice = rules[NT]['choices'][index]['choice']
# Generate a tree for this choice.
parent = Tree(NT, None)
# Generate a codon for this choice.
parent.codon = generate_codon(NT, choice)
# Set the snippet key for the parent.
parent.snippet = key
# Create child for terminal.
child = Tree(T, parent)
# Add child to parent.
parent.children.append(child)
# Add snippet to snippets repository.
trackers.snippets[key] = parent
