def map_tree_from_genome(genome):
"""
Maps a full tree from a given genome.
:param genome: A genome to be mapped.
:return: All components necessary for a fully mapped individual.
"""
# Initialise an instance of the tree class
tree = Tree(str(params['BNF_GRAMMAR'].start_rule["symbol"]), None)
# Map tree from the given genome
output, used_codons, nodes, depth, max_depth, invalid = \
genome_tree_map(tree, genome, [], 0, 0, 0, 0)
# Build phenotype.
phenotype = "".join(output)
if invalid:
# Return "None" phenotype if invalid
return None, genome, tree, nodes, invalid, max_depth, \
used_codons
else:
return phenotype, genome, tree, nodes, invalid, max_depth, \
used_codons
def generate_ind_tree(max_depth, method):
"""
Generate an individual using a given subtree initialisation method.
:param max_depth: The maximum depth for the initialised subtree.
:param method: The method of subtree initialisation required.
:return: A fully built individual.
"""
# Initialise an instance of the tree class
ind_tree = Tree(str(params['BNF_GRAMMAR'].start_rule["symbol"]), None)
# Generate a tree
genome, output, nodes, _, depth = generate_tree(ind_tree, [], [], method,
0, 0, 0, max_depth)
# Get remaining individual information
phenotype, invalid, used_cod = "".join(output), False, len(genome)
if params['BNF_GRAMMAR'].python_mode:
# Grammar contains python code
phenotype = python_filter(phenotype)
# Initialise individual
ind = individual.Individual(genome, ind_tree, map_ind=False)
# Set individual parameters
ind.phenotype, ind.nodes = phenotype, nodes
ind.depth, ind.used_codons, ind.invalid = depth, used_cod, invalid
# Generate random tail for genome.
ind.genome = genome + [
randint(0, params['CODON_SIZE'])
for _ in range(int(ind.used_codons / 2))
]
return ind
def reduce(solution):
"""
Takes a list of all matching subtrees found in the target string and
iteratively combines and reduces subtrees to generate larger matching
subtrees. This process continues until the list of matching subtrees has
been completely reduced into a target string.
:param solution: A list of all snippets (i.e. matching subtrees found in
the target string.
:return: Nothing.
"""
# Find all non-terminals in the grammar that can be used to concatenate
# subtrees to new/larger subtrees.
reduce_NTs = params['BNF_GRAMMAR'].concat_NTs
# Pre-load the target string.
target = params['REVERSE_MAPPING_TARGET']
for idx, snippet_info in enumerate(solution):
# Get current snippet.
snippet = snippet_info[2]
# Find current snippet info.
NT = snippet_info[1]
# Get indexes of the current snippet
indexes = snippet_info[0]
start, end = indexes[0], indexes[1]
# Find if the snippet root (NT) exists anywhere in the
# reduction NTs.
if NT in reduce_NTs:
for reduce in reduce_NTs[NT]:
# Now we're searching for a specific subset of keys in the
# snippets dictionary.
# Generate list of only the desired Non Terminals.
NTs = reduce[2]
if len(NTs) == 1:
# This choice leads directly to the parent, check if parent
# snippet already exists.
# Child is current snippet.
child = [[snippet, trackers.snippets[snippet]]]
# Get key for new snippet.
key, start, end = generate_key_and_check(
start, end, reduce, child)
# Create a new node for the solution list.
new_entry = [indexes, reduce[1], key]
# Insert the current node into the solution.
if new_entry not in solution:
solution.insert(idx + 1, new_entry)
else:
# Find the index of the snippet root in the current
# reduction production choice.
NT_locs = [i for i, x in enumerate(NTs) if x[0] == NT]
for loc in NT_locs:
# We want to check each possible reduction option.
# Set where the original snippet starts and ends on
# the target string.
if loc == 0:
# The current snippet is at the start of the
# reduction attempt.
pre, aft = None, end
elif start == 0 and loc != 0:
# The current snippet is at the start of the target
# string, but we are trying to reduce_trees it with
# something before it.
break
elif end == len(params['TARGET']) and loc != \
NT_locs[-1]:
# The current snippet is at the end of the target
# string, but we are trying to reduce_trees it with
# something after it.
break
elif loc == len(NTs):
# The current snippet is at the end of the
# reduction attempt.
pre, aft = start, None
else:
# The current snippet is in the middle of the
# reduction attempt.
pre, aft = start, end
alt_cs = list(range(len(NTs)))
# Initialise a list of children to be reduced.
children = [[] for _ in range(len(NTs))]
# Set original snippet into children.
children[loc] = [snippet, trackers.snippets[snippet]]
curr_idx = solution.index(snippet_info)
# Step 1: reduce everything before the current loc.
for item in reversed(alt_cs[:loc]):
if NTs[item][1] == "T":
# This is a terminal, decrement by length of T.
# Check output of target string.
check = target[pre - len(NTs[item][0]):pre]
if check == NTs[item][0]:
# We have a match.
# Generate fake key for snippets dict.
key = str([pre - len(NTs[item][0]), pre])
# Create new tree from this terminal.
T_tree = Tree(check, None)
# Add to children.
children[item] = [key, T_tree]
# Decrement target string index.
pre -= len(NTs[item][0])
else:
# No match.
break
else:
# This is a NT. Check solution list for
# matching node.
available = [
sol for sol in solution[:curr_idx] if
sol[1] == NTs[item][0] and sol[0][1] == pre
]
for check in available:
# We have a match.
# Set the correct child in our
# children.
children[item] = [
check[2], trackers.snippets[check[2]]
]
# Decrement target string index.
child_len = get_num_from_str(check[2])
pre -= child_len[1] - child_len[0]
break
# Step 2: reduce everything after the loc.
for i, item in enumerate(alt_cs[loc + 1:]):
if NTs[item][1] == "T":
# This is a terminal, decrement by length of T.
# Check output of target string.
check = target[aft:aft + len(NTs[item][0])]
if check == NTs[item][0]:
# We have a match.
# Generate fake key for snippets dict.
key = str([aft, aft + len(NTs[item][0])])
# Create new tree from this terminal.
T_tree = Tree(check, None)
# Add to children.
children[item] = [key, T_tree]
# Increment target string index.
aft += len(NTs[item][0])
else:
# No match.
break
else:
# We haven't looked ahead in the string,
# we can't add things we don't know yet.
break
if all([child != [] for child in children]):
# We have expanded all children and can collapse
# a node.
key, pre, aft = generate_key_and_check(
pre, aft, reduce, children)
# Create a new node for the solution list.
new_entry = [[pre, aft], reduce[1], key]
# Add the new reduced entry to the solution.
if new_entry not in solution:
solution.insert(idx + 1, new_entry)
def parse_terminals(target):
"""
Given a target string, build up a list of terminals which match certain
portions of the target string.
:return: A list of terminals in order of appearance in the target string.
"""
if params['VERBOSE']:
print("Target:\n", target)
# Pre-load all terminals and non-terminal rules from the grammar.
terms, rules = params['BNF_GRAMMAR'].terminals, params['BNF_GRAMMAR'].rules
# Initialise dict for storing the snippets for compiling a complete
# solution. The key for each entry is the portion of the target string on
# which the output matches, along with the root node of the subtree. The
# value is the subtree itself.
trackers.snippets = {}
# Initialise dict for deleted snippets, to ensure they aren't generated
# again.
trackers.deleted_snippets = []
for T in sorted(terms.keys()):
# Iterate over all Terminals.
# Find all occurances of this terminal in the target string.
occurrances = []
index = 0
while index < len(target):
index = target.find(T, index)
if index not in occurrances and index != -1:
occurrances.append(index)
index += len(T)
else:
break
for idx in occurrances:
# Check each occurrence of this terminal in the target string.
for NT in terms[T]:
if any([[T] == i
for i in [[sym['symbol'] for sym in choice['choice']]
for choice in rules[NT]['choices']]]):
# Check if current T is the entire production choice of any
# particular rule.
# Generate a key for the snippets repository.
key = " ".join([str([idx, idx + len(T)]), NT])
# Get index of production choice.
index = [[sym['symbol'] for sym in choice['choice']]
for choice in rules[NT]['choices']].index([T])
# Get production choice.
choice = rules[NT]['choices'][index]['choice']
# Generate a tree for this choice.
parent = Tree(NT, None)
# Generate a codon for this choice.
parent.codon = generate_codon(NT, choice)
# Set the snippet key for the parent.
parent.snippet = key
# Create child for terminal.
child = Tree(T, parent)
# Add child to parent.
parent.children.append(child)
# Add snippet to snippets repository.
trackers.snippets[key] = parent
def generate_tree(tree, genome, output, method, nodes, depth, max_depth,
depth_limit):
"""
Recursive function to derive a tree using a given method.
:param tree: An instance of the Tree class.
:param genome: The list of all codons in a tree.
:param output: The list of all terminal nodes in a subtree. This is
joined to become the phenotype.
:param method: A string of the desired tree derivation method,
e.g. "full" or "random".
:param nodes: The total number of nodes in the tree.
:param depth: The depth of the current node.
:param max_depth: The maximum depth of any node in the tree.
:param depth_limit: The maximum depth the tree can expand to.
:return: genome, output, nodes, depth, max_depth.
"""
# Increment nodes and depth, set depth of current node.
nodes += 1
depth += 1
tree.depth = depth
# Find the productions possible from the current root.
productions = params['BNF_GRAMMAR'].rules[tree.root]
if depth_limit:
# Set remaining depth.
remaining_depth = depth_limit - depth
else:
remaining_depth = depth_limit
# Find which productions can be used based on the derivation method.
available = legal_productions(method, remaining_depth, tree.root,
productions['choices'])
# Randomly pick a production choice.
chosen_prod = choice(available)
# Find the index of the chosen production and set a matching codon based
# on that index.
prod_index = productions['choices'].index(chosen_prod)
codon = randrange(productions['no_choices'],
params['BNF_GRAMMAR'].codon_size,
productions['no_choices']) + prod_index
# Set the codon for the current node and append codon to the genome.
tree.codon = codon
genome.append(codon)
# Initialise empty list of children for current node.
tree.children = []
for symbol in chosen_prod['choice']:
# Iterate over all symbols in the chosen production.
if symbol["type"] == "T":
# The symbol is a terminal. Append new node to children.
tree.children.append(Tree(symbol["symbol"], tree))
# Append the terminal to the output list.
output.append(symbol["symbol"])
elif symbol["type"] == "NT":
# The symbol is a non-terminal. Append new node to children.
tree.children.append(Tree(symbol["symbol"], tree))
# recurse on the new node.
genome, output, nodes, d, max_depth = \
generate_tree(tree.children[-1], genome, output, method,
nodes, depth, max_depth, depth_limit)
NT_kids = [
kid for kid in tree.children
if kid.root in params['BNF_GRAMMAR'].non_terminals
]
if not NT_kids:
# Then the branch terminates here
depth += 1
nodes += 1
if depth > max_depth:
# Set new maximum depth
max_depth = depth
return genome, output, nodes, depth, max_depth
def pi_grow(tree, max_depth):
"""
Grows a tree until a single branch reaches a specified depth. Does this
by only using recursive production choices until a single branch of the
tree has reached the specified maximum depth. After that any choices are
allowed.
:param tree: An instance of the representation.tree.Tree class.
:param max_depth: The maximum depth to which to derive a tree.
:return: The fully derived tree.
"""
# Initialise derivation queue.
queue = [[
tree,
ret_true(params['BNF_GRAMMAR'].non_terminals[tree.root]['recursive'])
]]
# Initialise empty genome. With PI operators we can't use a depth-first
# traversal of the tree to build the genome, we need to build it as we
# encounter each node.
genome = []
while queue:
# Loop until no items remain in the queue.
# Pick a random item from the queue.
chosen = randint(0, len(queue) - 1)
# Pop the next item from the queue.
all_node = queue.pop(chosen)
node, recursive = all_node[0], all_node[0]
# Get depth of current node.
if node.parent is not None:
node.depth = node.parent.depth + 1
# Get maximum depth of overall tree.
_, overall_depth = get_nodes_and_depth(tree)
# Find the productions possible from the current root.
productions = params['BNF_GRAMMAR'].rules[node.root]
# Set remaining depth.
remaining_depth = max_depth - node.depth
if (overall_depth < max_depth) or \
(recursive and (not any([item[1] for item in queue]))):
# We want to prevent the tree from creating terminals until a
# single branch has reached the full depth. Only select recursive
# choices.
# Find which productions can be used based on the derivation method.
available = legal_productions("full", remaining_depth, node.root,
productions['choices'])
else:
# Any production choices can be made.
# Find which productions can be used based on the derivation method.
available = legal_productions("random", remaining_depth, node.root,
productions['choices'])
# Randomly pick a production choice.
chosen_prod = choice(available)
# Find the index of the chosen production and set a matching codon
# based on that index.
prod_index = productions['choices'].index(chosen_prod)
codon = randrange(productions['no_choices'],
params['BNF_GRAMMAR'].codon_size,
productions['no_choices']) + prod_index
# Set the codon for the current node and append codon to the genome.
node.codon = codon
# Insert codon into the genome.
genome.append(codon)
# Initialise empty list of children for current node.
node.children = []
for i, symbol in enumerate(chosen_prod['choice']):
# Iterate over all symbols in the chosen production.
# Create new child.
child = Tree(symbol["symbol"], node)
# Append new node to children.
node.children.append(child)
if symbol["type"] == "NT":
# The symbol is a non-terminal.
# Check whether child is recursive
recur_child = ret_true(params['BNF_GRAMMAR'].non_terminals[
child.root]['recursive'])
# Insert new child into the correct position in the queue.
queue.insert(chosen + i, [child, recur_child])
# genome, output, invalid, depth, and nodes can all be generated by
# recursing through the tree once.
_, output, invalid, depth, \
nodes = tree.get_tree_info(params['BNF_GRAMMAR'].non_terminals.keys(),
[], [])
return genome, output, nodes, depth
def genome_tree_map(tree, genome, output, index, depth, max_depth, nodes,
invalid=False):
"""
Recursive function which builds a tree using production choices from a
given genome. Not guaranteed to terminate.
:param tree: An instance of the representation.tree.Tree class.
:param genome: A full genome.
:param output: The list of all terminal nodes in a subtree. This is
joined to become the phenotype.
:param index: The index of the current location on the genome.
:param depth: The current depth in the tree.
:param max_depth: The maximum overall depth in the tree so far.
:param nodes: The total number of nodes in the tree thus far.
:param invalid: A boolean flag indicating whether or not the individual
is invalid.
:return: index, the index of the current location on the genome,
nodes, the total number of nodes in the tree thus far,
depth, the current depth in the tree,
max_depth, the maximum overall depth in the tree,
invalid, a boolean flag indicating whether or not the
individual is invalid.
"""
if not invalid and index < len(genome) * (params['MAX_WRAPS'] + 1):
# If the solution is not invalid thus far, and if we still have
# remaining codons in the genome, then we can continue to map the tree.
if params['MAX_TREE_DEPTH'] and (max_depth > params['MAX_TREE_DEPTH']):
# We have breached our maximum tree depth limit.
invalid = True
# Increment and set number of nodes and current depth.
nodes += 1
depth += 1
tree.id, tree.depth = nodes, depth
# Find all production choices and the number of those production
# choices that can be made by the current root non-terminal.
productions = params['BNF_GRAMMAR'].rules[tree.root]['choices']
no_choices = params['BNF_GRAMMAR'].rules[tree.root]['no_choices']
# Set the current codon value from the genome.
tree.codon = genome[index % len(genome)]
# Select the index of the correct production from the list.
selection = tree.codon % no_choices
# Set the chosen production
chosen_prod = productions[selection]
# Increment the index
index += 1
# Initialise an empty list of children.
tree.children = []
for symbol in chosen_prod['choice']:
# Add children to the derivation tree by creating a new instance
# of the representation.tree.Tree class for each child.
if symbol["type"] == "T":
# Append the child to the parent node. Child is a terminal, do
# not recurse.
tree.children.append(Tree(symbol["symbol"], tree))
output.append(symbol["symbol"])
elif symbol["type"] == "NT":
# Append the child to the parent node.
tree.children.append(Tree(symbol["symbol"], tree))
# Recurse by calling the function again to map the next
# non-terminal from the genome.
output, index, nodes, d, max_depth, invalid = \
genome_tree_map(tree.children[-1], genome, output,
index, depth, max_depth, nodes,
invalid=invalid)
else:
# Mapping incomplete, solution is invalid.
return output, index, nodes, depth, max_depth, True
# Find all non-terminals in the chosen production choice.
NT_kids = [kid for kid in tree.children if kid.root in
params['BNF_GRAMMAR'].non_terminals]
if not NT_kids:
# There are no non-terminals in the chosen production choice, the
# branch terminates here.
depth += 1
nodes += 1
if not invalid:
# The solution is valid thus far.
if depth > max_depth:
# Set the new maximum depth.
max_depth = depth
if params['MAX_TREE_DEPTH'] and (max_depth > params['MAX_TREE_DEPTH']):
# If our maximum depth exceeds the limit, the solution is invalid.
invalid = True
return output, index, nodes, depth, max_depth, invalid