def test_possible_insertion_points(self):
conv = Conv3x3()
after, before = mutation.get_possible_insertion_points(self.module, operation=conv)
self.assertEqual(len(after), 0, "Conv 2D could be inserted after a 1D layer or input.")
self.assertEqual(len(before), len(self.module.children)-1, "Should be able to be inserted before any except first.")
# Network setup:
# 2D input -> conv1 -> op1 -> op2
# \ /
# -> conv2 ->
module = Module()
conv1 = Conv5x5()
conv2 = Conv3x3()
op1 = dense.DenseS()
op2 = dense.DenseL()
module = mutation_ops.append(module, conv1)
module = mutation_ops.append(module, op1)
module = mutation_ops.append(module, op2)
module = mutation_ops.insert(module, first=conv1, last=op1, op=conv2)
after, before = mutation.get_possible_insertion_points(module, operation=conv)
self.assertIn(conv1, after, "Should be able to insert after conv1")
self.assertIn(conv2, after, "Should be able to insert after conv2")
self.assertEqual(len(after), 2, "More nodes in after than it should be...")
self.assertNotIn(conv1, before, "Should not be able to insert before first node.")
op3 = dense.DenseM()
after, before = mutation.get_possible_insertion_points(module, operation=op3)
self.assertEqual(len(after), 3, "Should have been able to insert after every node except last.")
self.assertIn(op1, after, "Op1 should have been in after list.")
self.assertIn(op1, before, "Op1 should have been in before list.")
self.assertIn(op2, before, "Op2 should have been in before list.")
def test_remove_node_multi_connections(self):
# Initial model: Final model:
# op1 -> op2 -> op3 op1 -> op3
# \ / \ /
# -> op4 -> -> op4 ->
# Setup:
op4 = dense.Dropout()
self.module = mutation_ops.insert(self.module, self.op2, self.op3, op=op4, between=False)
# Performing action:
self.module = mutation_ops.remove(self.module, op=self.op2)
# Testing if all connected nodes children of module
for child in self.module.children:
for node in child.prev:
self.assertIn(node, self.module.children)
for node in child.next:
self.assertIn(node, self.module.children)
# Testing each of the connections manually:
self.assertEqual(len(self.op1.prev), 0, "Wrong number of nodes connected to op1.prev")
self.assertEqual(len(self.op1.next), 2, "Wrong number of nodes connected to op1.next")
self.assertEqual(len(op4.prev), 1, "Wrong number of nodes connected to op4.prev")
self.assertEqual(len(op4.next), 1, "Wrong number of nodes connected to op4.next")
self.assertEqual(len(self.op3.prev), 2, "Wrong number of nodes connected to op3.prev")
self.assertEqual(len(self.op3.next), 0, "Wrong number of nodes connected to op3.next")
self.assertEqual(len(self.op2.next) + len(self.op2.prev), 0, "Removed node was not emptied")
def test_remove_last_node(self):
# Initial model should look like: Final model (no changes):
# op1 -> op2 -> op3 op1 -> op2 -> op3
# \ / \ /
# -> op4 -> -> op4 ->
op4 = dense.Dropout()
self.module = mutation_ops.insert(self.module, self.op2, self.op3, op=op4, between=False)
self.module = mutation_ops.remove(self.module, op=self.op3)
self.assertIn(self.op3, self.module.children, "op3 was removed when illegal to remove.")
self.assertEqual(len(self.op3.prev), 2, "op3 was disconnected from its previous...")
# Initial model: middel step model: final step model:
# op1 -> op2 -> op3 op1 -> op2 -> op3 -> op5 op1 -> op2 -> op3
# \ / \ / \ /
# -> op4 -> -> op4 -> -> op4 ->
# Adding a node to the end will make deletion possible:
op5 = dense.Dropout()
self.module = mutation_ops.append(self.module, op5)
self.assertEqual(self.module.find_last()[0], op5, "When appending op5, it was placed incorrectly...")
self.module = mutation_ops.remove(self.module, op5)
self.assertEqual(self.module.find_last()[0], self.op3, "Last operation was not op3 as expected after remove.")
self.assertEqual(len(self.op3.next), 0, "Next-connection not removed from op3.next")
self.assertEqual(len(self.op3.prev), 2, "previous connecitons for op3 was not maintained")
self.assertEqual(len(op5.prev), 0, "op5.prev was not emptied...")
def test_remove_node_single_connections(self):
# Initial model: Final model:
# op1 -> op2 -> op3 op1 -> op2 -> op3
# \ /
# -> op4 ->
op4 = dense.Dropout()
self.module = mutation_ops.insert(self.module, self.op2, self.op3, op=op4, between=False)
self.module = mutation_ops.remove(self.module, op=op4)
# Testing each of the connections manually:
self.assertEqual(len(self.op1.prev), 0, "self.op1.prev should have no connections...")
self.assertEqual(len(self.op1.next), 1, "self.op1.next is connected to more nodes than it should...")
self.assertEqual(len(self.op2.prev), 1, "self.op2.prev is connected to more nodes than it should...")
self.assertEqual(len(self.op2.next), 1, "self.op2.next is connected to more nodes than it should...")
self.assertEqual(len(self.op3.prev), 1, "self.op3.prev is connected to more nodes than it should...")
self.assertEqual(len(self.op3.next), 0, "self.op3.next should have no connections...")
self.assertEqual(len(op4.prev), 0, "self.op4.prev should have no connections...")
self.assertEqual(len(op4.next), 0, "self.op4.next should have no connections...")
# Testing which nodes is connected to which:
self.assertNotIn(op4, self.module.children, "Op4 is child of module after remove...")
self.assertNotIn(op4, self.op3.prev, "Op4 is previous node of op3 after remove...")
self.assertNotIn(op4, self.op2.next, "Op4 is next node for op2 after remove...")
self.assertNotIn(self.op2, op4.prev, "Op4 still connected to its next node...")
self.assertNotIn(self.op3, op4.next, "Op4 still connected to its previous node...")
# Testing for duplicate connections:
check_for_duplicates(self, self.op3.prev, "Found duplicate in list self.op3.prev...")
check_for_duplicates(self, self.op2.next, "Found duplicate in list self.op2.next...")
def sub_module_insert(module: Module, target_module: Module,
config: dict) -> Module:
# Copying both modules to avoid destructive changes:
target = deepcopy(target_module)
# Finding actionspace
action_space = []
for start_node in get_insertion_points_after(module, target):
for end_node in get_insertion_points_before(module, target,
start_node):
action_space += [(start_node, end_node)]
new = []
for predecessor_after, predecessor_before in action_space:
# Insert module non-destructive
mutated = deepcopy(module) # type: module
target = deepcopy(target_module)
# Finding nodes in new copy:
before, after = None, None
for child in mutated.children:
if child.ID == predecessor_before.ID:
before = child
elif child.ID == predecessor_after.ID:
after = child
if before and after: break
if not before or not after:
continue
# raise UnboundLocalError(
# "Insert will fail when either before ({}) or after ({}) is None".format(before, after)
# )
# Transferring weights and assembles module:
mutated = insert(mutated, after, before, target)
new += [mutated]
if new:
new_config = deepcopy(config)
new_config.epochs = 0
scheduler.queue_all(new, new_config, priority=True)
scheduler.await_all_jobs_finish()
new.sort(key=lambda x: x.fitness[-1] if x.fitness else 0.0)
best = new[-1]
print(" - Accuracy of new ({}) vs old ({})".format(
best.fitness[-1], module.fitness[-1]))
return best
else:
print(" - No new solutions found...")
return None
def test_insert(self):
# Final product should look like:
# op1 -> op2 -> op3
# \ /
# -> op4 ->
op4 = dense.Dropout()
self.module = mutation_ops.insert(self.module, self.op3, self.op2, op=op4, between=False)
self.assertIn(op4, self.module.children, "Op4 not child of module...")
self.assertIn(op4, self.op2.next, "Op4 was expected to be in next list of op2, but was not.")
self.assertIn(op4, self.op3.prev, "Op4 was expected to be in prev list of op3, but was not.")
self.assertIn(self.op2, self.op3.prev, "Op4 was expected to be in prev list of op3, but was not.")
self.assertIn(self.op3, self.op2.next, "Op4 was expected to be in prev list of op3, but was not.")
def test_dropout_is_placed_correctly_in_complex_assembled_network(self):
from tensorflow import keras
from src.frameworks.keras import module_to_model
module = Module(name="TestDropout")
l1, l2, l3, l4, l5, l6 = Conv5x5(), DenseL(), Conv3x3(dropout=False), AvgPooling2x2(), DenseL(), Conv3x3(dropout=False)
# Building model with form:
# -> l5 ->
# / \
# |----------> l3 ->
# | \
# -> l1 -------> l2 ------> l5 ->
# \ /
# -----> l4 ----->
module = mutate.append(module, l1)
module = mutate.append(module, l2)
module = mutate.append(module, l5)
module = mutate.insert(module, first=l1, last=l5, op=l3, between=False)
module = mutate.insert(module, first=l1, last=l5, op=l4, between=False)
module = mutate.insert(module, first=l1, last=l3, op=l6, between=False)
# Assemble and display:
model = module_to_model(module, input_shape=[32, 32, 3], classes=10)
keras.utils.plot_model(model, to_file="tests/output/TestDropoutComplex.png")
def test_dropout_is_placed_correctly_in_simple_assembled_network(self):
from tensorflow import keras
from src.frameworks.keras import module_to_model
module = Module(name="TestDropout")
l1, l2, l3, l4 = DenseL(), DenseM(), DenseS(), DenseM()
# Building model like:
# l1 -> l2 -> l3
# \ /
# -> l4 ->
module = mutate.append(module, l1)
module = mutate.append(module, l2)
module = mutate.append(module, l3)
module = mutate.insert(module, first=l1, last=l3, op=l4, between=False)
# Assemble and display:
model = module_to_model(module, input_shape=[784], classes=10)
keras.utils.plot_model(model, to_file="tests/output/TestDropoutSimple.png")
def test_insert_between(self):
# Final product should look like:
# op1 -> op2 -> op4 -> op3
op4 = dense.Dropout()
self.module = mutation_ops.insert(self.module, self.op2, self.op3, op=op4, between=True)
self.assertIn(op4, self.module.children, "Op4 not child of module...")
self.assertIn(op4, self.op2.next, "Op4 was expected to be in next list of op2, but was not.")
self.assertIn(op4, self.op3.prev, "Op4 was expected to be in prev list of op3, but was not.")
self.assertNotIn(self.op2, self.op3.prev, "Op4 was not expected to be in prev list of op3, but was.")
self.assertNotIn(self.op3, self.op2.next, "Op4 was not expected to be in prev list of op3, but was.")
self.assertTrue(len(op4.prev) == len(op4.next) == 1, "Wrong number of elements in op4 connections.")
for i, op in enumerate(self.module.children[:-1]):
expected_next = 1 if i < len(self.module.children[:-1]) -1 else 0
self.assertEqual(len(op.next), expected_next, "Next list contained wrong number of children")
expected_prev = 1 if i > 0 else 0
self.assertEqual(len(op.prev), expected_prev, "Next list contained wrong number of children")
def apply_mutation_operator(module: Module, operator: Base,
operators: list) -> Module:
if operator == "append":
last = module.find_last()[0]
if is1D(last):
operation = random_sample(operators1D_votes)()
else:
operation = random_sample(operators2D_votes + operators1D_votes)()
module = append(module, operation)
elif operator == "remove":
module = remove(module, random_sample(module.children))
elif operator == "insert" or operator == "insert-between":
operation = random_sample(operators)()
first, last = find_placement(module, operation)
i = 0
while not first or not last:
operation = random_sample(operators)()
first, last = find_placement(module, operation)
i += 1
if i == 20:
break
if first and last:
module = insert(module, first, last, operation,
operator == "insert-between")
elif operator == "connect":
possibilities = list(range(len(module.children)))
module = connect(
module=module,
first=module.children[possibilities.pop(
random.randint(0,
len(possibilities) - 1))],
last=module.children[possibilities.pop(
random.randint(0,
len(possibilities) - 1))],
)
# Else: operator == "identity": do nothing...
return module