def test_crossbreed(self):
"""
Test if crossbreeding works properly and results in a entirely new WeightSet
"""
# Get two arbitrary arrays
for _ in range(1000):
# Get arbitrary array for crossbreeding
ws2 = WeightSet(layer_sizes=self.layer_sizes)
crossbreeded_ws = WeightSet.crossbreed(self.weightset, ws2)
# Crossbreeded WeightSet cannot be the same as the input instances
self.assertNotEqual(crossbreeded_ws, self.weightset)
self.assertNotEqual(crossbreeded_ws, ws2)
# Crossbreeded weights cannot be the same as either of the initial weights
self.assertNotEqual(crossbreeded_ws.weights,
self.weightset.weights)
self.assertNotEqual(crossbreeded_ws.weights, ws2.weights)
# Crossbreeded weightset must be a WeightSet instance
self.assertIsInstance(crossbreeded_ws, WeightSet)
self.assertIsInstance(crossbreeded_ws.weights, np.ndarray)
# Activation function must be the same as the first weightset
self.assertEqual(self.weightset.activation_name,
crossbreeded_ws.activation_name)
def reset(self):
"""
Initialize arbitrary WeightSet
"""
# Initialize random layers and sizes
model_depth = np.random.randint(3, 10)
self.layer_sizes = [
np.random.randint(2, 50) for _ in range(model_depth)
]
# Output size must be smaller than last hidden layer
self.output_size = np.random.randint(1, self.layer_sizes[-1])
self.weightset = WeightSet(layer_sizes=self.layer_sizes +
[self.output_size])
# Save input size for testing
self.input_size = self.layer_sizes[0]
def generate_next_generation(self):
"""
With the current generation and the achieved scores of each WeightSet, construct a new set of WeightSets
to use for the next generation.
"""
# Create a list for the new population we are creating
new_population = list()
# Sort the population based on the achieved scores
sorted_population = [
weight_set
for _, weight_set in sorted(zip(self.scores, self.population),
key=lambda pair: pair[0],
reverse=True)
]
# Calculate how many of elites, crossovers and mutations we will have in the new population
num_elites = int(self.POPULATION_SIZE * self.ELITE_FRACTION)
num_crossovers = int(
(self.POPULATION_SIZE - num_elites) * self.CROSSOVER_FRACTION)
num_mutations = self.POPULATION_SIZE - num_elites - num_crossovers
# Let the best elite percentage directly carry over
new_population.extend(sorted_population[:num_elites])
# Scale the scores so that together they sum to 1 (used to then sample randomly from it)
if np.sum(self.scores) <= 0:
# Prevent division by 0 if no WeightSet achieved any points
scaled_scores = np.full(self.POPULATION_SIZE,
1.0 / self.POPULATION_SIZE)
else:
scaled_scores = np.array(self.scores) / np.sum(self.scores)
# Cross-overs from 2 parents sampled with probability linked to their score
for parent1, parent2 in np.random.choice(self.population,
p=scaled_scores,
size=(num_crossovers, 2)):
# New child from crossover
child = WeightSet.crossbreed(parent1, parent2)
# Also mutate a little bit if wanted
if self.CROSSOVER_MUTATION_PROBABILITY > 0:
child.mutate(self.CROSSOVER_MUTATION_PROBABILITY)
new_population.append(child)
# Mutate the remainder from current WeightSets sampled with probability linked to their score
for parent in np.random.choice(self.population,
p=scaled_scores,
size=num_mutations):
mutant: WeightSet = parent.clone()
mutant.mutate(self.MUTATION_PROBABILITY)
new_population.append(mutant)
# Replace the old generation's population with the new population (which now does not have a tested score yet)
self.population = new_population
self.scores = list()
def test_action(self):
"""
Make sure the action is gotten from the (correct) active WeightSet by feed-forwarding the input
"""
GeneticModel.POPULATION_SIZE = 50
genetic_model = GeneticModel(
game_name="SnakeGen-v1",
input_shape=(2, ),
action_space=self.environment.action_space)
for i in {0, 4, 10, 24, 49}:
weight_set = WeightSet(layer_sizes=[2, 4,
5]) # Create a new WeightSet
genetic_model.current_weight_set_id = i # Select the i'th WeightSet
genetic_model.population[
i] = weight_set # Insert our WeightSet in spot i in the population
inputs = [(i * 17) % 21, (i * 11) % 7] # Slightly random input
# Make sure the action gotten from the genetic model is actually the action you get when
# you feedforward our input in our WeightSet that we injected
self.assertEqual(genetic_model.action(inputs),
weight_set.feedforward(inputs))
def __init__(self,
game_name,
input_shape,
action_space,
logger_path="output/genetic_model"):
"""
Initialize the population
"""
# Initialize variables
self.population = list(
) # Set of WeightSets containing the weights of all agents in the current population
self.scores = list(
) # The scores that the WeightSets of the population achieved when running a game
self.current_weight_set_id = 0 # The index of the WeightSet that is currently playing
self.generation_id = 0 # The generation number we are currently in
self.generation_outcomes = list() # Stores results for each generation
self.weight_set_score = list(
) # The scores of the GAMES_PER_GENERATION games played with one WeightSet
# Calculate sizes of all layers of the neural net
input_size = int(np.prod(input_shape))
output_size = action_space.n
layer_sizes = [input_size, *self.HIDDEN_LAYERS, output_size]
# Populate with POPULATION_SIZE nets / WeightSets
for i in range(self.POPULATION_SIZE):
self.population.append(WeightSet(layer_sizes))
# If enabled, create a replayer that plays games with the best WeightSets of the previous generation
if self.RENDER_BEST_WEIGHTSETS:
self.replayer = PlayWeightSet(game_name, self.population[0])
self.replayer.start()
# Prevent any plots from making the terminal hang
if self.PLOT_STATS:
plt.ion()
# If saving certain logs, make sure the folders for it exists
if self.SAVE_BEST_WEIGHTSETS or self.SAVE_GEN_SCORES:
# Make sure the LOGS folder exists
if not os.path.exists(self.dir_logs):
os.makedirs(self.dir_logs)
if self.SAVE_GEN_SCORES:
# Make sure the SCORES folder exists
if not os.path.exists(self.dir_scores):
os.makedirs(self.dir_scores)
if self.SAVE_GEN_SCORES:
# Make sure the WEIGHTSETS folder exists
if not os.path.exists(self.dir_weightsets):
os.makedirs(self.dir_weightsets)
super().__init__(game_name, input_shape, action_space, logger_path)
class TestWeightSet(unittest.TestCase):
"""
Test class for the WeightSet class.
"""
def setUp(self):
"""
Setup WeightSet for every test
"""
# Initialize random layers and sizes
model_depth = np.random.randint(3, 10)
self.layer_sizes = [
int(np.random.randint(2, 50)) for _ in range(model_depth)
]
# Output size must be smaller than last hidden layer
self.output_size = np.random.randint(1, self.layer_sizes[-1])
self.weightset = WeightSet(layer_sizes=self.layer_sizes +
[self.output_size])
# Save input size for testing
self.input_size = self.layer_sizes[0]
def reset(self):
"""
Initialize arbitrary WeightSet
"""
# Initialize random layers and sizes
model_depth = np.random.randint(3, 10)
self.layer_sizes = [
np.random.randint(2, 50) for _ in range(model_depth)
]
# Output size must be smaller than last hidden layer
self.output_size = np.random.randint(1, self.layer_sizes[-1])
self.weightset = WeightSet(layer_sizes=self.layer_sizes +
[self.output_size])
# Save input size for testing
self.input_size = self.layer_sizes[0]
def test_clone(self):
"""
Test cloning (deepcopy)
"""
clone = self.weightset.clone()
# Test if they are not the same instance
self.assertNotEqual(self.weightset, clone)
# Test if arguments are the same
self.assertEqual(self.weightset.layer_sizes, clone.layer_sizes)
self.assertEqual(self.weightset.activation, clone.activation)
self.assertEqual(self.weightset.initialization_name,
clone.initialization_name)
# Test if deepcopy argument change does not change original
clone.activation = 'foo'
clone.layer_sizes = [-999, -999]
clone.initialization_name = 'foo'
self.assertNotEqual(self.weightset.activation, clone.activation)
self.assertNotEqual(self.weightset.activation, clone.activation)
self.assertNotEqual(self.weightset.initialization_name,
clone.initialization_name)
def test_randn_init(self):
"""
Test random initialization of weights
"""
for _ in range(1000):
self.reset()
self.assertIsInstance(self.weightset.weights, np.ndarray)
def test_feedforward(self):
"""
Test the feedforward method
"""
for _ in range(100):
# Get random set of weights
self.reset()
# The feedforward pass must work on all activation functions
for act_func in self.weightset.activation_dir.values():
self.weightset.activation = act_func
# Make feedforward pass
model_input = np.random.uniform(-999, 999, self.input_size)
output = self.weightset.feedforward(model_input)
# Output must be an integer
self.assertIsInstance(output, np.int64)
def test_mutate(self):
"""
Test if mutate changes the weights in-place
"""
for _ in range(1000):
mutation_probability = np.random.uniform(0.00001, 1)
weights_before = deepcopy(self.weightset.weights)
self.weightset.mutate(mutation_probability)
weights_after = self.weightset.weights
# Weights cannot be the same after mutating if mutation_probability > 0
self.assertNotEqual(weights_before, weights_after)
def test_crossbreed(self):
"""
Test if crossbreeding works properly and results in a entirely new WeightSet
"""
# Get two arbitrary arrays
for _ in range(1000):
# Get arbitrary array for crossbreeding
ws2 = WeightSet(layer_sizes=self.layer_sizes)
crossbreeded_ws = WeightSet.crossbreed(self.weightset, ws2)
# Crossbreeded WeightSet cannot be the same as the input instances
self.assertNotEqual(crossbreeded_ws, self.weightset)
self.assertNotEqual(crossbreeded_ws, ws2)
# Crossbreeded weights cannot be the same as either of the initial weights
self.assertNotEqual(crossbreeded_ws.weights,
self.weightset.weights)
self.assertNotEqual(crossbreeded_ws.weights, ws2.weights)
# Crossbreeded weightset must be a WeightSet instance
self.assertIsInstance(crossbreeded_ws, WeightSet)
self.assertIsInstance(crossbreeded_ws.weights, np.ndarray)
# Activation function must be the same as the first weightset
self.assertEqual(self.weightset.activation_name,
crossbreeded_ws.activation_name)
def test_get_init_std(self):
"""
Test if all initialization schemes are working properly
"""
# Create arbitrary values for rows and columns
for _ in range(1000):
rows = np.random.randint(1, 1000)
columns = np.random.randint(1, 1000)
for init in initializations:
self.weightset.initialization_name = init
std = self.weightset.get_init_std(rows, columns)
self.assertIsInstance(std, np.float64)
self.assertGreater(std, 0)
def test_activations(self):
"""
Test all activations that are included in the activation directory
"""
for _ in range(100):
# Initialize random weights and get arbitrary layer
self.weightset.weights = self.weightset.randn_init()
weight_array = self.weightset.weights[0][0]
bias_array = self.weightset.weights[0][0]
for act_func in self.weightset.activation_dir.values():
result1 = act_func(weight_array)
result2 = act_func(bias_array)
self.assertIsInstance(result1, np.ndarray)
self.assertIsInstance(result2, np.ndarray)