def create_new(self, config: Config, num_genomes): """Create a new (random initialized) population.""" new_genomes = dict() for i in range(num_genomes): key = next(self.genome_indexer) g = Genome(key, num_outputs=config.genome.num_outputs, bot_config=config.bot) g.configure_new(config.genome) new_genomes[key] = g return new_genomes
def get_invalid5(cfg: Config): """ Genome with connections between the hidden nodes and to one output node. Configuration: 0 1 | 2--3 -1 -2 -3 """ # Create a dummy genome genome = Genome( key=4, num_outputs=cfg.genome.num_outputs, bot_config=cfg.bot, ) # Reset the nodes genome.nodes[0] = OutputNodeGene(key=0, cfg=cfg.genome) # OutputNode 0 genome.nodes[0].bias = 0 genome.nodes[1] = OutputNodeGene(key=1, cfg=cfg.genome) # OutputNode 1 genome.nodes[1].bias = 0 genome.nodes[2] = SimpleNodeGene(key=2, cfg=cfg.genome) # Hidden node genome.nodes[2].bias = 0 genome.nodes[3] = SimpleNodeGene(key=3, cfg=cfg.genome) # Hidden node genome.nodes[3].bias = 0 # Reset the connections genome.connections = dict() for key in [(2, 3), (3, 2), (3, 1)]: genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[key].weight = 1 genome.connections[key].enabled = True return genome
def get_invalid4(cfg: Config): """ Genome with connection from start to recurrent node, and from another recurrent node to the output. Configuration: 0 1 | 2> 3> | -1 -2 -3 """ # Create a dummy genome genome = Genome( key=4, num_outputs=cfg.genome.num_outputs, bot_config=cfg.bot, ) # Reset the nodes genome.nodes[0] = OutputNodeGene(key=0, cfg=cfg.genome) # OutputNode 0 genome.nodes[0].bias = 0 genome.nodes[1] = OutputNodeGene(key=1, cfg=cfg.genome) # OutputNode 1 genome.nodes[1].bias = 0 genome.nodes[2] = SimpleNodeGene(key=2, cfg=cfg.genome) # Hidden node genome.nodes[2].bias = 0 genome.nodes[3] = SimpleNodeGene(key=3, cfg=cfg.genome) # Hidden node genome.nodes[3].bias = 0 # Reset the connections genome.connections = dict() for key in [(-1, 2), (2, 2), (3, 3), (3, 1)]: genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[key].weight = 1 genome.connections[key].enabled = True return genome
def get_circular2(cfg: Config): """ Genome with circular connections, not connected to the output genome. Configuration: 0 1 2---3 | | -1 -2 -3 """ # Create a dummy genome genome = Genome( key=2, num_outputs=cfg.genome.num_outputs, bot_config=cfg.bot, ) # Reset the nodes genome.nodes[0] = OutputNodeGene(key=0, cfg=cfg.genome) # OutputNode 0 genome.nodes[0].bias = 0 genome.nodes[1] = OutputNodeGene(key=1, cfg=cfg.genome) # OutputNode 1 genome.nodes[1].bias = 0 genome.nodes[2] = SimpleNodeGene(key=2, cfg=cfg.genome) # Hidden node genome.nodes[2].bias = 0 genome.nodes[3] = SimpleNodeGene(key=3, cfg=cfg.genome) # Hidden node genome.nodes[3].bias = 0 # Reset the connections genome.connections = dict() for key in [(-1, 2), (2, 3), (3, 2), (-2, 3)]: genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[key].weight = 1 genome.connections[key].enabled = True return genome
def get_topology1(cfg, random_init: bool = False): """ Simple genome with only two connections: 0 1 / 2 \ -1 """ # Create an initial dummy genome with fixed configuration genome = Genome( key=0, num_outputs=cfg.genome.num_outputs, bot_config=cfg.bot, ) # Create the nodes genome.nodes[0] = OutputNodeGene(key=0, cfg=cfg.genome) # OutputNode 0 genome.nodes[0].bias = 0 # Drive with 0.5 actuation by default genome.nodes[1] = OutputNodeGene(key=1, cfg=cfg.genome) # OutputNode 1 genome.nodes[1].bias = 0 # Drive with 0.5 actuation by default genome.nodes[2] = GruNodeGene(key=2, cfg=cfg.genome, input_keys=[-1], input_keys_full=[-1]) # Hidden node genome.nodes[2].bias = 0 # Bias is irrelevant for GRU-node if not random_init: genome.nodes[2].bias_h = np.zeros((3, )) genome.nodes[2].weight_xh_full = np.zeros((3, 1)) genome.nodes[2].weight_hh = np.zeros((3, 1)) # Create the connections genome.connections = dict() # Input-GRU key = (-1, 2) genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[key].weight = 1 # Simply forward distance genome.connections[key].enabled = True # GRU-Output key = (2, 0) genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[ key].weight = 3 # Increase magnitude to be a value between -3..3 (possible to steer left wheel) genome.connections[key].enabled = True genome.update_rnn_nodes(config=cfg.genome) return genome
def get_pruned2(cfg: Config): """ Genome with partially valid connections and nodes (dangling node on other hidden node). Configuration: 0 1 / 2---3> | -1 -2 -3 """ # Create a dummy genome genome = Genome( key=2, num_outputs=cfg.genome.num_outputs, bot_config=cfg.bot, ) # Reset the nodes genome.nodes[0] = OutputNodeGene(key=0, cfg=cfg.genome) # OutputNode 0 genome.nodes[0].bias = 0 genome.nodes[1] = OutputNodeGene(key=1, cfg=cfg.genome) # OutputNode 1 genome.nodes[1].bias = 0 genome.nodes[2] = SimpleNodeGene(key=2, cfg=cfg.genome) # Hidden node genome.nodes[2].bias = 0 genome.nodes[3] = SimpleNodeGene(key=3, cfg=cfg.genome) # Hidden node genome.nodes[3].bias = 0 # Reset the connections genome.connections = dict() for key in [(-1, 2), (2, 0), (2, 3), (3, 3)]: genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[key].weight = 1 genome.connections[key].enabled = True return genome
def positions( genome: Genome, genome_plus: Genome, genome_minus: Genome, save_name: str, gid: int, duration: int = 60, title: str = '', ): """Visualize the deviated genomes.""" # Check if valid genome (contains at least one hidden GRU, first GRU is monitored) assert len([ n for n in genome.get_used_nodes().values() if type(n) == GruNodeGene ]) >= 1 assert len([ n for n in genome_plus.get_used_nodes().values() if type(n) == GruNodeGene ]) >= 1 assert len([ n for n in genome_minus.get_used_nodes().values() if type(n) == GruNodeGene ]) >= 1 # Trace the genomes positions, game = get_positions(genome=genome, gid=gid, duration=duration) positions_plus, _ = get_positions(genome=genome_plus, gid=gid, duration=duration) positions_minus, _ = get_positions(genome=genome_minus, gid=gid, duration=duration) # Visualize the genome paths pos_dict = dict() pos_dict['default'] = positions pos_dict[f'plus'] = positions_plus pos_dict[f'minus'] = positions_minus path = get_save_path(gid=genome.key, save_name=save_name) visualize_positions( positions=pos_dict, game=game, annotate_time=False, save_path=f"{path}.png", show=False, title=title, )
def store_genome(genome: Genome, g_name: str = None): """Persist a single genome.""" if not g_name: genomes = glob(f"genomes_gru/genomes/genome*.pickle") g_name = f"genome{len(genomes) + 1}" genome.key = len(genomes) + 1 with open(f"genomes_gru/genomes/{g_name}.pickle", 'wb') as f: return pickle.dump(genome, f)
def get_invalid3(cfg: Config): """ Genome without connections to the input nodes. Configuration: 0 1 | 2> -1 -2 -3 """ # Create a dummy genome genome = Genome( key=3, num_outputs=cfg.genome.num_outputs, bot_config=cfg.bot, ) # Reset the nodes genome.nodes[0] = OutputNodeGene(key=0, cfg=cfg.genome) # OutputNode 0 genome.nodes[0].bias = 0 genome.nodes[1] = OutputNodeGene(key=1, cfg=cfg.genome) # OutputNode 1 genome.nodes[1].bias = 0 genome.nodes[2] = SimpleNodeGene(key=2, cfg=cfg.genome) # Hidden node genome.nodes[2].bias = 0 # Reset the connections genome.connections = dict() for key in [(2, 2), (2, 1)]: genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[key].weight = 1 genome.connections[key].enabled = True return genome
def get_valid2(cfg: Config): """ Network with a recurrent connection (at node 2). Configuration: 0 1 / 2> / \ -1 -2 -3 """ # Create a dummy genome genome = Genome( key=2, num_outputs=cfg.genome.num_outputs, bot_config=cfg.bot, ) # Reset the nodes genome.nodes[0] = OutputNodeGene(key=0, cfg=cfg.genome) # OutputNode 0 genome.nodes[0].bias = 0 genome.nodes[1] = OutputNodeGene(key=1, cfg=cfg.genome) # OutputNode 1 genome.nodes[1].bias = 0 genome.nodes[2] = SimpleNodeGene(key=2, cfg=cfg.genome) # Hidden node genome.nodes[2].bias = 0 # Reset the connections genome.connections = dict() for key in [(-1, 2), (-2, 2), (2, 0), (2, 2)]: genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[key].weight = 1 genome.connections[key].enabled = True return genome
def get_valid1(cfg: Config): """ Simple network with only one input and one output used. Configuration: 0 1 | | | -1 -2 -3 """ # Create a dummy genome genome = Genome( key=1, num_outputs=cfg.genome.num_outputs, bot_config=cfg.bot, ) # Reset the nodes genome.nodes[0] = OutputNodeGene(key=0, cfg=cfg.genome) # OutputNode 0 genome.nodes[0].bias = 0 genome.nodes[1] = OutputNodeGene(key=1, cfg=cfg.genome) # OutputNode 1 genome.nodes[1].bias = 0 genome.nodes[2] = SimpleNodeGene(key=2, cfg=cfg.genome) # Hidden node genome.nodes[2].bias = 0 # Reset the connections genome.connections = dict() for key in [(-3, 1)]: genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[key].weight = 1 genome.connections[key].enabled = True return genome
def get_genome2(cfg: Config): """ Genome with all biases set to 0, only simple hidden nodes used, all connections enabled with weight 1. Configuration: 0 1 / | 2 \ / | -1 -2 -3 """ # Create a dummy genome genome = Genome( key=2, num_outputs=cfg.genome.num_outputs, bot_config=cfg.bot, ) # Reset the nodes genome.nodes[0] = OutputNodeGene(key=0, cfg=cfg.genome) # OutputNode 0 genome.nodes[0].bias = 0 genome.nodes[1] = OutputNodeGene(key=1, cfg=cfg.genome) # OutputNode 1 genome.nodes[1].bias = 0 genome.nodes[2] = SimpleNodeGene(key=2, cfg=cfg.genome) # Hidden node genome.nodes[2].bias = 0 # Reset the connections genome.connections = dict() for key in [(-1, 2), (2, 0), (-3, 1)]: genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[key].weight = 1 genome.connections[key].enabled = True return genome
def get_deep_genome3(cfg: Config): """ Genome with all biases set to 0, only simple hidden nodes used, all connections enabled with weight 1. Configuration: 0 1 | | 16 | | \ | 15 | | | | | 14 | | | | | -1 -2 -3 """ # Create a dummy genome genome = Genome( key=3, num_outputs=cfg.genome.num_outputs, bot_config=cfg.bot, ) # Reset the nodes genome.nodes[0] = OutputNodeGene(key=0, cfg=cfg.genome) # OutputNode 0 genome.nodes[0].bias = 0 genome.nodes[1] = OutputNodeGene(key=1, cfg=cfg.genome) # OutputNode 1 genome.nodes[1].bias = 0 for k in [14, 15, 16]: genome.nodes[k] = SimpleNodeGene(key=k, cfg=cfg.genome) # Hidden node genome.nodes[k].bias = 0 # Reset the connections genome.connections = dict() for key in [(-1, 14), (14, 15), (15, 16), (16, 0), (-2, 16), (-3, 1)]: genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[key].weight = 1 genome.connections[key].enabled = True return genome
def monitor( game_id: int, population: Population, debug: bool = False, duration: int = 0, genome: Genome = None, ): """Monitor a single run of the given genome that contains a single GRU-node.""" print("\n===> MONITORING GENOME <===\n") if genome is None: genome = population.best_genome game_config = deepcopy(population.config) if duration > 0: game_config.game.duration = duration # Take first GRU or SRU node node_type = None for n in genome.get_used_nodes().values(): t = type(n) if t != OutputNodeGene and t != SimpleNodeGene: node_type = t break if node_type is None: raise Exception(f"No hidden node to monitor in genome {genome}") if node_type == GruNodeGene: from population.utils.visualizing.monitor_genome_single_gru import main as gru_monitor gru_monitor( population=population, game_id=game_id, genome=genome, game_cfg=game_config, debug=debug, ) elif node_type == GruNoResetNodeGene: from population.utils.visualizing.monitor_genome_single_gru_nr import main as gru_nr_monitor gru_nr_monitor( population=population, game_id=game_id, genome=genome, game_cfg=game_config, debug=debug, ) elif node_type == SimpleRnnNodeGene or node_type == FixedRnnNodeGene: from population.utils.visualizing.monitor_genome_single_sru import main as sru_monitor sru_monitor( average=2, population=population, game_id=game_id, genome=genome, game_cfg=game_config, debug=debug, ) elif node_type == LstmNodeGene: from population.utils.visualizing.monitor_genome_single_lstm import main as lstm_monitor lstm_monitor( population=population, game_id=game_id, genome=genome, game_cfg=game_config, debug=debug, ) else: raise Exception(f"Not able to monitor the genome of config:\n{genome}")
def get_positions(genome: Genome, gid: int, debug: bool = False, duration: int = 60): """Get the position of the genome at every 0.5 seconds during the given simulation.""" cfg = Config() cfg.game.duration = duration cfg.update() # Check if valid genome (contains at least one hidden GRU, first GRU is monitored) assert len([ n for n in genome.get_used_nodes().values() if type(n) == GruNodeGene ]) >= 1 # Get the game game = get_game(i=gid, cfg=cfg, noise=False) state = game.reset()[D_SENSOR_LIST] step_num = 0 # Create the network net = make_net( genome=genome, genome_config=cfg.genome, batch_size=1, initial_read=state, ) # Containers to monitor position = [] target_found = [] score = 0 # Initialize the containers position.append(game.player.pos.get_tuple()) if debug: print(f"Step: {step_num}") print( f"\t> Position: {(round(position[-1][0], 2), round(position[-1][1], 2))!r}" ) print(f"\t> Score: {score!r}") # Start monitoring while True: # Check if maximum iterations is reached if step_num == duration * cfg.game.fps: break # Determine the actions made by the agent for each of the states action = net(np.asarray([state])) # Check if each game received an action assert len(action) == 1 # Proceed the game with one step, based on the predicted action obs = game.step(l=action[0][0], r=action[0][1]) finished = obs[D_DONE] # Update the score-count if game.score > score: target_found.append(step_num) score = game.score # Update the candidate's current state state = obs[D_SENSOR_LIST] # Stop if agent reached target in all the games if finished: break step_num += 1 # Update the containers position.append(game.player.pos.get_tuple()) if debug: print(f"Step: {step_num}") print( f"\t> Position: {(round(position[-1][0], 2), round(position[-1][1], 2))!r}" ) print(f"\t> Score: {score!r}") return position, game
def main( genome: Genome, gid: int, delta: float = 1e-1, duration: int = 60, mut_bias: bool = False, mut_hh: bool = False, mut_xh: bool = False, ): """Load the genome, deviate candidate hidden state's weights and bias and plot the results.""" # Check if valid genome (contains at least one hidden GRU, first GRU is monitored) assert len([ n for n in genome.get_used_nodes().values() if type(n) == GruNodeGene ]) >= 1 # Get the GRU node's ID gru_id = None for nid, n in genome.get_used_nodes().items(): if type(n) == GruNodeGene: gru_id = nid break # Get the deviated genomes genome_plus = deepcopy(genome) genome_minus = deepcopy(genome) # Perform the requested mutations if mut_bias: genome_plus.nodes[gru_id].bias_h[2] += delta genome_minus.nodes[gru_id].bias_h[2] -= delta if mut_hh: genome_plus.nodes[gru_id].weight_hh[2, 0] += delta genome_minus.nodes[gru_id].weight_hh[2, 0] -= delta if mut_xh: genome_plus.nodes[gru_id].weight_xh_full[2, 0] += delta genome_plus.nodes[gru_id].update_weight_xh() genome_minus.nodes[gru_id].weight_xh_full[2, 0] -= delta genome_minus.nodes[gru_id].update_weight_xh() # Plot the positions name = '' if mut_bias: name += f"bias{delta}" if mut_hh: name += f"{'_' if name else ''}hh{delta}" if mut_xh: name += f"{'_' if name else ''}xh{delta}" plot_positions( genome=genome, genome_plus=genome_plus, genome_minus=genome_minus, save_name=f'candidate_hidden/{name}_trajectory', gid=gid, duration=duration, title="candidate hidden state", ) # Plot the activations default = monitor_activation(genome=genome, gid=gid, duration=duration) plus = monitor_activation(genome=genome_plus, gid=gid, duration=duration) minus = monitor_activation(genome=genome_minus, gid=gid, duration=duration) states = dict() states['default'] = default states['plus'] = plus states['minus'] = minus plot_states( gid=genome.key, states=states, save_name=f"candidate_hidden/{name}", ) # Merge the two graphs together merge( gid=genome.key, save_name=f"candidate_hidden/{name}", )
def main(population: Population, game_id: int, genome: Genome = None, game_cfg: Config = None, average: int = 1, debug: bool = False): """ Monitor the genome on the following elements: * Position * Hidden state of SRU (Ht) * Actuation of both wheels * Distance * Delta distance """ # Make sure all parameters are set if not genome: genome = population.best_genome if not game_cfg: game_cfg = pop.config # Check if valid genome (contains at least one hidden SRU, first SRU is monitored) - also possible for fixed RNNs a = len([n for n in genome.get_used_nodes().values() if type(n) == SimpleRnnNodeGene]) >= 1 b = len([n for n in genome.get_used_nodes().values() if type(n) == FixedRnnNodeGene]) >= 1 assert a or b # Get the game game = get_game(game_id, cfg=game_cfg, noise=False) state = game.reset()[D_SENSOR_LIST] step_num = 0 # Create the network net = make_net(genome=genome, genome_config=population.config.genome, batch_size=1, initial_read=state, ) # Containers to monitor actuation = [] distance = [] delta_distance = [] position = [] Ht = [] target_found = [] score = 0 # Initialize the containers actuation.append([0, 0]) distance.append(state[0]) delta_distance.append(0) position.append(game.player.pos.get_tuple()) Ht.append(net.rnn_state[0, 0, 0]) if debug: print(f"Step: {step_num}") print(f"\t> Actuation: {(round(actuation[-1][0], 5), round(actuation[-1][1], 5))!r}") print(f"\t> Distance: {round(distance[-1], 5)} - Delta distance: {round(delta_distance[-1], 5)}") print(f"\t> Position: {(round(position[-1][0], 2), round(position[-1][1], 2))!r}") print(f"\t> SRU state: Ht={round(Ht[-1], 5)}") # Start monitoring while True: # Check if maximum iterations is reached if step_num == game_cfg.game.duration * game_cfg.game.fps: break # Determine the actions made by the agent for each of the states action = net(np.asarray([state])) # Check if each game received an action assert len(action) == 1 # Proceed the game with one step, based on the predicted action obs = game.step(l=action[0][0], r=action[0][1]) finished = obs[D_DONE] # Update the score-count if game.score > score: target_found.append(step_num) score = game.score # Update the candidate's current state state = obs[D_SENSOR_LIST] # Stop if agent reached target in all the games if finished: break step_num += 1 # Update the containers actuation.append(action[0]) distance.append(state[0]) delta_distance.append(distance[-2] - distance[-1]) position.append(game.player.pos.get_tuple()) Ht.append(net.rnn_state[0, 0, 0]) if debug: print(f"Step: {step_num}") print(f"\t> Actuation: {(round(actuation[-1][0], 5), round(actuation[-1][1], 5))!r}") print(f"\t> Distance: {round(distance[-1], 5)} - Delta distance: {round(delta_distance[-1], 5)}") print(f"\t> Position: {(round(position[-1][0], 2), round(position[-1][1], 2))!r}") print(f"\t> SRU state: Ht={round(Ht[-1], 5)}") if average > 1: # Average out the noise x, y = zip(*actuation) x = SMA(x, window=average) y = SMA(y, window=average) actuation = list(zip(x, y)) distance = SMA(distance, window=average) delta_distance = SMA(delta_distance, window=average) Ht = SMA(Ht, window=average) # Resolve weird artifacts at the beginning for i in range(average, 0, -1): actuation[i - 1] = actuation[i] distance[i - 1] = distance[i] delta_distance[i - 1] = delta_distance[i] Ht[i - 1] = Ht[i] # Visualize the monitored values path = get_subfolder(f"population{'_backup' if population.use_backup else ''}/" f"storage/" f"{population.folder_name}/" f"{population}/", "images") path = get_subfolder(path, f"monitor") path = get_subfolder(path, f"{genome.key}") path = get_subfolder(path, f"{game_id}") visualize_actuation(actuation, target_found=target_found, game_cfg=game_cfg.game, save_path=f"{path}actuation.png") visualize_distance(distance, target_found=target_found, game_cfg=game_cfg.game, save_path=f"{path}distance.png") visualize_hidden_state(Ht, target_found=target_found, game_cfg=game_cfg.game, save_path=f"{path}hidden_state.png") visualize_position(position, game=game, save_path=f"{path}trace.png") merge(f"Monitored genome={genome.key} on game={game.id}", path=path)
def reproduce(self, config: Config, species: DefaultSpecies, generation: int, logger=None): """Handles creation of genomes, either from scratch or by sexual or asexual reproduction from parents.""" # Check is one of the species has become stagnant (i.e. must be removed) remaining_fitness = [] remaining_species = [] for stag_sid, stag_s, stagnant in self.stagnation.update( config=config, species_set=species, gen=generation): # If specie is stagnant, then remove if stagnant: self.reporters.species_stagnant(stag_sid, stag_s, logger=logger) # Add the specie to remaining_species and save each of its members' fitness else: remaining_fitness.extend(m.fitness for m in itervalues(stag_s.members)) remaining_species.append(stag_s) # If no species is left then force hard-reset if not remaining_species: species.species = dict() return dict() # Calculate the adjusted fitness, normalized by the minimum fitness across the entire population for specie in remaining_species: # Adjust a specie's fitness in a fitness sharing manner. A specie's fitness gets normalized by the number of # members it has, this to ensure that a better performing specie does not takes over the population # A specie's fitness is determined by its most fit genome specie_fitness = max([m.fitness for m in specie.members.values()]) specie_size = len(specie.members) specie.adjusted_fitness = specie_fitness / max( specie_size, config.population.min_specie_size) # Minimum specie-size is defined by the number of elites and the minimal number of genomes in a population spawn_amounts = self.compute_spawn( adjusted_fitness=[s.adjusted_fitness for s in remaining_species], previous_sizes=[len(s.members) for s in remaining_species], pop_size=config.population.pop_size, min_species_size=max(config.population.min_specie_size, config.population.genome_elitism)) # Setup the next generation by filling in the new species with their elites and offspring new_population = dict() species.species = dict() for spawn_amount, specie in zip(spawn_amounts, remaining_species): # If elitism is enabled, each species will always at least gets to retain its elites spawn_amount = max(spawn_amount, config.population.genome_elitism) assert spawn_amount > 0 # Get all the specie's old (evaluated) members old_members = list(iteritems( specie.members)) # Temporarily save members of last generation specie.members = dict() # Reset members species.species[specie.key] = specie # Sort members in order of descending fitness (i.e. most fit members in front) old_members.sort(reverse=True, key=lambda x: x[1].fitness) # Make sure that all the specie's elites are added to the new generation if config.population.genome_elitism > 0: # Add the specie's elites to the global population for i, m in old_members[:config.population.genome_elitism]: new_population[i] = m spawn_amount -= 1 # Add the specie's past elites as well if requested for i in range( min(len(specie.elite_list), config.population.genome_elite_stagnation - 1)): gid, g = specie.elite_list[-(i + 1)] if gid not in new_population: # Only add genomes not yet present in the population new_population[gid] = g spawn_amount -= 1 # Update the specie's elite_list specie.elite_list.append(old_members[0]) # Check if the specie has the right to add more genomes to the population if spawn_amount <= 0: continue # Only use the survival threshold fraction to use as parents for the next generation, use at least all the # elite of a population as parents reproduction_cutoff = max( round(config.population.parent_selection * len(old_members)), config.population.genome_elitism) # Since asexual reproduction, at least one parent must be chosen reproduction_cutoff = max(reproduction_cutoff, 1) parents = old_members[:reproduction_cutoff] # Add the elites again to the parent-set such that these have a greater likelihood of being chosen parents += old_members[:config.population.genome_elitism] # Fill the specie with offspring based, which is a mutation of the chosen parent while spawn_amount > 0: spawn_amount -= 1 # Init genome dummy (values are overwritten later) gid = next(self.genome_indexer) child: Genome = Genome(gid, num_outputs=config.genome.num_outputs, bot_config=config.bot) # Choose the parents, note that if the parents are not distinct, crossover will produce a genetically # identical clone of the parent (but with a different ID) p1_id, p1 = choice(parents) child.connections = copy.deepcopy(p1.connections) child.nodes = copy.deepcopy(p1.nodes) # Mutate the child child.mutate(config.genome) # Ensure that the child is connected while len(child.get_used_connections()) == 0: child.mutate_add_connection(config.genome) # Add the child to the global population new_population[gid] = child return new_population
def monitor_activation(genome: Genome, gid: int, debug: bool = False, duration: int = 60): """ Monitor the activation of the candidate hidden state. Note: game is started again, no worries since deterministic. """ cfg = Config() cfg.game.duration = duration cfg.update() # Check if valid genome (contains at least one hidden GRU, first GRU is monitored) assert len([ n for n in genome.get_used_nodes().values() if type(n) == GruNodeGene ]) >= 1 # Get the game game = get_game(i=gid, cfg=cfg, noise=False) state = game.reset()[D_SENSOR_LIST] step_num = 0 # Create the network net = make_net( genome=genome, genome_config=cfg.genome, batch_size=1, initial_read=state, ) # Containers to monitor Ht = [] Ht_tilde = [] target_found = [] score = 0 # Initialize the containers ht, ht_tilde, _, _ = get_gru_states(gru=net.rnn_array[0], x=np.asarray([state])) Ht.append(ht) Ht_tilde.append(ht_tilde) if debug: print(f"Step: {step_num}") print(f"\t> Hidden state: {round(Ht[-1], 5)}") print(f"\t> Candidate hidden state: {round(Ht_tilde[-1], 5)}") # Start monitoring while True: # Check if maximum iterations is reached if step_num == duration * cfg.game.fps: break # Determine the actions made by the agent for each of the states action = net(np.asarray([state])) # Check if each game received an action assert len(action) == 1 # Proceed the game with one step, based on the predicted action obs = game.step(l=action[0][0], r=action[0][1]) finished = obs[D_DONE] # Update the score-count if game.score > score: target_found.append(step_num) score = game.score # Update the candidate's current state state = obs[D_SENSOR_LIST] # Stop if agent reached target in all the games if finished: break step_num += 1 # Update the containers ht, ht_tilde, _, _ = get_gru_states(gru=net.rnn_array[0], x=np.asarray([state])) Ht.append(ht) Ht_tilde.append(ht_tilde) if debug: print(f"Step: {step_num}") print(f"\t> Hidden state: {round(Ht[-1], 5)}") print(f"\t> Candidate hidden state: {round(Ht_tilde[-1], 5)}") return Ht_tilde, Ht, target_found
def visualize_score(pop: Population, pop_cache: Population, genome: Genome, d: int, range_width: int): """Visualize the score of the evaluated population.""" pop_cache.log(f"{pop_cache.name} - Fetching fitness scores...") dim = (2 * range_width + 1) pbar = tqdm(range((dim ** 2) * 3), desc="Fetching fitness scores") genome_key = 0 # Fetching scores of reset-mutations reset_scores = np.zeros((dim, dim)) for a in range(dim): for b in range(dim): reset_scores[a, b] = pop_cache.population[genome_key].fitness genome_key += 1 pbar.update() # Fetching scores of update-mutations update_scores = np.zeros((dim, dim)) for a in range(dim): for b in range(dim): update_scores[a, b] = pop_cache.population[genome_key].fitness genome_key += 1 pbar.update() # Fetching scores of candidate-mutations candidate_scores = np.zeros((dim, dim)) for a in range(dim): for b in range(dim): candidate_scores[a, b] = pop_cache.population[genome_key].fitness genome_key += 1 pbar.update() pbar.close() # Visualize the result pop_cache.log(f"{pop_cache.name} - Visualizing the result...") # GRU-node needed for labels genome.update_rnn_nodes(config=pop.config.genome) gru_node = None for node in genome.nodes.values(): if type(node) == GruNodeGene: gru_node = node # Create the points and retrieve data for the plot points = [[x, y] for x in range(dim) for y in range(dim)] points_normalized = [[(p1 + -range_width) / d, (p2 + -range_width) / d] for p1, p2 in points] values_reset = [reset_scores[p[0], p[1]] for p in points] values_update = [update_scores[p[0], p[1]] for p in points] values_candidate = [candidate_scores[p[0], p[1]] for p in points] # K-nearest neighbours with hops of 0.01 is performed grid_x, grid_y = np.mgrid[-range_width / d:range_width / d:0.01, -range_width / d:range_width / d:0.01] # Perform k-NN data_reset = griddata(points_normalized, values_reset, (grid_x, grid_y), method='nearest') data_update = griddata(points_normalized, values_update, (grid_x, grid_y), method='nearest') data_candidate = griddata(points_normalized, values_candidate, (grid_x, grid_y), method='nearest') # Create the plots plt.figure(figsize=(15, 5)) plt.subplot(131) plt.imshow(data_reset.T, vmin=0, vmax=1, extent=(-range_width / d, range_width / d, -range_width / d, range_width / d), origin='lower') plt.title('Reset-gate mutation') plt.xlabel(r'$\Delta W_{hr}$' + f' (init={round(gru_node.weight_hh[0, 0], 3)})') plt.ylabel(r'$\Delta W_{xr}$' + f' (init={round(gru_node.weight_xh[0, 0], 3)})') plt.subplot(132) plt.imshow(data_update.T, vmin=0, vmax=1, extent=(-range_width / d, range_width / d, -range_width / d, range_width / d), origin='lower') plt.title('Update-gate mutation') plt.xlabel(r'$\Delta W_{hz}$' + f' (init={round(gru_node.weight_hh[1, 0], 3)})') plt.ylabel(r'$\Delta W_{xz}$' + f' (init={round(gru_node.weight_xh[1, 0], 3)})') plt.subplot(133) plt.imshow(data_candidate.T, vmin=0, vmax=1, extent=(-range_width / d, range_width / d, -range_width / d, range_width / d), origin='lower') plt.title('Candidate-state mutation') plt.xlabel(r'$\Delta W_{hh}$' + f' (init={round(gru_node.weight_hh[2, 0], 3)})') plt.ylabel(r'$\Delta W_{xh}$' + f' (init={round(gru_node.weight_xh[2, 0], 3)})') # Store the plot plt.tight_layout() path = f"population{'_backup' if pop.use_backup else ''}/storage/{pop.folder_name}/{pop}/" path = get_subfolder(path, 'images') path = get_subfolder(path, 'gru_analysis') plt.savefig(f"{path}{genome.key}.png") plt.savefig(f"{path}{genome.key}.eps", format="eps") plt.close() # Create overview pop_cache.log("Overview of results:") log = dict() # Overview: reset-mutation max_index_reset, max_value_reset, min_index_reset, min_value_reset = None, 0, None, 1 for index, x in np.ndenumerate(reset_scores): if x < min_value_reset: min_index_reset, min_value_reset = index, x if x > max_value_reset: max_index_reset, max_value_reset = index, x pop_cache.log(f"\tReset-gate mutation:") pop_cache.log(f"\t > Maximum fitness: {round(max_value_reset, 2)} for index {max_index_reset!r}") pop_cache.log(f"\t > Average fitness: {round(np.average(reset_scores), 2)}") pop_cache.log(f"\t > Minimum fitness: {round(min_value_reset, 2)} for index {min_index_reset!r}") log['Reset-gate maximum fitness'] = f"{round(max_value_reset, 2)} for index {max_index_reset!r}" log['Reset-gate average fitness'] = f"{round(np.average(reset_scores), 2)}" log['Reset-gate minimum fitness'] = f"{round(min_value_reset, 2)} for index {min_index_reset!r}" # Overview: update-mutation max_index_update, max_value_update, min_index_update, min_value_update = None, 0, None, 1 for index, x in np.ndenumerate(update_scores): if x < min_value_update: min_index_update, min_value_update = index, x if x > max_value_update: max_index_update, max_value_update = index, x pop_cache.log(f"\tUpdate-gate mutation:") pop_cache.log(f"\t > Maximum fitness: {round(max_value_update, 2)} for index {max_index_update!r}") pop_cache.log(f"\t > Average fitness: {round(np.average(update_scores), 2)}") pop_cache.log(f"\t > Minimum fitness: {round(min_value_update, 2)} for index {min_index_update!r}") log['Update-gate maximum fitness'] = f"{round(max_value_update, 2)} for index {max_index_update!r}" log['Update-gate average fitness'] = f"{round(np.average(update_scores), 2)}" log['Update-gate minimum fitness'] = f"{round(min_value_update, 2)} for index {min_index_update!r}" # Overview: candidate-mutation max_index_candidate, max_value_candidate, min_index_candidate, min_value_candidate = None, 0, None, 1 for index, x in np.ndenumerate(candidate_scores): if x < min_value_candidate: min_index_candidate, min_value_candidate = index, x if x > max_value_candidate: max_index_candidate, max_value_candidate = index, x pop_cache.log(f"\tCandidate-state mutation:") pop_cache.log(f"\t > Maximum fitness: {round(max_value_candidate, 2)} for index {max_index_candidate!r}") pop_cache.log(f"\t > Average fitness: {round(np.average(candidate_scores), 2)}") pop_cache.log(f"\t > Minimum fitness: {round(min_value_candidate, 2)} for index {min_index_candidate!r}") log['Candidate-state maximum fitness'] = f"{round(max_value_candidate, 2)} for index {max_index_candidate!r}" log['Candidate-state average fitness'] = f"{round(np.average(candidate_scores), 2)}" log['Candidate-state minimum fitness'] = f"{round(min_value_candidate, 2)} for index {min_index_candidate!r}" update_dict(f'{path}{genome.key}.txt', log)
def get_topology222(gid: int, cfg: Config): """ Create a uniformly and randomly sampled genome of fixed topology: Sigmoid with bias 1.5 --> Actuation default of 95,3% (key=0, bias=1.5) (key=1, bias=2.11) ____ / / / / FIXED / | _____/ | / (key=-1) """ # Create an initial dummy genome with fixed configuration genome = Genome( key=gid, num_outputs=cfg.genome.num_outputs, bot_config=cfg.bot, ) # Setup the parameter-ranges conn_range = cfg.genome.weight_max_value - cfg.genome.weight_min_value bias_range = cfg.genome.bias_max_value - cfg.genome.bias_min_value rnn_range = cfg.genome.rnn_max_value - cfg.genome.rnn_min_value # Create the nodes genome.nodes[0] = OutputNodeGene(key=0, cfg=cfg.genome) # OutputNode 0 genome.nodes[0].bias = 1.5 # Drive with 0.953 actuation by default genome.nodes[1] = OutputNodeGene(key=1, cfg=cfg.genome) # OutputNode 1 genome.nodes[1].bias = 2.11 # Found by GRU network it tries to mimic genome.nodes[2] = FixedRnnNodeGene(key=2, cfg=cfg.genome, input_keys=[-1]) # Hidden node # Create the connections genome.connections = dict() # input2gru key = (-1, 2) genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[key].weight = 1 # Simply forward distance genome.connections[key].enabled = True # gru2output - Uniformly sampled key = (2, 1) genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[key].weight = 3 # Enforce capabilities of full spectrum genome.connections[key].enabled = True # input2output - Uniformly sampled key = (-1, 1) genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[ key].weight = -2.82 # Found by GRU network it tries to mimic genome.connections[key].enabled = True genome.update_rnn_nodes(config=cfg.genome) return genome
def get_topology3333(gid: int, cfg: Config): """ Create a uniformly and randomly sampled genome of fixed topology: Sigmoid with bias 1.5 --> Actuation default of 95,3% (key=0, bias=1.5) (key=1, bias=?) ____ / / / / GRU-NR / | _____/ | / (key=-1) """ # Create an initial dummy genome with fixed configuration genome = Genome( key=gid, num_outputs=cfg.genome.num_outputs, bot_config=cfg.bot, ) # Setup the parameter-ranges conn_range = cfg.genome.weight_max_value - cfg.genome.weight_min_value bias_range = cfg.genome.bias_max_value - cfg.genome.bias_min_value rnn_range = cfg.genome.rnn_max_value - cfg.genome.rnn_min_value # Create the nodes genome.nodes[0] = OutputNodeGene(key=0, cfg=cfg.genome) # OutputNode 0 genome.nodes[0].bias = 1.5 # Drive with 0.953 actuation by default genome.nodes[1] = OutputNodeGene(key=1, cfg=cfg.genome) # OutputNode 1 genome.nodes[1].bias = random( ) * bias_range + cfg.genome.bias_min_value # Uniformly sampled bias genome.nodes[2] = GruNoResetNodeGene(key=2, cfg=cfg.genome, input_keys=[-1], input_keys_full=[-1]) # Hidden node genome.nodes[2].bias = 0 # Bias is irrelevant for GRU-node # Uniformly sample the genome's GRU-component genome.nodes[2].bias_h = rand_arr( (2, )) * bias_range + cfg.genome.bias_min_value genome.nodes[2].weight_xh_full = rand_arr( (2, 1)) * rnn_range + cfg.genome.weight_min_value genome.nodes[2].weight_hh = rand_arr( (2, 1)) * rnn_range + cfg.genome.weight_min_value # Create the connections genome.connections = dict() # input2gru key = (-1, 2) genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[ key].weight = random() * conn_range + cfg.genome.weight_min_value genome.connections[key].enabled = True # gru2output - Uniformly sampled key = (2, 1) genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[ key].weight = random() * conn_range + cfg.genome.weight_min_value genome.connections[key].enabled = True # input2output - Uniformly sampled key = (-1, 1) genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[ key].weight = random() * conn_range + cfg.genome.weight_min_value genome.connections[key].enabled = True genome.update_rnn_nodes(config=cfg.genome) return genome
def get_topology(pop_name, gid: int, cfg: Config): """ Create a uniformly and randomly sampled genome of fixed topology: Sigmoid with bias 1.5 --> Actuation default of 95,3% (key=0, bias=1.5) (key=1, bias=?) ____ / / / / GRU / | _____/ | / (key=-1) """ # Create an initial dummy genome with fixed configuration genome = Genome( key=gid, num_outputs=cfg.genome.num_outputs, bot_config=cfg.bot, ) # Setup the parameter-ranges conn_range = cfg.genome.weight_max_value - cfg.genome.weight_min_value bias_range = cfg.genome.bias_max_value - cfg.genome.bias_min_value rnn_range = cfg.genome.rnn_max_value - cfg.genome.rnn_min_value # Create the output nodes genome.nodes[0] = OutputNodeGene(key=0, cfg=cfg.genome) # OutputNode 0 genome.nodes[0].bias = 1.5 # Drive with 0.953 actuation by default genome.nodes[1] = OutputNodeGene(key=1, cfg=cfg.genome) # OutputNode 1 if pop_name in [P_BIASED]: genome.nodes[1].bias = normal( 1.5, .1) # Initially normally distributed around bias of other output else: genome.nodes[1].bias = random( ) * bias_range + cfg.genome.bias_min_value # Uniformly sampled bias # Setup the recurrent unit if pop_name in [P_GRU_NR]: genome.nodes[2] = GruNoResetNodeGene(key=2, cfg=cfg.genome, input_keys=[-1], input_keys_full=[-1]) # Hidden genome.nodes[2].bias_h = rand_arr( (2, )) * bias_range + cfg.genome.bias_min_value genome.nodes[2].weight_xh_full = rand_arr( (2, 1)) * rnn_range + cfg.genome.weight_min_value genome.nodes[2].weight_hh = rand_arr( (2, 1)) * rnn_range + cfg.genome.weight_min_value else: genome.nodes[2] = GruNodeGene(key=2, cfg=cfg.genome, input_keys=[-1], input_keys_full=[-1]) # Hidden node genome.nodes[2].bias_h = rand_arr( (3, )) * bias_range + cfg.genome.bias_min_value genome.nodes[2].weight_xh_full = rand_arr( (3, 1)) * rnn_range + cfg.genome.weight_min_value genome.nodes[2].weight_hh = rand_arr( (3, 1)) * rnn_range + cfg.genome.weight_min_value genome.nodes[2].bias = 0 # Bias is irrelevant for GRU-node # Create the connections genome.connections = dict() # input2gru - Uniformly sampled on the positive spectrum key = (-1, 2) genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) if pop_name in [P_BIASED]: genome.connections[ key].weight = 6 # Maximize connection, GRU can always lower values flowing through else: genome.connections[ key].weight = random() * conn_range + cfg.genome.weight_min_value genome.connections[key].enabled = True # gru2output - Uniformly sampled on the positive spectrum key = (2, 1) genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[ key].weight = random() * conn_range + cfg.genome.weight_min_value if pop_name in [P_BIASED]: genome.connections[key].weight = abs( genome.connections[key].weight) # Always positive! genome.connections[key].enabled = True # input2output - Uniformly sampled key = (-1, 1) genome.connections[key] = ConnectionGene(key=key, cfg=cfg.genome) genome.connections[ key].weight = random() * conn_range + cfg.genome.weight_min_value if pop_name in [P_BIASED]: genome.connections[key].weight = -abs( genome.connections[key].weight) # Always negative! genome.connections[key].enabled = True # Enforce the topology constraints enforce_topology(pop_name=pop_name, genome=genome) genome.update_rnn_nodes(config=cfg.genome) return genome
def main(population: Population, game_id: int, genome: Genome = None, game_cfg: Config = None, debug: bool = False): """ Monitor the genome on the following elements: * Position * Update gate (Zt) * Hidden state of GRU (Ht) * Actuation of both wheels * Distance """ # Make sure all parameters are set if not genome: genome = population.best_genome if not game_cfg: game_cfg = pop.config # Check if valid genome (contains at least one hidden GRU, first GRU is monitored) assert len([ n for n in genome.get_used_nodes().values() if type(n) == GruNoResetNodeGene ]) >= 1 # Get the game game = get_game(game_id, cfg=game_cfg, noise=False) state = game.reset()[D_SENSOR_LIST] step_num = 0 # Create the network net = make_net( genome=genome, genome_config=population.config.genome, batch_size=1, initial_read=state, ) # Containers to monitor actuation = [] distance = [] position = [] Ht = [] Ht_tilde = [] Zt = [] target_found = [] score = 0 # Initialize the containers actuation.append([0, 0]) distance.append(state[0]) position.append(game.player.pos.get_tuple()) ht, ht_tilde, zt = get_gru_states(net=net, x=np.asarray([state])) Ht.append(ht) Ht_tilde.append(ht_tilde) Zt.append(zt) if debug: print(f"Step: {step_num}") print( f"\t> Actuation: {(round(actuation[-1][0], 5), round(actuation[-1][1], 5))!r}" ) print(f"\t> Distance: {round(distance[-1], 5)}") print( f"\t> Position: {(round(position[-1][0], 2), round(position[-1][1], 2))!r}" ) print(f"\t> GRU states: " f"\t\tHt={round(Ht[-1], 5)}" f"\t\tHt_tilde={round(Ht_tilde[-1], 5)}" f"\t\tZt={round(Zt[-1], 5)}") # Start monitoring while True: # Check if maximum iterations is reached if step_num == game_cfg.game.duration * game_cfg.game.fps: break # Determine the actions made by the agent for each of the states action = net(np.asarray([state])) # Check if each game received an action assert len(action) == 1 # Proceed the game with one step, based on the predicted action obs = game.step(l=action[0][0], r=action[0][1]) finished = obs[D_DONE] # Update the score-count if game.score > score: target_found.append(step_num) score = game.score # Update the candidate's current state state = obs[D_SENSOR_LIST] # Stop if agent reached target in all the games if finished: break step_num += 1 # Update the containers actuation.append(action[0]) distance.append(state[0]) position.append(game.player.pos.get_tuple()) ht, ht_tilde, zt = get_gru_states(net=net, x=np.asarray([state])) Ht.append(ht) Ht_tilde.append(ht_tilde) Zt.append(zt) if debug: print(f"Step: {step_num}") print( f"\t> Actuation: {(round(actuation[-1][0], 5), round(actuation[-1][1], 5))!r}" ) print(f"\t> Distance: {round(distance[-1], 5)}") print( f"\t> Position: {(round(position[-1][0], 2), round(position[-1][1], 2))!r}" ) print(f"\t> GRU states: " f"\t\tHt={round(Ht[-1], 5)}" f"\t\tHt_tilde={round(Ht_tilde[-1], 5)}" f"\t\tZt={round(Zt[-1], 5)}") # Visualize the monitored values path = get_subfolder( f"population{'_backup' if population.use_backup else ''}/" f"storage/" f"{population.folder_name}/" f"{population}/", "images") path = get_subfolder(path, f"monitor") path = get_subfolder(path, f"{genome.key}") path = get_subfolder(path, f"{game_id}") visualize_actuation(actuation, target_found=target_found, game_cfg=game_cfg.game, save_path=f"{path}actuation.png") visualize_distance(distance, target_found=target_found, game_cfg=game_cfg.game, save_path=f"{path}distance.png") visualize_hidden_state(Ht, target_found=target_found, game_cfg=game_cfg.game, save_path=f"{path}hidden_state.png") visualize_candidate_hidden_state( Ht_tilde, target_found=target_found, game_cfg=game_cfg.game, save_path=f"{path}candidate_hidden_state.png") visualize_update_gate(Zt, target_found=target_found, game_cfg=game_cfg.game, save_path=f"{path}update_gate.png") visualize_position(position, game=game, save_path=f"{path}trace.png") merge(f"Monitored genome={genome.key} on game={game.id}", path=path)