def elite_architecture(pop: Population, show: bool = False, del_cache: bool = True): """ Visualize each architectural change in the population's elites. :param pop: Population object :param show: Show the result :param del_cache: Remove intermediate architecture-results when images are created """ # Initialize the architecture-list with the first genome distance = GenomeDistanceCache(config=pop.config.genome) new_architectures = [(0, pop.best_genome_hist[0][1]) ] # Only interested in genome itself for gen in range(1, pop.generation): gen_genome = pop.best_genome_hist[gen][1] if distance( gen_genome, new_architectures[-1] [1]) > 2: # Take only the more significant changes into account new_architectures.append((gen, gen_genome)) new_architectures.append( (pop.generation - 1, pop.best_genome_hist[pop.generation - 1][1])) # Create the architectures of the unique genomes for _, g in new_architectures: pop.visualize_genome( debug=False, # Keep the networks simple genome=g, show=False, ) # Combine in one figure hor = min(len(new_architectures), 5) vert = max((len(new_architectures) - 1) // 5 + 1, 1) plt.figure(figsize=(5 * hor, 5 * vert)) plt.tight_layout() f_images = get_subfolder( f"population{'_backup' if pop.use_backup else ''}/storage/{pop.folder_name}/{pop}/", "images") f_architectures = get_subfolder(f_images, "architectures") for i, (gen, g) in enumerate(new_architectures): plt.subplot(vert, hor, i + 1) img = mpimg.imread(f'{f_architectures}genome_{g.key}.png') plt.imshow(img) plt.title(f'Generation {gen}') plt.axis('off') # Save the result f_elites = get_subfolder(f_images, "elites") plt.savefig(f'{f_elites}/architecture_timeline.png', bbox_inches='tight') if show: plt.show() plt.close() if del_cache: for (_, g) in new_architectures: path = f'{f_architectures}genome_{g.key}.png' if os.path.exists(path): os.remove(path)
def evaluate_training(experiment_id: int, pop_folder: str, folder: str = None, max_v: int = 50): """Evaluate the fitness of a population's elite each training generation.""" if pop_folder[-1] != '/': pop_folder += '/' folder = folder if folder else get_folder(experiment_id) if folder[-1] != '/': folder += '/' # Get dummy population pop = Population( name=f"{pop_folder}v1", folder_name=folder, log_print=False, use_backup=True, ) max_gen = pop.generation # Initialize data container training_fitness = dict() for g in range(0, max_gen + 1, HOPS): training_fitness[g] = [] # Pull the training scores print( f"\n===> PULLING TRAINING FITNESS OF THE {pop_folder} POPULATIONS <===" ) pbar = tqdm(range(int(max_v * (max_gen / HOPS + 1)))) for v in range(1, max_v + 1): name = f"{pop_folder}v{v}" pop = Population( name=name, folder_name=folder, log_print=False, use_backup=True, ) # Perform the evaluations max_gen = pop.generation for gen in range(0, max_gen + 1, HOPS): if not pop.load(gen=gen): raise Exception( f"Population {name} is not trained for generation {gen}") training_fitness[gen].append( pop.best_genome.fitness if pop.best_genome else 0) pbar.update() pbar.close() # Plot the result path = get_subfolder(f'population_backup/storage/{folder}{pop_folder}', 'evaluation') update_dict(f'{path}training', training_fitness, overwrite=True) path_images = get_subfolder(path, 'images') plot_result(d=training_fitness, ylabel="fitness", title="Average training fitness", save_path=f'{path_images}training')
def add_evaluation_result(self, eval_result, overwrite: bool = False): """Append the result of the evaluation.""" sf = get_subfolder( f"population{'_backup' if self.use_backup else ''}/" f"storage/" f"{self.folder_name}/" f'{self}/', 'evaluation') sf = get_subfolder(sf, f"{self.generation:05d}") update_dict(f'{sf}results', eval_result, overwrite=overwrite)
def trace_genomes(self, pop: Population, given_genome: Genome = None, parallel: bool = True): """ Create blueprints that contain the walking-traces for all the requested mazes. :param pop: Population object :param given_genome: Single genomes for which the trace must be made :param parallel: Create the traces in parallel """ multi_env = get_multi_env(pop=pop, game_config=self.game_config) if len(self.games) > 20 and given_genome is None: raise Exception( "It is not advised to evaluate on more than 20 at once") elif len(self.games) > 100: raise Exception( "It is not advised to evaluate on more than 100 at once") # Set the games for which traces will be made multi_env.set_games(self.games, noise=False) # Fetch the dictionary of genomes genomes = [(given_genome.key, given_genome)] if given_genome else list( iteritems(pop.population)) if parallel: # Initialize the evaluation-pool pool = mp.Pool(mp.cpu_count() - self.unused_cpu) manager = mp.Manager() return_dict = manager.dict() # Evaluate the genomes for genome in genomes: pool.apply_async(func=multi_env.trace_genome, args=(genome, return_dict)) pool.close() # Close the pool pool.join() # Postpone continuation until everything is finished else: # Train sequentially return_dict = dict() for genome in tqdm(genomes, desc="sequential evaluating"): multi_env.trace_genome(genome, return_dict) # Create blueprint of final result game_objects = [get_game(g, cfg=self.game_config) for g in self.games] path = get_subfolder( f"population{'_backup' if pop.use_backup else ''}/storage/{pop.folder_name}/{pop}/", 'images') path = get_subfolder(path, 'games') create_traces( traces=return_dict, games=game_objects, gen=pop.generation, save_path=path, save_name=f'trace_{given_genome.key}' if given_genome else 'trace', )
def get_save_path(gid: int, save_name): """Get the correct path to save a certain file in.""" save_path = get_subfolder(f"genomes_gru/", f"images") save_path = get_subfolder(save_path, f"genome{gid}") if len(save_name.split('/')) == 1: path = f"{save_path}{save_name}" elif len(save_name.split('/')) == 2: path = get_subfolder(f"{save_path}", save_name.split('/')[0]) path = f"{path}{save_name.split('/')[1]}" else: raise Exception(f"Too long save_name: '{save_name}'") return path
def visualize_best_genome(pop: Population): """Visualize the population's fittest genome.""" pop.log( f"Most fit genome: {pop.best_genome.key} with fitness: {round(pop.best_genome.fitness, 3)}" ) name = f"genome_{pop.best_genome.key}" sf = get_subfolder(f'population/storage/{pop.folder_name}/{pop}/', 'images') sf = get_subfolder(sf, f'gen{pop.generation:05d}') draw_net(config=pop.config.genome, genome=pop.best_genome, debug=True, filename=f'{sf}{name}', view=False)
def get_initial_keys(topology_id: int, use_backup: bool): """Get the genome-key based on CSV-file's length.""" path = get_subfolder( f"population{'_backup' if use_backup else ''}/storage/", "experiment6") path = get_subfolder(path, "data") csv_name = f"topology_{topology_id}" path = f"{path}{csv_name}.csv" # CSV exists, count number of rows if os.path.exists(path): with open(path, 'r') as f: return sum(1 for _ in f), path # CSV does not exist, create new else: with open(path, 'w', newline='') as f: writer = csv.writer(f) # Construct the CSV's head, all genomes have the full GRU-parameter suite head = [] if topology_id in [1, 2, 3, 30]: # GRU populations head += [ 'bias_r', 'bias_z', 'bias_h', 'weight_xr', 'weight_xz', 'weight_xh', 'weight_hr', 'weight_hz', 'weight_hh' ] elif topology_id in [22, 33]: # SRU populations head += ['bias_h', 'weight_xh', 'weight_hh'] elif topology_id in [222]: head += ['delay', 'scale', 'bias_h'] elif topology_id in [2222, 3333]: head += [ 'bias_z', 'bias_h', 'weight_xz', 'weight_xh', 'weight_hz', 'weight_hh' ] elif topology_id in [22222]: head += ['bias_h', 'weight'] else: raise Exception(f"Topology ID '{topology_id}' not supported!") if topology_id in [1]: head += ['conn1', 'conn2'] elif topology_id in [2, 22, 222, 2222, 22222]: head += ['bias_rw', 'conn2'] elif topology_id in [3, 30, 33, 3333]: head += ['bias_rw', 'conn0', 'conn1', 'conn2'] else: raise Exception(f"Topology ID '{topology_id}' not supported!") head += ['fitness'] writer.writerow(head) return 1, path
def elite_fitness(pop: Population, window: int = 5, show: bool = True): """ Visualize the elites of the given population. Each generation, the average fitness of the three stored elites is taken. :param pop: Population object :param window: Window-size used in the function :param show: Show the result """ f = get_subfolder( f"population{'_backup' if pop.use_backup else ''}/storage/{pop.folder_name}/{pop}/", 'images') f = get_subfolder(f, 'elites') for func in [Forward, SMA, EMA]: # Fetch name based on used function name = f'{"EMA_" if func == EMA else "SMA_" if func == SMA else ""}gen_{pop.generation}' # Load in the relevant data history = sorted(pop.best_fitness.items(), key=lambda x: x[0]) generations, fitness = zip(*history) # Create the figure ax = plt.figure().gca() plt.plot(generations, func(fitness, window)) if func == SMA: plt.title( f"Elite fitness in population: {pop}\nSimple Moving Average (window={window})" ) elif func == EMA: plt.title( f"Elite fitness in population: {pop}\nExponential Moving Average (window={window})" ) else: plt.title(f"Elite fitness in population: {pop}") plt.xlabel("generation") plt.ylabel("fitness") ax.xaxis.set_major_locator( MaxNLocator(integer=True)) # Forces to use only integers if max(fitness) <= 1: plt.yticks([i / 10 for i in range(11) ]) # Fitness expressed in range of 0..1 (hops of 0.1) plt.grid(axis='y') plt.tight_layout() # Save the result plt.savefig(f'{f}{name}') if show: plt.show() plt.close()
def get_csv_path(topology_id: int, use_backup: bool, batch_size: int): """Get the genome-key based on CSV-file's length.""" path = get_subfolder(f"population{'_backup' if use_backup else ''}/storage/", "experiment6") path = get_subfolder(path, "data_neat") csv_name = f"topology_{topology_id}" path = f"{path}{csv_name}.csv" # If CSV exists, check if not yet full if os.path.exists(path): with open(path, 'r') as f: rows = sum(1 for _ in f) - 1 # Do not count header if rows < batch_size: return path, csv_name, rows # CSV does not yet exist, or is already full, create new CSV else: with open(path, 'w', newline='') as f: writer = csv.writer(f) # Construct the CSV's head head = [] if topology_id in [1, 2, 3, 30]: # GRU populations head += ['bias_r', 'bias_z', 'bias_h', 'weight_xr', 'weight_xz', 'weight_xh', 'weight_hr', 'weight_hz', 'weight_hh'] elif topology_id in [22, 33]: # SRU populations head += ['bias_h', 'weight_xh', 'weight_hh'] elif topology_id in [222]: head += ['delay', 'scale', 'resting'] elif topology_id in [2222, 3333]: head += ['bias_z', 'bias_h', 'weight_xz', 'weight_xh', 'weight_hz', 'weight_hh'] elif topology_id in [22222]: head += ['bias_h', 'weight'] else: raise Exception(f"Topology ID '{topology_id}' not supported!") if topology_id in [1]: head += ['conn1', 'conn2'] elif topology_id in [2, 22, 222, 2222, 22222]: head += ['bias_rw', 'conn2'] elif topology_id in [3, 30, 33, 3333]: head += ['bias_rw', 'conn0', 'conn1', 'conn2'] else: raise Exception(f"Topology ID '{topology_id}' not supported!") head += ['fitness'] writer.writerow(head) return path, csv_name, 0
def blueprint_genomes(self, pop: Population, parallel: bool = True): """ Create blueprints for all the requested mazes. :param pop: Population object :param parallel: Evaluate the population in parallel """ multi_env = get_multi_env(pop=pop, game_config=self.game_config) if len(self.games) > 100: raise Exception( "It is not advised to evaluate on more than 100 at once") multi_env.set_games(self.games, noise=False) # Fetch the dictionary of genomes genomes = list(iteritems(pop.population)) if parallel: # Initialize the evaluation-pool pool = mp.Pool(mp.cpu_count() - self.unused_cpu) manager = mp.Manager() return_dict = manager.dict() # Evaluate the genomes for genome in genomes: pool.apply_async(func=multi_env.eval_genome, args=(genome, return_dict)) pool.close() # Close the pool pool.join() # Postpone continuation until everything is finished else: # Evaluate sequentially return_dict = dict() for genome in genomes: multi_env.eval_genome(genome, return_dict) # Create blueprint of final result game_objects = [get_game(g, cfg=self.game_config) for g in self.games] path = get_subfolder( f"population{'_backup' if pop.use_backup else ''}/storage/{pop.folder_name}/{pop}/", 'images') path = get_subfolder(path, 'games') create_blueprints( final_observations=return_dict, games=game_objects, gen=pop.generation, save_path=path, )
def main(pop: Population, show: bool = True): """ Visualize the elites of the given population. Each generation, the average fitness of the three stored elites is taken. :param pop: Population object :param show: Show the result """ # Determine the distance between the representatives cache = GenomeDistanceCache(config=pop.config.genome) temp = [(s.key, s.representative) for s in pop.species.species.values()] species, representatives = zip(*sorted(temp, key=lambda x: x[0])) distances = zeros((len(representatives), ) * 2) # Square matrix for row, r1 in enumerate(representatives): for col, r2 in enumerate(representatives): distances[row, col] = cache(r1, r2) # Create the figure ax = sns.heatmap(distances, linewidth=0.5, annot=True, xticklabels=species, yticklabels=species) bottom, top = ax.get_ylim() ax.set_ylim(bottom + 0.5, top - 0.5) plt.title( f"Distance between elite representatives at generation {pop.generation}" ) plt.xlabel("specie") plt.ylabel("specie") plt.tick_params(labelbottom='on', labeltop='on', labelleft='on', labelright='on') plt.tight_layout() # Save the result f = get_subfolder( f"population{'_backup' if pop.use_backup else ''}/storage/{pop.folder_name}/{pop}/", 'images') f = get_subfolder(f, 'species') plt.savefig(f'{f}distances_gen_{pop.generation}') if show: plt.show() plt.close()
def main(genome: Genome, show: bool = False): """Visualize the genome's network.""" cfg = Config() path = get_subfolder(f'genomes_gru/images/', f"genome{genome.key}") draw_net(config=cfg.genome, genome=genome, debug=True, filename=f'{path}architecture', view=show)
def save(self): """ Save the population as the current generation. """ # Create needed subfolder if not yet exist f = get_subfolder( f"population{'_backup' if self.use_backup else ''}/storage/", f'{self.folder_name}') if len(str(self).split("/")) > 1: get_subfolder(f, f'{str(self).split("/")[0]}') f = get_subfolder(f, f'{self}') f = get_subfolder(f, 'generations') # Save the population store_pickle(self, f'{f}gen_{self.generation:05d}') self.log( f"Population '{self}' saved! Current generation: {self.generation}" )
def specie_representatives(pop: Population, show: bool = True, del_cache: bool = True): """Show for each of the current species their representative's architecture.""" species = pop.species.species elite_id = dict() for sid, s in sorted(species.items()): elite_id[sid] = s.representative.key pop.visualize_genome( debug=False, # Keep the networks simple genome=s.representative, show=False, ) hor = min(len(elite_id), 5) vert = max(len(elite_id) // 5 + 1, 1) plt.figure(figsize=(5 * hor, 5 * vert)) plt.tight_layout() path = get_subfolder( f"population{'_backup' if pop.use_backup else ''}/storage/{pop.folder_name}/{pop}/", "images") path_architectures = get_subfolder(path, "architectures") for i, (sid, eid) in enumerate(elite_id.items()): plt.subplot(vert, hor, i + 1) img = mpimg.imread(f'{path_architectures}genome_{eid}.png') plt.imshow(img) plt.title(f'Specie {sid}') plt.axis('off') # Save the result path_species = get_subfolder(path, 'species') plt.savefig(f'{path_species}representatives_gen{pop.generation}.png', bbox_inches='tight') if show: plt.show() plt.close() if del_cache: for eid in elite_id.values(): path = f'{path_architectures}genome_{eid}.png' if os.path.exists(path): os.remove(path)
def visualize_bar(topology_id: int, rounding: int = 2, use_backup: bool = False): """Visualize a bar-plot of how many genomes obtained which fitness score""" fitness = [] path_shared = get_subfolder( f"population{'_backup' if use_backup else ''}/storage/", "experiment6") path_data = get_subfolder(path_shared, "data") path_images = get_subfolder(path_shared, 'images') name = f"topology_{topology_id}" csv_name = f"{path_data}{name}.csv" # Read in the scores total_size = 0 with open(csv_name, 'r') as f: reader = csv.reader(f) next(reader, None) # skip the headers for row in reader: fitness.append(round(float(row[-1]), rounding)) total_size += 1 # Count the scores c = Counter() for f in fitness: c[f] += 1 # Plot the result plt.figure(figsize=(10, 5)) x, y = zip(*sorted(c.items())) i = 1 while i <= max(y): plt.axhline(i, color="grey", linewidth=0.5) i *= 10 plt.bar(x, y, width=1 / (10**rounding)) plt.yscale('log') plt.title("Fitness-distribution of uniformly sampled genome-space") plt.ylabel("Number of genomes") plt.xlabel("Fitness score") plt.savefig(f"{path_images}{name}.png") # plt.show() plt.close()
def evaluate_population(self, pop, game_ids=None): # TODO: Not used, remove? """ Evaluate the population on a set of games and create blueprints of the final positions afterwards. :param pop: Population object :param game_ids: List of game-ids """ if game_ids: self.set_games(game_ids) if len(self.games) > 20: raise Exception( "It is not advised to evaluate a whole population on more than 20 games at once" ) # Create the environment which is responsible for evaluating the genomes multi_env = get_multi_env(pop=pop, game_config=self.game_config) # Initialize the evaluation-pool pool = mp.Pool(mp.cpu_count()) manager = mp.Manager() return_dict = manager.dict() # Fetch the dictionary of genomes genomes = list(iteritems(pop.population)) # Progress bar during evaluation pbar = tqdm(total=len(genomes), desc="parallel training") def cb(*_): """Update progressbar after finishing a single genome's evaluation.""" pbar.update() # Evaluate the genomes for genome in genomes: pool.apply_async(func=multi_env.eval_genome, args=(genome, return_dict), callback=cb) pool.close() # Close the pool pool.join() # Postpone continuation until everything is finished pbar.close() # Close the progressbar # Create blueprint of final result game_objects = [get_game(g, cfg=self.game_config) for g in self.games] create_blueprints(final_observations=return_dict, games=game_objects, gen=pop.generation, save_path=get_subfolder( f'population/storage/{pop.folder_name}/{pop}/', 'images'))
def visualize_genome(self, debug=False, genome=None, show: bool = True): """ Visualize the architecture of the given genome. :param debug: Add excessive genome-specific details in the plot :param genome: Genome that must be visualized, best genome is chosen if none :param show: Directly visualize the architecture """ if not genome: genome = self.best_genome if self.best_genome else list( self.population.values())[0] name = f"genome_{genome.key}" sf = get_subfolder( f"population{'_backup' if self.use_backup else ''}/" f"storage/" f"{self.folder_name}/" f"{self}/", 'images') sf = get_subfolder(sf, f'architectures{"_debug" if debug else ""}') draw_net(config=self.config.genome, genome=genome, debug=debug, filename=f'{sf}{name}', view=show)
def trace_genomes(self, pop: Population, given_genome: Genome = None): """ Create blueprints that contain the walking-traces for all the requested mazes. :param pop: Population object :param given_genome: Single genomes for which the trace must be made """ multi_env = get_multi_env(pop=pop, game_config=self.game_config) if len(self.games) > 20: raise Exception( "It is not advised to evaluate on more than 20 at once") multi_env.set_games(self.games) # Initialize the evaluation-pool pool = mp.Pool(mp.cpu_count()) manager = mp.Manager() return_dict = manager.dict() # Fetch the dictionary of genomes genomes = [(given_genome.key, given_genome)] if given_genome else list( iteritems(pop.population)) # Progress bar during evaluation pbar = tqdm(total=len(genomes), desc="parallel evaluating") def cb(*_): """Update progressbar after finishing a single genome's evaluation.""" pbar.update() # Evaluate the genomes for genome in genomes: pool.apply_async(func=multi_env.trace_genome, args=(genome, return_dict), callback=cb) pool.close() # Close the pool pool.join() # Postpone continuation until everything is finished # Create blueprint of final result game_objects = [get_game(g, cfg=self.game_config) for g in self.games] create_traces( traces=return_dict, games=game_objects, gen=pop.generation, save_path=get_subfolder( f'population/storage/{pop.folder_name}/{pop}/', 'images'), save_name=f'trace_{given_genome.key}' if given_genome else 'trace', )
def create_blueprints(final_observations: dict, games: list, gen: int, save_path: str): """ Save images in the relative 'images/' subfolder of the population. :param final_observations: Dictionary of all the final game observations made :param games: List Game-objects used during evaluation :param gen: Population's current generation :param save_path: Path of 'images'-folder under which image must be saved """ genome_keys = list(final_observations.keys()) for g in games: # Get the game's blueprint g.get_blueprint() # Add arrow to indicate initial direction of robot x = g.player.init_pos[0] y = g.player.init_pos[1] dx = cos(g.player.init_angle) dy = sin(g.player.init_angle) plt.arrow(x, y, dx, dy, head_width=0.1, length_includes_head=True) # Get all the final positions of the agents positions = [] for gk in genome_keys: positions += [ fo[D_POS] for fo in final_observations[gk] if fo[D_GAME_ID] == g.id ] # Plot the positions dot_x = [p[0] for p in positions] dot_y = [p[1] for p in positions] plt.plot(dot_x, dot_y, 'ro') # Add title plt.title(f"Blueprint - Game {g.id:05d} - Generation {gen:05d}") # Save figure game_path = get_subfolder(save_path, 'game{id:05d}'.format(id=g.id)) plt.savefig(f'{game_path}blueprint_gen{gen:05d}') plt.close()
def evaluate_fitness(pop: Population, hops: float, weight_range: float, mutate_combine: bool = False, mutate_bias: bool = True, mutate_reset: bool = True, mutate_update: bool = True, mutate_candidate: bool = True): """Visualize the fitness-values of the population.""" # Initialization r = int(weight_range / hops) dim = 2 * r + 1 genome_key = 0 # Enroll the previous best genome init_gru = pop.best_genome.nodes[2] best_bias_genome = None best_reset_genome = None best_update_genome = None best_candidate_genome = None # Create genome-mutations if mutate_bias: bias_result = np.zeros((3, dim)) for i in range(3): for a in range(dim): g = pop.population[genome_key] bias_result[i, a] = g.fitness if best_bias_genome is None or g.fitness > best_bias_genome.fitness: best_bias_genome = g genome_key += 1 # Formalize the data points = [[x, y] for x in range(3) for y in range(dim)] points_normalized = [[p1, (p2 - r) * hops] for p1, p2 in points] values = [bias_result[p[0], p[1]] for p in points] grid_x, grid_y = np.mgrid[0:3:1, -r * hops:(r + 1) * hops:hops] # Create the figure plt.figure(figsize=( 10, 2.5)) # Rather horizontal plot due to limited number of rows knn_data = griddata(points_normalized, values, (grid_x, grid_y), method='nearest') ax = sns.heatmap( knn_data, annot=True, fmt='.3g', # vmin=0, # vmax=1, xticklabels=[round((i - r) * hops, 2) for i in range(dim)], yticklabels=['r', 'z', 'h'], cbar_kws={ "pad": 0.02, "fraction": 0.05 }, ) ax.invert_yaxis() plt.title('Bias mutation') plt.xlabel(r'$\Delta bias_h$' + f' (init={list(init_gru.bias_h)!r})') plt.ylabel(f'bias components') plt.tight_layout() path = f"population/storage/{pop.folder_name}/{pop}/" path = get_subfolder(path, 'images') path = get_subfolder(path, f'gen{pop.generation:05d}') plt.savefig(f"{path}bias.png") plt.close() if mutate_reset: reset_result = np.zeros((dim, dim)) for a in range(dim): for b in range(dim): g = pop.population[genome_key] reset_result[a, b] = g.fitness if best_reset_genome is None or g.fitness > best_reset_genome.fitness: best_reset_genome = g genome_key += 1 # Formalize the data points = [[x, y] for x in range(dim) for y in range(dim)] points_normalized = [[(p1 - r) * hops, (p2 - r) * hops] for p1, p2 in points] values = [reset_result[p[0], p[1]] for p in points] grid_x, grid_y = np.mgrid[-r * hops:(r + 1) * hops:hops, -r * hops:(r + 1) * hops:hops] # Create the figure plt.figure( figsize=(15, 15) ) # Rather horizontal plot due to limited number of rows TODO set back to (5, 5) knn_data = griddata(points_normalized, values, (grid_x, grid_y), method='nearest') ax = sns.heatmap( knn_data, annot=True, fmt='.3g', # vmin=0, # vmax=1, xticklabels=[round((i - r) * hops, 2) for i in range(dim)], yticklabels=[round((i - r) * hops, 2) for i in range(dim)], cbar_kws={ "pad": 0.02, "fraction": 0.05 }, ) ax.invert_yaxis() plt.title('Reset-gate mutation') plt.xlabel(r'$\Delta W_{hr}$' + f' (init={round(init_gru.weight_hh[0, 0], 3)})') plt.ylabel(r'$\Delta W_{xr}$' + f' (init={round(init_gru.weight_xh[0, 0], 3)})') plt.tight_layout() path = f"population/storage/{pop.folder_name}/{pop}/" path = get_subfolder(path, 'images') path = get_subfolder(path, f'gen{pop.generation:05d}') plt.savefig(f"{path}reset_gate.png") plt.close() if mutate_update: update_result = np.zeros((dim, dim)) for a in range(dim): for b in range(dim): g = pop.population[genome_key] update_result[a, b] = g.fitness if best_update_genome is None or g.fitness > best_update_genome.fitness: best_update_genome = g genome_key += 1 # Formalize the data points = [[x, y] for x in range(dim) for y in range(dim)] points_normalized = [[(p1 - r) * hops, (p2 - r) * hops] for p1, p2 in points] values = [update_result[p[0], p[1]] for p in points] grid_x, grid_y = np.mgrid[-r * hops:(r + 1) * hops:hops, -r * hops:(r + 1) * hops:hops] # Create the figure plt.figure( figsize=(15, 15) ) # Rather horizontal plot due to limited number of rows TODO set back to (5, 5) knn_data = griddata(points_normalized, values, (grid_x, grid_y), method='nearest') ax = sns.heatmap( knn_data, annot=True, fmt='.3g', # vmin=0, # vmax=1, xticklabels=[round((i - r) * hops, 2) for i in range(dim)], yticklabels=[round((i - r) * hops, 2) for i in range(dim)], cbar_kws={ "pad": 0.02, "fraction": 0.05 }, ) ax.invert_yaxis() plt.title('Update-gate mutation') plt.xlabel(r'$\Delta W_{hz}$' + f' (init={round(init_gru.weight_hh[1, 0], 3)})') plt.ylabel(r'$\Delta W_{xz}$' + f' (init={round(init_gru.weight_xh[1, 0], 3)})') plt.tight_layout() path = f"population/storage/{pop.folder_name}/{pop}/" path = get_subfolder(path, 'images') path = get_subfolder(path, f'gen{pop.generation:05d}') plt.savefig(f"{path}update_gate.png") plt.close() if mutate_candidate: candidate_result = np.zeros((dim, dim)) for a in range(dim): for b in range(dim): g = pop.population[genome_key] candidate_result[a, b] = g.fitness if best_candidate_genome is None or g.fitness > best_candidate_genome.fitness: best_candidate_genome = g genome_key += 1 # Formalize the data points = [[x, y] for x in range(dim) for y in range(dim)] points_normalized = [[(p1 - r) * hops, (p2 - r) * hops] for p1, p2 in points] values = [candidate_result[p[0], p[1]] for p in points] grid_x, grid_y = np.mgrid[-r * hops:(r + 1) * hops:hops, -r * hops:(r + 1) * hops:hops] # Create the figure plt.figure( figsize=(15, 15) ) # Rather horizontal plot due to limited number of rows TODO set back to (5, 5) knn_data = griddata(points_normalized, values, (grid_x, grid_y), method='nearest') ax = sns.heatmap( knn_data, annot=True, fmt='.3g', # vmin=0, # vmax=1, xticklabels=[round((i - r) * hops, 2) for i in range(dim)], yticklabels=[round((i - r) * hops, 2) for i in range(dim)], cbar_kws={ "pad": 0.02, "fraction": 0.05 }, ) ax.invert_yaxis() plt.title('Candidate-state mutation') plt.xlabel(r'$\Delta W_{hh}$' + f' (init={round(init_gru.weight_hh[2, 0], 3)})') plt.ylabel(r'$\Delta W_{xh}$' + f' (init={round(init_gru.weight_xh[2, 0], 3)})') plt.tight_layout() path = f"population/storage/{pop.folder_name}/{pop}/" path = get_subfolder(path, 'images') path = get_subfolder(path, f'gen{pop.generation:05d}') plt.savefig(f"{path}candidate_state.png") plt.close() # Set the most fit genome pop.best_genome.fitness = 0 if mutate_bias and best_bias_genome.fitness > pop.best_genome.fitness: pop.best_genome = copy.deepcopy(best_bias_genome) if mutate_reset and best_reset_genome.fitness > pop.best_genome.fitness: pop.best_genome = copy.deepcopy(best_reset_genome) if mutate_update and best_update_genome.fitness > pop.best_genome.fitness: pop.best_genome = copy.deepcopy(best_update_genome) if mutate_candidate and best_candidate_genome.fitness > pop.best_genome.fitness: pop.best_genome = copy.deepcopy(best_candidate_genome) if mutate_combine: pop.best_genome.nodes[2].bias_h = best_bias_genome.nodes[2].bias_h pop.best_genome.nodes[2].weight_xh_full[ 0, 0] = best_reset_genome.nodes[2].weight_xh_full[0, 0] pop.best_genome.nodes[2].weight_xh_full[ 1, 0] = best_update_genome.nodes[2].weight_xh_full[1, 0] pop.best_genome.nodes[2].weight_xh_full[ 2, 0] = best_candidate_genome.nodes[2].weight_xh_full[2, 0] pop.best_genome.nodes[2].weight_hh[ 0, 0] = best_reset_genome.nodes[2].weight_hh[0, 0] pop.best_genome.nodes[2].weight_hh[ 1, 0] = best_update_genome.nodes[2].weight_hh[1, 0] pop.best_genome.nodes[2].weight_hh[ 2, 0] = best_candidate_genome.nodes[2].weight_hh[2, 0]
def visualize_score(pop: Population, pop_cache: Population, genome: Genome, d: int, range_width: int): """Visualize the score of the evaluated population.""" pop_cache.log(f"{pop_cache.name} - Fetching fitness scores...") dim = (2 * range_width + 1) pbar = tqdm(range((dim ** 2) * 3), desc="Fetching fitness scores") genome_key = 0 # Fetching scores of reset-mutations reset_scores = np.zeros((dim, dim)) for a in range(dim): for b in range(dim): reset_scores[a, b] = pop_cache.population[genome_key].fitness genome_key += 1 pbar.update() # Fetching scores of update-mutations update_scores = np.zeros((dim, dim)) for a in range(dim): for b in range(dim): update_scores[a, b] = pop_cache.population[genome_key].fitness genome_key += 1 pbar.update() # Fetching scores of candidate-mutations candidate_scores = np.zeros((dim, dim)) for a in range(dim): for b in range(dim): candidate_scores[a, b] = pop_cache.population[genome_key].fitness genome_key += 1 pbar.update() pbar.close() # Visualize the result pop_cache.log(f"{pop_cache.name} - Visualizing the result...") # GRU-node needed for labels genome.update_rnn_nodes(config=pop.config.genome) gru_node = None for node in genome.nodes.values(): if type(node) == GruNodeGene: gru_node = node # Create the points and retrieve data for the plot points = [[x, y] for x in range(dim) for y in range(dim)] points_normalized = [[(p1 + -range_width) / d, (p2 + -range_width) / d] for p1, p2 in points] values_reset = [reset_scores[p[0], p[1]] for p in points] values_update = [update_scores[p[0], p[1]] for p in points] values_candidate = [candidate_scores[p[0], p[1]] for p in points] # K-nearest neighbours with hops of 0.01 is performed grid_x, grid_y = np.mgrid[-range_width / d:range_width / d:0.01, -range_width / d:range_width / d:0.01] # Perform k-NN data_reset = griddata(points_normalized, values_reset, (grid_x, grid_y), method='nearest') data_update = griddata(points_normalized, values_update, (grid_x, grid_y), method='nearest') data_candidate = griddata(points_normalized, values_candidate, (grid_x, grid_y), method='nearest') # Create the plots plt.figure(figsize=(15, 5)) plt.subplot(131) plt.imshow(data_reset.T, vmin=0, vmax=1, extent=(-range_width / d, range_width / d, -range_width / d, range_width / d), origin='lower') plt.title('Reset-gate mutation') plt.xlabel(r'$\Delta W_{hr}$' + f' (init={round(gru_node.weight_hh[0, 0], 3)})') plt.ylabel(r'$\Delta W_{xr}$' + f' (init={round(gru_node.weight_xh[0, 0], 3)})') plt.subplot(132) plt.imshow(data_update.T, vmin=0, vmax=1, extent=(-range_width / d, range_width / d, -range_width / d, range_width / d), origin='lower') plt.title('Update-gate mutation') plt.xlabel(r'$\Delta W_{hz}$' + f' (init={round(gru_node.weight_hh[1, 0], 3)})') plt.ylabel(r'$\Delta W_{xz}$' + f' (init={round(gru_node.weight_xh[1, 0], 3)})') plt.subplot(133) plt.imshow(data_candidate.T, vmin=0, vmax=1, extent=(-range_width / d, range_width / d, -range_width / d, range_width / d), origin='lower') plt.title('Candidate-state mutation') plt.xlabel(r'$\Delta W_{hh}$' + f' (init={round(gru_node.weight_hh[2, 0], 3)})') plt.ylabel(r'$\Delta W_{xh}$' + f' (init={round(gru_node.weight_xh[2, 0], 3)})') # Store the plot plt.tight_layout() path = f"population{'_backup' if pop.use_backup else ''}/storage/{pop.folder_name}/{pop}/" path = get_subfolder(path, 'images') path = get_subfolder(path, 'gru_analysis') plt.savefig(f"{path}{genome.key}.png") plt.savefig(f"{path}{genome.key}.eps", format="eps") plt.close() # Create overview pop_cache.log("Overview of results:") log = dict() # Overview: reset-mutation max_index_reset, max_value_reset, min_index_reset, min_value_reset = None, 0, None, 1 for index, x in np.ndenumerate(reset_scores): if x < min_value_reset: min_index_reset, min_value_reset = index, x if x > max_value_reset: max_index_reset, max_value_reset = index, x pop_cache.log(f"\tReset-gate mutation:") pop_cache.log(f"\t > Maximum fitness: {round(max_value_reset, 2)} for index {max_index_reset!r}") pop_cache.log(f"\t > Average fitness: {round(np.average(reset_scores), 2)}") pop_cache.log(f"\t > Minimum fitness: {round(min_value_reset, 2)} for index {min_index_reset!r}") log['Reset-gate maximum fitness'] = f"{round(max_value_reset, 2)} for index {max_index_reset!r}" log['Reset-gate average fitness'] = f"{round(np.average(reset_scores), 2)}" log['Reset-gate minimum fitness'] = f"{round(min_value_reset, 2)} for index {min_index_reset!r}" # Overview: update-mutation max_index_update, max_value_update, min_index_update, min_value_update = None, 0, None, 1 for index, x in np.ndenumerate(update_scores): if x < min_value_update: min_index_update, min_value_update = index, x if x > max_value_update: max_index_update, max_value_update = index, x pop_cache.log(f"\tUpdate-gate mutation:") pop_cache.log(f"\t > Maximum fitness: {round(max_value_update, 2)} for index {max_index_update!r}") pop_cache.log(f"\t > Average fitness: {round(np.average(update_scores), 2)}") pop_cache.log(f"\t > Minimum fitness: {round(min_value_update, 2)} for index {min_index_update!r}") log['Update-gate maximum fitness'] = f"{round(max_value_update, 2)} for index {max_index_update!r}" log['Update-gate average fitness'] = f"{round(np.average(update_scores), 2)}" log['Update-gate minimum fitness'] = f"{round(min_value_update, 2)} for index {min_index_update!r}" # Overview: candidate-mutation max_index_candidate, max_value_candidate, min_index_candidate, min_value_candidate = None, 0, None, 1 for index, x in np.ndenumerate(candidate_scores): if x < min_value_candidate: min_index_candidate, min_value_candidate = index, x if x > max_value_candidate: max_index_candidate, max_value_candidate = index, x pop_cache.log(f"\tCandidate-state mutation:") pop_cache.log(f"\t > Maximum fitness: {round(max_value_candidate, 2)} for index {max_index_candidate!r}") pop_cache.log(f"\t > Average fitness: {round(np.average(candidate_scores), 2)}") pop_cache.log(f"\t > Minimum fitness: {round(min_value_candidate, 2)} for index {min_index_candidate!r}") log['Candidate-state maximum fitness'] = f"{round(max_value_candidate, 2)} for index {max_index_candidate!r}" log['Candidate-state average fitness'] = f"{round(np.average(candidate_scores), 2)}" log['Candidate-state minimum fitness'] = f"{round(min_value_candidate, 2)} for index {min_index_candidate!r}" update_dict(f'{path}{genome.key}.txt', log)
def evaluate_populations(folder: str, pop_folder: str, max_v: int = 50): """ Evaluate the various populations against each other. Note that it is assumed that 'evaluate_generations' has ran first. """ if folder[-1] != '/': folder += '/' if pop_folder[-1] != '/': pop_folder += '/' # Load in dummy population print( f"\n===> COMBINING POPULATION RESULTS OF FOLDER {folder}{pop_folder} <===" ) pop = Population( name=f'{pop_folder}v1', folder_name=folder, log_print=False, use_backup=True, ) max_gen = pop.generation # Parse the results fitness_dict = dict() finished_dict = dict() score_dict = dict() distance_dict = dict() time_dict = dict() for g in range(0, max_gen + 1, HOPS): fitness_dict[g] = [] finished_dict[g] = [] score_dict[g] = [] distance_dict[g] = [] time_dict[g] = [] for v in range(1, max_v + 1): results: dict = load_dict( f"population_backup/storage/{folder}{pop_folder}v{v}/evaluation/{g:05d}/results" ) fitness_dict[g].append( max([results[k][D_FITNESS] for k in results.keys()])) finished_dict[g].append( max([results[k][D_FINISHED] / 100 for k in results.keys()])) score_dict[g].append( max([results[k][D_SCORE_AVG] for k in results.keys()])) distance_dict[g].append( min([results[k][D_DISTANCE_AVG] for k in results.keys()])) time_dict[g].append( min([results[k][D_TIME_AVG] for k in results.keys()])) # Save received data in evaluation subfolder of the population folder path = get_subfolder(f'population_backup/storage/{folder}{pop_folder}', 'evaluation') update_dict(f'{path}fitness', fitness_dict, overwrite=True) update_dict(f'{path}finished', finished_dict, overwrite=True) update_dict(f'{path}score', score_dict, overwrite=True) update_dict(f'{path}distance', distance_dict, overwrite=True) update_dict(f'{path}time', time_dict, overwrite=True) # Visualize the data path_images = get_subfolder(path, 'images') plot_result(d=fitness_dict, ylabel="fitness", title="Average fitness", save_path=f'{path_images}fitness') plot_result(d=finished_dict, ylabel="finished ratio", title="Averaged finished ratio", save_path=f'{path_images}finished') plot_result(d=score_dict, ylabel="score", title="Average score", save_path=f'{path_images}score') plot_result(d=distance_dict, ylabel="distance (m)", title="Average final distance to target", save_path=f'{path_images}distance') plot_result(d=time_dict, ylabel="time (s)", title="Average simulation time", save_path=f'{path_images}time')
def compute_complexity( folder: str, neat: bool = False, neat_gru: bool = False, neat_lstm: bool = False, neat_sru: bool = False, neat_sru_s: bool = False, gen: int = 500, max_v: int = 50, ): """Compute the complexity of the populations' elites.""" # Collect all the populations populations = [] if neat: populations.append(D_NEAT) if neat_gru: populations.append(D_NEAT_GRU) if neat_lstm: populations.append(D_NEAT_LSTM) if neat_sru: populations.append(D_NEAT_SRU) if neat_sru_s: populations.append(D_NEAT_SRU_S) if len(populations) == 0: return # Go over all possibilities print(f"\n===> COMPUTING POPULATION'S ELITE COMPLEXITY <===") path = f"population_backup/storage/{folder}/" genes_dict = dict() for pop in populations: path_eval = get_subfolder(f"{path}{pop}/", 'evaluation') complexity = Counter() genes = Counter() genes_detailed = dict() for v in range(1, max_v + 1): population = Population( name=f'{pop}/v{v}', folder_name=folder, use_backup=True, ) if population.generation == 0: raise Exception(f"Population {pop}/v{v} loaded incorrectly") if population.generation != gen: population.load(gen=gen) s = population.best_genome.size() complexity[str(s)] += 1 c = str(s[0] + s[1]) genes[c] += 1 if c in genes_detailed: genes_detailed[c].append(v) else: genes_detailed[c] = [v] # Store results at populations themselves update_dict(f'{path_eval}complexity_topology', complexity, overwrite=True) update_dict(f'{path_eval}complexity_genes', genes, overwrite=True) update_dict(f'{path_eval}complexity_genes_detailed', genes_detailed, overwrite=True) # Update global dictionary keys = list(genes.keys()) for k in keys: genes[int(k)] = genes[k] del genes[k] genes_dict[pop] = list(sorted(genes.items())) plt.figure(figsize=(10, 2.5)) max_x = max([max([a for a, _ in genes_dict[pop]]) for pop in populations]) min_x = min([min([a for a, _ in genes_dict[pop]]) for pop in populations]) for idx, pop in enumerate(populations): keys = [a for a, _ in genes_dict[pop]] for x in range(max_x): if x not in keys: genes_dict[pop].append((x, 0)) x, y = zip(*genes_dict[pop]) width = 0.8 / len(populations) plt.bar(x=np.asarray(x) - 0.4 + width / 2 + idx * width, height=y, width=width, linewidth=2, label=pop, color=COLORS[pop]) # Beautify the plot plt.xlim(min_x - .5, max_x + .5) plt.xticks([i for i in range(min_x, max_x + 1)]) leg = plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.18), fancybox=True, fontsize=10, ncol=len(populations)) for line in leg.get_lines(): line.set_linewidth(4.0) plt.grid(axis='y') plt.tight_layout() plt.xlabel("complexity expressed in #genes") plt.ylabel("#elites") plt.savefig(f"population_backup/storage/{folder}/images/complexity.png", bbox_inches='tight', pad_inches=0.02) plt.savefig(f"population_backup/storage/{folder}/images/complexity.eps", format='eps', bbox_inches='tight', pad_inches=0.02) # plt.show() plt.close() # Also create a violin plot of the distribution if only two populations if len(populations) == 2: max_x = 0 min_x = float('inf') df = pd.DataFrame() palette = [] for idx, pop in enumerate(populations): values = [] for a, b in genes_dict[pop]: for _ in range(b): values.append(a) # Remove outliers values = sorted(values) q1 = min(values[int(round(1 / 4 * len(values)))], values[int(round(3 / 4 * len(values)))]) q3 = max(values[int(round(1 / 4 * len(values)))], values[int(round(3 / 4 * len(values)))]) iqr = q3 - q1 for i in range(len(values) - 1, -1, -1): if (values[i] < (q1 - 1.5 * iqr)) or (values[i] > (q3 + 1.5 * iqr)): del values[i] if min(values) < min_x: min_x = min(values) if max(values) > max_x: max_x = max(values) df = df.append( pd.DataFrame({ 'complexity': values, 'y': 'ignore', 'pop': pop })) palette.append(COLORS[pop]) # Create the plot plt.figure(figsize=(10, 2.5)) sns.violinplot(data=df, x="complexity", y="y", hue="pop", palette=palette, split=True, inner="quartile") plt.xlim(min_x - .5, max_x + .5) plt.xticks([i for i in range(min_x, max_x + 1)]) plt.xlabel("complexity expressed in #genes") plt.yticks([]) plt.ylabel('elite genome density') leg = plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.25), fancybox=True, fontsize=10, ncol=len(populations)) for line in leg.get_lines(): line.set_linewidth(4.0) plt.tight_layout() plt.savefig( f"population_backup/storage/{folder}/images/complexity_violin.png", bbox_inches='tight', pad_inches=0.02) plt.savefig( f"population_backup/storage/{folder}/images/complexity_violin.eps", format='eps', bbox_inches='tight', pad_inches=0.02) plt.show() plt.close()
def plot_distribution( folder: str, neat: bool = False, neat_gru: bool = False, neat_lstm: bool = False, neat_sru: bool = False, neat_sru_s: bool = False, gen: int = 500, ): """ Plot the one-dimensional distribution of all of the populations on each of the evaluation measures for the requested generation. It is assumed that the evaluation-data has already been collected. """ # Collect all the populations populations = [] if neat: populations.append(D_NEAT) if neat_gru: populations.append(D_NEAT_GRU) if neat_lstm: populations.append(D_NEAT_LSTM) if neat_sru: populations.append(D_NEAT_SRU) if neat_sru_s: populations.append(D_NEAT_SRU_S) if len(populations) == 0: return # Collect all the measure options OPTIONS = ['distance', 'finished', 'fitness', 'score', 'time', 'training'] # Go over all possibilities print(f"\n===> CREATING POPULATION DISTRIBUTIONS <===") path = f"population_backup/storage/{folder}/" path_images = get_subfolder(path, 'images') for option in OPTIONS: plt.figure(figsize=(10, 2.5)) min_val = float("inf") max_val = -float("inf") for pop in populations: d = load_dict(f"{path}{pop}/evaluation/{option}") dist = d[str(gen)] if min(dist) < min_val: min_val = min(dist) if max(dist) > max_val: max_val = max(dist) # Remove outliers first dist = sorted(dist) q1 = min(dist[int(round(1 / 4 * len(dist)))], dist[int(round(3 / 4 * len(dist)))]) q3 = max(dist[int(round(1 / 4 * len(dist)))], dist[int(round(3 / 4 * len(dist)))]) iqr = q3 - q1 for i in range(len(dist) - 1, -1, -1): if (dist[i] < (q1 - 1.5 * iqr)) or (dist[i] > (q3 + 1.5 * iqr)): del dist[i] sns.distplot( dist, hist=False, kde=True, norm_hist=True, bins=100, color=COLORS[pop], kde_kws={'linewidth': 2}, label=pop, ) plt.xlim(min_val, max_val) # plt.title(f"Probability density across populations for '{option}' at generation {gen}") plt.xlabel(option) # plt.yticks([]) plt.ylabel('probability density') leg = plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.2), fancybox=True, fontsize=8, ncol=len(populations)) for line in leg.get_lines(): line.set_linewidth(4.0) plt.tight_layout() plt.savefig(f"{path_images}dist_{option}.png", bbox_inches='tight', pad_inches=0.02) plt.savefig(f"{path_images}dist_{option}.eps", format='eps', bbox_inches='tight', pad_inches=0.02) # plt.show() plt.close()
def main(population: Population, game_id: int, genome: Genome = None, game_cfg: Config = None, average: int = 1, debug: bool = False): """ Monitor the genome on the following elements: * Position * Hidden state of SRU (Ht) * Actuation of both wheels * Distance * Delta distance """ # Make sure all parameters are set if not genome: genome = population.best_genome if not game_cfg: game_cfg = pop.config # Check if valid genome (contains at least one hidden SRU, first SRU is monitored) - also possible for fixed RNNs a = len([n for n in genome.get_used_nodes().values() if type(n) == SimpleRnnNodeGene]) >= 1 b = len([n for n in genome.get_used_nodes().values() if type(n) == FixedRnnNodeGene]) >= 1 assert a or b # Get the game game = get_game(game_id, cfg=game_cfg, noise=False) state = game.reset()[D_SENSOR_LIST] step_num = 0 # Create the network net = make_net(genome=genome, genome_config=population.config.genome, batch_size=1, initial_read=state, ) # Containers to monitor actuation = [] distance = [] delta_distance = [] position = [] Ht = [] target_found = [] score = 0 # Initialize the containers actuation.append([0, 0]) distance.append(state[0]) delta_distance.append(0) position.append(game.player.pos.get_tuple()) Ht.append(net.rnn_state[0, 0, 0]) if debug: print(f"Step: {step_num}") print(f"\t> Actuation: {(round(actuation[-1][0], 5), round(actuation[-1][1], 5))!r}") print(f"\t> Distance: {round(distance[-1], 5)} - Delta distance: {round(delta_distance[-1], 5)}") print(f"\t> Position: {(round(position[-1][0], 2), round(position[-1][1], 2))!r}") print(f"\t> SRU state: Ht={round(Ht[-1], 5)}") # Start monitoring while True: # Check if maximum iterations is reached if step_num == game_cfg.game.duration * game_cfg.game.fps: break # Determine the actions made by the agent for each of the states action = net(np.asarray([state])) # Check if each game received an action assert len(action) == 1 # Proceed the game with one step, based on the predicted action obs = game.step(l=action[0][0], r=action[0][1]) finished = obs[D_DONE] # Update the score-count if game.score > score: target_found.append(step_num) score = game.score # Update the candidate's current state state = obs[D_SENSOR_LIST] # Stop if agent reached target in all the games if finished: break step_num += 1 # Update the containers actuation.append(action[0]) distance.append(state[0]) delta_distance.append(distance[-2] - distance[-1]) position.append(game.player.pos.get_tuple()) Ht.append(net.rnn_state[0, 0, 0]) if debug: print(f"Step: {step_num}") print(f"\t> Actuation: {(round(actuation[-1][0], 5), round(actuation[-1][1], 5))!r}") print(f"\t> Distance: {round(distance[-1], 5)} - Delta distance: {round(delta_distance[-1], 5)}") print(f"\t> Position: {(round(position[-1][0], 2), round(position[-1][1], 2))!r}") print(f"\t> SRU state: Ht={round(Ht[-1], 5)}") if average > 1: # Average out the noise x, y = zip(*actuation) x = SMA(x, window=average) y = SMA(y, window=average) actuation = list(zip(x, y)) distance = SMA(distance, window=average) delta_distance = SMA(delta_distance, window=average) Ht = SMA(Ht, window=average) # Resolve weird artifacts at the beginning for i in range(average, 0, -1): actuation[i - 1] = actuation[i] distance[i - 1] = distance[i] delta_distance[i - 1] = delta_distance[i] Ht[i - 1] = Ht[i] # Visualize the monitored values path = get_subfolder(f"population{'_backup' if population.use_backup else ''}/" f"storage/" f"{population.folder_name}/" f"{population}/", "images") path = get_subfolder(path, f"monitor") path = get_subfolder(path, f"{genome.key}") path = get_subfolder(path, f"{game_id}") visualize_actuation(actuation, target_found=target_found, game_cfg=game_cfg.game, save_path=f"{path}actuation.png") visualize_distance(distance, target_found=target_found, game_cfg=game_cfg.game, save_path=f"{path}distance.png") visualize_hidden_state(Ht, target_found=target_found, game_cfg=game_cfg.game, save_path=f"{path}hidden_state.png") visualize_position(position, game=game, save_path=f"{path}trace.png") merge(f"Monitored genome={genome.key} on game={game.id}", path=path)
def main(population: Population, game_id: int, genome: Genome = None, game_cfg: Config = None, debug: bool = False): """ Monitor the genome on the following elements: * Position * Update gate (Zt) * Hidden state of GRU (Ht) * Actuation of both wheels * Distance """ # Make sure all parameters are set if not genome: genome = population.best_genome if not game_cfg: game_cfg = pop.config # Check if valid genome (contains at least one hidden GRU, first GRU is monitored) assert len([ n for n in genome.get_used_nodes().values() if type(n) == GruNoResetNodeGene ]) >= 1 # Get the game game = get_game(game_id, cfg=game_cfg, noise=False) state = game.reset()[D_SENSOR_LIST] step_num = 0 # Create the network net = make_net( genome=genome, genome_config=population.config.genome, batch_size=1, initial_read=state, ) # Containers to monitor actuation = [] distance = [] position = [] Ht = [] Ht_tilde = [] Zt = [] target_found = [] score = 0 # Initialize the containers actuation.append([0, 0]) distance.append(state[0]) position.append(game.player.pos.get_tuple()) ht, ht_tilde, zt = get_gru_states(net=net, x=np.asarray([state])) Ht.append(ht) Ht_tilde.append(ht_tilde) Zt.append(zt) if debug: print(f"Step: {step_num}") print( f"\t> Actuation: {(round(actuation[-1][0], 5), round(actuation[-1][1], 5))!r}" ) print(f"\t> Distance: {round(distance[-1], 5)}") print( f"\t> Position: {(round(position[-1][0], 2), round(position[-1][1], 2))!r}" ) print(f"\t> GRU states: " f"\t\tHt={round(Ht[-1], 5)}" f"\t\tHt_tilde={round(Ht_tilde[-1], 5)}" f"\t\tZt={round(Zt[-1], 5)}") # Start monitoring while True: # Check if maximum iterations is reached if step_num == game_cfg.game.duration * game_cfg.game.fps: break # Determine the actions made by the agent for each of the states action = net(np.asarray([state])) # Check if each game received an action assert len(action) == 1 # Proceed the game with one step, based on the predicted action obs = game.step(l=action[0][0], r=action[0][1]) finished = obs[D_DONE] # Update the score-count if game.score > score: target_found.append(step_num) score = game.score # Update the candidate's current state state = obs[D_SENSOR_LIST] # Stop if agent reached target in all the games if finished: break step_num += 1 # Update the containers actuation.append(action[0]) distance.append(state[0]) position.append(game.player.pos.get_tuple()) ht, ht_tilde, zt = get_gru_states(net=net, x=np.asarray([state])) Ht.append(ht) Ht_tilde.append(ht_tilde) Zt.append(zt) if debug: print(f"Step: {step_num}") print( f"\t> Actuation: {(round(actuation[-1][0], 5), round(actuation[-1][1], 5))!r}" ) print(f"\t> Distance: {round(distance[-1], 5)}") print( f"\t> Position: {(round(position[-1][0], 2), round(position[-1][1], 2))!r}" ) print(f"\t> GRU states: " f"\t\tHt={round(Ht[-1], 5)}" f"\t\tHt_tilde={round(Ht_tilde[-1], 5)}" f"\t\tZt={round(Zt[-1], 5)}") # Visualize the monitored values path = get_subfolder( f"population{'_backup' if population.use_backup else ''}/" f"storage/" f"{population.folder_name}/" f"{population}/", "images") path = get_subfolder(path, f"monitor") path = get_subfolder(path, f"{genome.key}") path = get_subfolder(path, f"{game_id}") visualize_actuation(actuation, target_found=target_found, game_cfg=game_cfg.game, save_path=f"{path}actuation.png") visualize_distance(distance, target_found=target_found, game_cfg=game_cfg.game, save_path=f"{path}distance.png") visualize_hidden_state(Ht, target_found=target_found, game_cfg=game_cfg.game, save_path=f"{path}hidden_state.png") visualize_candidate_hidden_state( Ht_tilde, target_found=target_found, game_cfg=game_cfg.game, save_path=f"{path}candidate_hidden_state.png") visualize_update_gate(Zt, target_found=target_found, game_cfg=game_cfg.game, save_path=f"{path}update_gate.png") visualize_position(position, game=game, save_path=f"{path}trace.png") merge(f"Monitored genome={genome.key} on game={game.id}", path=path)
def combine_all_populations( folder: str, max_v: int = None, neat: bool = False, neat_gru: bool = False, neat_lstm: bool = False, neat_sru: bool = False, neat_sru_s: bool = False, ): """Combine the scores for all of the populations in a given folder.""" # Collect all the populations populations = [] if neat: populations.append(D_NEAT) if neat_gru: populations.append(D_NEAT_GRU) if neat_lstm: populations.append(D_NEAT_LSTM) if neat_sru: populations.append(D_NEAT_SRU) if neat_sru_s: populations.append(D_NEAT_SRU_S) if len(populations) == 0: return # Collect all the measure options OPTIONS = ['distance', 'finished', 'fitness', 'score', 'time', 'training'] # OPTIONS = ['fitness'] # Go over all possibilities print(f"\n===> COMBINING POPULATIONS OF FOLDER {folder} <===") path = f"population_backup/storage/{folder}/" path_images = get_subfolder(path, 'images') for option in OPTIONS: plt.figure(figsize=(8, 2.5)) max_data = 0 max_gen = 0 for pop in populations: # Load the dictionary d = load_dict(f"{path}{pop}/evaluation/{option}") size = len(list(d.values())[0]) if max_v: assert size == max_v # Prepare the data containers q1 = [] q2 = [] # Median q3 = [] idx_q1 = int(round(1 / 4 * size)) idx_q2 = int(round(2 / 4 * size)) idx_q3 = int(round(3 / 4 * size)) # Loop over each iteration x = sorted([int(k) for k in d.keys()]) for g in x: if g > max_gen: max_gen = g lst = sorted(d[str(g)]) # Sort values from low to high q1.append(lst[idx_q1]) q2.append(lst[idx_q2]) q3.append(lst[idx_q3]) # Plot the results plt.plot(x, q1, color=COLORS[pop], linestyle=":", linewidth=.5) plt.plot(x, q3, color=COLORS[pop], linestyle=":", linewidth=.5) plt.plot(x, q2, color=COLORS[pop], linestyle="-", linewidth=2, label=pop) plt.fill_between(x, q1, q3, color=COLORS[pop], alpha=0.2) # Update the max-counter if max(q3) > max_data: max_data = max(q3) # Finalize the figure leg = plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.25), fancybox=True, fontsize=10, ncol=len(populations)) for line in leg.get_lines(): line.set_linewidth(4.0) # plt.xticks([i * 100 for i in range(11)]) # TODO plt.xlabel("generation") plt.xlim(0, max_gen) # plt.yticks([i for i in range(7)]) # TODO plt.ylabel(option) plt.ylim(0, max(max_data * 1.05, 1.05)) # plt.ylim(0, 6) # TODO plt.grid() plt.tight_layout() plt.savefig(f"{path_images}comb_{option}.png", bbox_inches='tight', pad_inches=0.02, dpi=500) # plt.savefig(f"{path_images}comb_{option}.eps", format="eps", bbox_inches='tight', pad_inches=0.02) # plt.show() plt.close()
def visualize_generations(name, experiment_id, folder=None, hops: int = 10): """Visualize the result of the 'evaluate_generations' script. Should only be used for debugging!""" # Fetch population and evaluation games folder = folder if folder else get_folder(experiment_id) pop = Population( name=name, folder_name=folder, log_print=False, use_backup=True, ) # Parse the results fitness_dict = dict() finished_dict = dict() score_dict = dict() distance_dict = dict() time_dict = dict() max_gen = pop.generation for gen in tqdm(range(0, max_gen + 1, hops)): results: dict = load_dict( f"population{'_backup' if pop.use_backup else ''}/" f"storage/" f"{pop.folder_name}/" f"{pop}/" f"evaluation/" f"{gen:05d}/" f"results") # Fitness fitness = [results[k][D_FITNESS] for k in results.keys()] fitness_dict[gen] = fitness # Finished finished = [results[k][D_FINISHED] / 100 for k in results.keys()] # Normalize to percentage (0..1) finished_dict[gen] = finished # Score score = [results[k][D_SCORE_AVG] for k in results.keys()] score_dict[gen] = score # Distance distance = [results[k][D_DISTANCE_AVG] for k in results.keys()] distance_dict[gen] = distance # Time time = [results[k][D_TIME_AVG] for k in results.keys()] time_dict[gen] = time # Create visualizations for each of the results sf = get_subfolder( f"population{'_backup' if pop.use_backup else ''}/" f"storage/" f"{pop.folder_name}/" f"{pop}/" f"images/", 'evaluation') plot_population(fitness_dict, ylabel="Fitness score", title="Fitness (higher is better)", save_path=f'{sf}fitness_group.png') plot_elite(fitness_dict, f=max, ylabel="Fitness score", title="Fitness of population's elite (higher is better)", save_path=f'{sf}fitness_elite.png') plot_population(finished_dict, ylabel="Percentage finished (%)", title="Percentage finished (higher is better)", save_path=f'{sf}finished_group.png') plot_elite( finished_dict, f=max, ylabel="Percentage finished (%)", title="Percentage finished by population's elite (higher is better)", save_path=f'{sf}finished_elite.png') plot_population(score_dict, ylabel="Score", title="Final scores (higher is better)", save_path=f'{sf}score_group.png') plot_elite(score_dict, f=max, ylabel="Score", title="Final score of population's elite (higher is better)", save_path=f'{sf}score_elite.png') plot_population(distance_dict, ylabel="Distance (m)", title="Distance from target (lower is better)", save_path=f'{sf}distance_group.png') plot_elite( distance_dict, f=min, ylabel="Distance (m)", title="Distance from target for population's elite (lower is better)", save_path=f'{sf}distance_elite.png') plot_population(time_dict, ylabel="Time (s)", title="Time needed to reach target (lower is better)", save_path=f'{sf}time_group.png') plot_elite( time_dict, f=min, ylabel="Time (s)", title= "Time needed to reach target for population's elite (lower is better)", save_path=f'{sf}time_elite.png')