def evaluate_generations(name, experiment_id, folder=None, hops: int = 10, unused_cpu: int = 2): """ Evaluate the population across its lifetime. At each generation, the ten best genomes are evaluated together with the elite genome of the past five generations. :param name: Name of the population :param experiment_id: Experiment for which the population is trained (and now will be evaluated) :param folder: Population-folder (~experiment level) :param hops: Number of generations between each saved population :param unused_cpu: Number of CPU cores not used """ # Fetch population and evaluation games folder = folder if folder else get_folder(experiment_id) pop = Population( name=name, folder_name=folder, log_print=False, use_backup=True, ) _, game_ids_eval = get_game_ids(experiment_id=experiment_id) # Perform the evaluations max_gen = pop.generation for gen in tqdm(range(0, max_gen + 1, hops)): # Load in the current generation if not pop.load(gen=gen): raise Exception( f"Population {name} is not trained for generation {gen}") # Collect the used genomes if gen > 5: genomes = sorted([g for g in pop.population.values()], key=lambda x: x.fitness if x.fitness else 0, reverse=True)[:10] for i in range(1, 6): keys = [g.key for g in genomes] g = copy.deepcopy(pop.best_genome_hist[gen - i] [1]) # Copy since chance of mutation while g.key in keys: # Already added to genomes, update keys g.key += 1 genomes.append(g) else: # No history yet, use only the ten most fit genomes from the current generation genomes = sorted([g for g in pop.population.values()], key=lambda x: x.fitness if x.fitness else 0, reverse=True)[:15] # Evaluate the selected genomes evaluate( population=pop, games=game_ids_eval, genomes=genomes, unused_cpu=unused_cpu, overwrite=True, )
def evaluate_training(experiment_id: int, pop_folder: str, folder: str = None, max_v: int = 50): """Evaluate the fitness of a population's elite each training generation.""" if pop_folder[-1] != '/': pop_folder += '/' folder = folder if folder else get_folder(experiment_id) if folder[-1] != '/': folder += '/' # Get dummy population pop = Population( name=f"{pop_folder}v1", folder_name=folder, log_print=False, use_backup=True, ) max_gen = pop.generation # Initialize data container training_fitness = dict() for g in range(0, max_gen + 1, HOPS): training_fitness[g] = [] # Pull the training scores print( f"\n===> PULLING TRAINING FITNESS OF THE {pop_folder} POPULATIONS <===" ) pbar = tqdm(range(int(max_v * (max_gen / HOPS + 1)))) for v in range(1, max_v + 1): name = f"{pop_folder}v{v}" pop = Population( name=name, folder_name=folder, log_print=False, use_backup=True, ) # Perform the evaluations max_gen = pop.generation for gen in range(0, max_gen + 1, HOPS): if not pop.load(gen=gen): raise Exception( f"Population {name} is not trained for generation {gen}") training_fitness[gen].append( pop.best_genome.fitness if pop.best_genome else 0) pbar.update() pbar.close() # Plot the result path = get_subfolder(f'population_backup/storage/{folder}{pop_folder}', 'evaluation') update_dict(f'{path}training', training_fitness, overwrite=True) path_images = get_subfolder(path, 'images') plot_result(d=training_fitness, ylabel="fitness", title="Average training fitness", save_path=f'{path_images}training')
def compute_complexity( folder: str, neat: bool = False, neat_gru: bool = False, neat_lstm: bool = False, neat_sru: bool = False, neat_sru_s: bool = False, gen: int = 500, max_v: int = 50, ): """Compute the complexity of the populations' elites.""" # Collect all the populations populations = [] if neat: populations.append(D_NEAT) if neat_gru: populations.append(D_NEAT_GRU) if neat_lstm: populations.append(D_NEAT_LSTM) if neat_sru: populations.append(D_NEAT_SRU) if neat_sru_s: populations.append(D_NEAT_SRU_S) if len(populations) == 0: return # Go over all possibilities print(f"\n===> COMPUTING POPULATION'S ELITE COMPLEXITY <===") path = f"population_backup/storage/{folder}/" genes_dict = dict() for pop in populations: path_eval = get_subfolder(f"{path}{pop}/", 'evaluation') complexity = Counter() genes = Counter() genes_detailed = dict() for v in range(1, max_v + 1): population = Population( name=f'{pop}/v{v}', folder_name=folder, use_backup=True, ) if population.generation == 0: raise Exception(f"Population {pop}/v{v} loaded incorrectly") if population.generation != gen: population.load(gen=gen) s = population.best_genome.size() complexity[str(s)] += 1 c = str(s[0] + s[1]) genes[c] += 1 if c in genes_detailed: genes_detailed[c].append(v) else: genes_detailed[c] = [v] # Store results at populations themselves update_dict(f'{path_eval}complexity_topology', complexity, overwrite=True) update_dict(f'{path_eval}complexity_genes', genes, overwrite=True) update_dict(f'{path_eval}complexity_genes_detailed', genes_detailed, overwrite=True) # Update global dictionary keys = list(genes.keys()) for k in keys: genes[int(k)] = genes[k] del genes[k] genes_dict[pop] = list(sorted(genes.items())) plt.figure(figsize=(10, 2.5)) max_x = max([max([a for a, _ in genes_dict[pop]]) for pop in populations]) min_x = min([min([a for a, _ in genes_dict[pop]]) for pop in populations]) for idx, pop in enumerate(populations): keys = [a for a, _ in genes_dict[pop]] for x in range(max_x): if x not in keys: genes_dict[pop].append((x, 0)) x, y = zip(*genes_dict[pop]) width = 0.8 / len(populations) plt.bar(x=np.asarray(x) - 0.4 + width / 2 + idx * width, height=y, width=width, linewidth=2, label=pop, color=COLORS[pop]) # Beautify the plot plt.xlim(min_x - .5, max_x + .5) plt.xticks([i for i in range(min_x, max_x + 1)]) leg = plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.18), fancybox=True, fontsize=10, ncol=len(populations)) for line in leg.get_lines(): line.set_linewidth(4.0) plt.grid(axis='y') plt.tight_layout() plt.xlabel("complexity expressed in #genes") plt.ylabel("#elites") plt.savefig(f"population_backup/storage/{folder}/images/complexity.png", bbox_inches='tight', pad_inches=0.02) plt.savefig(f"population_backup/storage/{folder}/images/complexity.eps", format='eps', bbox_inches='tight', pad_inches=0.02) # plt.show() plt.close() # Also create a violin plot of the distribution if only two populations if len(populations) == 2: max_x = 0 min_x = float('inf') df = pd.DataFrame() palette = [] for idx, pop in enumerate(populations): values = [] for a, b in genes_dict[pop]: for _ in range(b): values.append(a) # Remove outliers values = sorted(values) q1 = min(values[int(round(1 / 4 * len(values)))], values[int(round(3 / 4 * len(values)))]) q3 = max(values[int(round(1 / 4 * len(values)))], values[int(round(3 / 4 * len(values)))]) iqr = q3 - q1 for i in range(len(values) - 1, -1, -1): if (values[i] < (q1 - 1.5 * iqr)) or (values[i] > (q3 + 1.5 * iqr)): del values[i] if min(values) < min_x: min_x = min(values) if max(values) > max_x: max_x = max(values) df = df.append( pd.DataFrame({ 'complexity': values, 'y': 'ignore', 'pop': pop })) palette.append(COLORS[pop]) # Create the plot plt.figure(figsize=(10, 2.5)) sns.violinplot(data=df, x="complexity", y="y", hue="pop", palette=palette, split=True, inner="quartile") plt.xlim(min_x - .5, max_x + .5) plt.xticks([i for i in range(min_x, max_x + 1)]) plt.xlabel("complexity expressed in #genes") plt.yticks([]) plt.ylabel('elite genome density') leg = plt.legend(loc='upper center', bbox_to_anchor=(0.5, 1.25), fancybox=True, fontsize=10, ncol=len(populations)) for line in leg.get_lines(): line.set_linewidth(4.0) plt.tight_layout() plt.savefig( f"population_backup/storage/{folder}/images/complexity_violin.png", bbox_inches='tight', pad_inches=0.02) plt.savefig( f"population_backup/storage/{folder}/images/complexity_violin.eps", format='eps', bbox_inches='tight', pad_inches=0.02) plt.show() plt.close()