def evaluate_generations(name,
experiment_id,
folder=None,
hops: int = 10,
unused_cpu: int = 2):
"""
Evaluate the population across its lifetime. At each generation, the ten best genomes are evaluated together with
the elite genome of the past five generations.
:param name: Name of the population
:param experiment_id: Experiment for which the population is trained (and now will be evaluated)
:param folder: Population-folder (~experiment level)
:param hops: Number of generations between each saved population
:param unused_cpu: Number of CPU cores not used
"""
# Fetch population and evaluation games
folder = folder if folder else get_folder(experiment_id)
pop = Population(
name=name,
folder_name=folder,
log_print=False,
use_backup=True,
)
_, game_ids_eval = get_game_ids(experiment_id=experiment_id)
# Perform the evaluations
max_gen = pop.generation
for gen in tqdm(range(0, max_gen + 1, hops)):
# Load in the current generation
if not pop.load(gen=gen):
raise Exception(
f"Population {name} is not trained for generation {gen}")
# Collect the used genomes
if gen > 5:
genomes = sorted([g for g in pop.population.values()],
key=lambda x: x.fitness if x.fitness else 0,
reverse=True)[:10]
for i in range(1, 6):
keys = [g.key for g in genomes]
g = copy.deepcopy(pop.best_genome_hist[gen - i]
[1]) # Copy since chance of mutation
while g.key in keys: # Already added to genomes, update keys
g.key += 1
genomes.append(g)
else:
# No history yet, use only the ten most fit genomes from the current generation
genomes = sorted([g for g in pop.population.values()],
key=lambda x: x.fitness if x.fitness else 0,
reverse=True)[:15]
# Evaluate the selected genomes
evaluate(
population=pop,
games=game_ids_eval,
genomes=genomes,
unused_cpu=unused_cpu,
overwrite=True,
)
def evaluate_training(experiment_id: int,
pop_folder: str,
folder: str = None,
max_v: int = 50):
"""Evaluate the fitness of a population's elite each training generation."""
if pop_folder[-1] != '/': pop_folder += '/'
folder = folder if folder else get_folder(experiment_id)
if folder[-1] != '/': folder += '/'
# Get dummy population
pop = Population(
name=f"{pop_folder}v1",
folder_name=folder,
log_print=False,
use_backup=True,
)
max_gen = pop.generation
# Initialize data container
training_fitness = dict()
for g in range(0, max_gen + 1, HOPS):
training_fitness[g] = []
# Pull the training scores
print(
f"\n===> PULLING TRAINING FITNESS OF THE {pop_folder} POPULATIONS <==="
)
pbar = tqdm(range(int(max_v * (max_gen / HOPS + 1))))
for v in range(1, max_v + 1):
name = f"{pop_folder}v{v}"
pop = Population(
name=name,
folder_name=folder,
log_print=False,
use_backup=True,
)
# Perform the evaluations
max_gen = pop.generation
for gen in range(0, max_gen + 1, HOPS):
if not pop.load(gen=gen):
raise Exception(
f"Population {name} is not trained for generation {gen}")
training_fitness[gen].append(
pop.best_genome.fitness if pop.best_genome else 0)
pbar.update()
pbar.close()
# Plot the result
path = get_subfolder(f'population_backup/storage/{folder}{pop_folder}',
'evaluation')
update_dict(f'{path}training', training_fitness, overwrite=True)
path_images = get_subfolder(path, 'images')
plot_result(d=training_fitness,
ylabel="fitness",
title="Average training fitness",
save_path=f'{path_images}training')
def compute_complexity(
folder: str,
neat: bool = False,
neat_gru: bool = False,
neat_lstm: bool = False,
neat_sru: bool = False,
neat_sru_s: bool = False,
gen: int = 500,
max_v: int = 50,
):
"""Compute the complexity of the populations' elites."""
# Collect all the populations
populations = []
if neat: populations.append(D_NEAT)
if neat_gru: populations.append(D_NEAT_GRU)
if neat_lstm: populations.append(D_NEAT_LSTM)
if neat_sru: populations.append(D_NEAT_SRU)
if neat_sru_s: populations.append(D_NEAT_SRU_S)
if len(populations) == 0: return
# Go over all possibilities
print(f"\n===> COMPUTING POPULATION'S ELITE COMPLEXITY <===")
path = f"population_backup/storage/{folder}/"
genes_dict = dict()
for pop in populations:
path_eval = get_subfolder(f"{path}{pop}/", 'evaluation')
complexity = Counter()
genes = Counter()
genes_detailed = dict()
for v in range(1, max_v + 1):
population = Population(
name=f'{pop}/v{v}',
folder_name=folder,
use_backup=True,
)
if population.generation == 0:
raise Exception(f"Population {pop}/v{v} loaded incorrectly")
if population.generation != gen: population.load(gen=gen)
s = population.best_genome.size()
complexity[str(s)] += 1
c = str(s[0] + s[1])
genes[c] += 1
if c in genes_detailed:
genes_detailed[c].append(v)
else:
genes_detailed[c] = [v]
# Store results at populations themselves
update_dict(f'{path_eval}complexity_topology',
complexity,
overwrite=True)
update_dict(f'{path_eval}complexity_genes', genes, overwrite=True)
update_dict(f'{path_eval}complexity_genes_detailed',
genes_detailed,
overwrite=True)
# Update global dictionary
keys = list(genes.keys())
for k in keys:
genes[int(k)] = genes[k]
del genes[k]
genes_dict[pop] = list(sorted(genes.items()))
plt.figure(figsize=(10, 2.5))
max_x = max([max([a for a, _ in genes_dict[pop]]) for pop in populations])
min_x = min([min([a for a, _ in genes_dict[pop]]) for pop in populations])
for idx, pop in enumerate(populations):
keys = [a for a, _ in genes_dict[pop]]
for x in range(max_x):
if x not in keys: genes_dict[pop].append((x, 0))
x, y = zip(*genes_dict[pop])
width = 0.8 / len(populations)
plt.bar(x=np.asarray(x) - 0.4 + width / 2 + idx * width,
height=y,
width=width,
linewidth=2,
label=pop,
color=COLORS[pop])
# Beautify the plot
plt.xlim(min_x - .5, max_x + .5)
plt.xticks([i for i in range(min_x, max_x + 1)])
leg = plt.legend(loc='upper center',
bbox_to_anchor=(0.5, 1.18),
fancybox=True,
fontsize=10,
ncol=len(populations))
for line in leg.get_lines():
line.set_linewidth(4.0)
plt.grid(axis='y')
plt.tight_layout()
plt.xlabel("complexity expressed in #genes")
plt.ylabel("#elites")
plt.savefig(f"population_backup/storage/{folder}/images/complexity.png",
bbox_inches='tight',
pad_inches=0.02)
plt.savefig(f"population_backup/storage/{folder}/images/complexity.eps",
format='eps',
bbox_inches='tight',
pad_inches=0.02)
# plt.show()
plt.close()
# Also create a violin plot of the distribution if only two populations
if len(populations) == 2:
max_x = 0
min_x = float('inf')
df = pd.DataFrame()
palette = []
for idx, pop in enumerate(populations):
values = []
for a, b in genes_dict[pop]:
for _ in range(b):
values.append(a)
# Remove outliers
values = sorted(values)
q1 = min(values[int(round(1 / 4 * len(values)))],
values[int(round(3 / 4 * len(values)))])
q3 = max(values[int(round(1 / 4 * len(values)))],
values[int(round(3 / 4 * len(values)))])
iqr = q3 - q1
for i in range(len(values) - 1, -1, -1):
if (values[i] < (q1 - 1.5 * iqr)) or (values[i] >
(q3 + 1.5 * iqr)):
del values[i]
if min(values) < min_x: min_x = min(values)
if max(values) > max_x: max_x = max(values)
df = df.append(
pd.DataFrame({
'complexity': values,
'y': 'ignore',
'pop': pop
}))
palette.append(COLORS[pop])
# Create the plot
plt.figure(figsize=(10, 2.5))
sns.violinplot(data=df,
x="complexity",
y="y",
hue="pop",
palette=palette,
split=True,
inner="quartile")
plt.xlim(min_x - .5, max_x + .5)
plt.xticks([i for i in range(min_x, max_x + 1)])
plt.xlabel("complexity expressed in #genes")
plt.yticks([])
plt.ylabel('elite genome density')
leg = plt.legend(loc='upper center',
bbox_to_anchor=(0.5, 1.25),
fancybox=True,
fontsize=10,
ncol=len(populations))
for line in leg.get_lines():
line.set_linewidth(4.0)
plt.tight_layout()
plt.savefig(
f"population_backup/storage/{folder}/images/complexity_violin.png",
bbox_inches='tight',
pad_inches=0.02)
plt.savefig(
f"population_backup/storage/{folder}/images/complexity_violin.eps",
format='eps',
bbox_inches='tight',
pad_inches=0.02)
plt.show()
plt.close()
