def elite_architecture(pop: Population,
show: bool = False,
del_cache: bool = True):
"""
Visualize each architectural change in the population's elites.
:param pop: Population object
:param show: Show the result
:param del_cache: Remove intermediate architecture-results when images are created
"""
# Initialize the architecture-list with the first genome
distance = GenomeDistanceCache(config=pop.config.genome)
new_architectures = [(0, pop.best_genome_hist[0][1])
] # Only interested in genome itself
for gen in range(1, pop.generation):
gen_genome = pop.best_genome_hist[gen][1]
if distance(
gen_genome, new_architectures[-1]
[1]) > 2: # Take only the more significant changes into account
new_architectures.append((gen, gen_genome))
new_architectures.append(
(pop.generation - 1, pop.best_genome_hist[pop.generation - 1][1]))
# Create the architectures of the unique genomes
for _, g in new_architectures:
pop.visualize_genome(
debug=False, # Keep the networks simple
genome=g,
show=False,
)
# Combine in one figure
hor = min(len(new_architectures), 5)
vert = max((len(new_architectures) - 1) // 5 + 1, 1)
plt.figure(figsize=(5 * hor, 5 * vert))
plt.tight_layout()
f_images = get_subfolder(
f"population{'_backup' if pop.use_backup else ''}/storage/{pop.folder_name}/{pop}/",
"images")
f_architectures = get_subfolder(f_images, "architectures")
for i, (gen, g) in enumerate(new_architectures):
plt.subplot(vert, hor, i + 1)
img = mpimg.imread(f'{f_architectures}genome_{g.key}.png')
plt.imshow(img)
plt.title(f'Generation {gen}')
plt.axis('off')
# Save the result
f_elites = get_subfolder(f_images, "elites")
plt.savefig(f'{f_elites}/architecture_timeline.png', bbox_inches='tight')
if show:
plt.show()
plt.close()
if del_cache:
for (_, g) in new_architectures:
path = f'{f_architectures}genome_{g.key}.png'
if os.path.exists(path): os.remove(path)
def evaluate_training(experiment_id: int,
pop_folder: str,
folder: str = None,
max_v: int = 50):
"""Evaluate the fitness of a population's elite each training generation."""
if pop_folder[-1] != '/': pop_folder += '/'
folder = folder if folder else get_folder(experiment_id)
if folder[-1] != '/': folder += '/'
# Get dummy population
pop = Population(
name=f"{pop_folder}v1",
folder_name=folder,
log_print=False,
use_backup=True,
)
max_gen = pop.generation
# Initialize data container
training_fitness = dict()
for g in range(0, max_gen + 1, HOPS):
training_fitness[g] = []
# Pull the training scores
print(
f"\n===> PULLING TRAINING FITNESS OF THE {pop_folder} POPULATIONS <==="
)
pbar = tqdm(range(int(max_v * (max_gen / HOPS + 1))))
for v in range(1, max_v + 1):
name = f"{pop_folder}v{v}"
pop = Population(
name=name,
folder_name=folder,
log_print=False,
use_backup=True,
)
# Perform the evaluations
max_gen = pop.generation
for gen in range(0, max_gen + 1, HOPS):
if not pop.load(gen=gen):
raise Exception(
f"Population {name} is not trained for generation {gen}")
training_fitness[gen].append(
pop.best_genome.fitness if pop.best_genome else 0)
pbar.update()
pbar.close()
# Plot the result
path = get_subfolder(f'population_backup/storage/{folder}{pop_folder}',
'evaluation')
update_dict(f'{path}training', training_fitness, overwrite=True)
path_images = get_subfolder(path, 'images')
plot_result(d=training_fitness,
ylabel="fitness",
title="Average training fitness",
save_path=f'{path_images}training')
def add_evaluation_result(self, eval_result, overwrite: bool = False):
"""Append the result of the evaluation."""
sf = get_subfolder(
f"population{'_backup' if self.use_backup else ''}/"
f"storage/"
f"{self.folder_name}/"
f'{self}/', 'evaluation')
sf = get_subfolder(sf, f"{self.generation:05d}")
update_dict(f'{sf}results', eval_result, overwrite=overwrite)
def trace_genomes(self,
pop: Population,
given_genome: Genome = None,
parallel: bool = True):
"""
Create blueprints that contain the walking-traces for all the requested mazes.
:param pop: Population object
:param given_genome: Single genomes for which the trace must be made
:param parallel: Create the traces in parallel
"""
multi_env = get_multi_env(pop=pop, game_config=self.game_config)
if len(self.games) > 20 and given_genome is None:
raise Exception(
"It is not advised to evaluate on more than 20 at once")
elif len(self.games) > 100:
raise Exception(
"It is not advised to evaluate on more than 100 at once")
# Set the games for which traces will be made
multi_env.set_games(self.games, noise=False)
# Fetch the dictionary of genomes
genomes = [(given_genome.key, given_genome)] if given_genome else list(
iteritems(pop.population))
if parallel:
# Initialize the evaluation-pool
pool = mp.Pool(mp.cpu_count() - self.unused_cpu)
manager = mp.Manager()
return_dict = manager.dict()
# Evaluate the genomes
for genome in genomes:
pool.apply_async(func=multi_env.trace_genome,
args=(genome, return_dict))
pool.close() # Close the pool
pool.join() # Postpone continuation until everything is finished
else: # Train sequentially
return_dict = dict()
for genome in tqdm(genomes, desc="sequential evaluating"):
multi_env.trace_genome(genome, return_dict)
# Create blueprint of final result
game_objects = [get_game(g, cfg=self.game_config) for g in self.games]
path = get_subfolder(
f"population{'_backup' if pop.use_backup else ''}/storage/{pop.folder_name}/{pop}/",
'images')
path = get_subfolder(path, 'games')
create_traces(
traces=return_dict,
games=game_objects,
gen=pop.generation,
save_path=path,
save_name=f'trace_{given_genome.key}' if given_genome else 'trace',
)
def get_save_path(gid: int, save_name):
"""Get the correct path to save a certain file in."""
save_path = get_subfolder(f"genomes_gru/", f"images")
save_path = get_subfolder(save_path, f"genome{gid}")
if len(save_name.split('/')) == 1:
path = f"{save_path}{save_name}"
elif len(save_name.split('/')) == 2:
path = get_subfolder(f"{save_path}", save_name.split('/')[0])
path = f"{path}{save_name.split('/')[1]}"
else:
raise Exception(f"Too long save_name: '{save_name}'")
return path
def visualize_best_genome(pop: Population):
"""Visualize the population's fittest genome."""
pop.log(
f"Most fit genome: {pop.best_genome.key} with fitness: {round(pop.best_genome.fitness, 3)}"
)
name = f"genome_{pop.best_genome.key}"
sf = get_subfolder(f'population/storage/{pop.folder_name}/{pop}/',
'images')
sf = get_subfolder(sf, f'gen{pop.generation:05d}')
draw_net(config=pop.config.genome,
genome=pop.best_genome,
debug=True,
filename=f'{sf}{name}',
view=False)
def get_initial_keys(topology_id: int, use_backup: bool):
"""Get the genome-key based on CSV-file's length."""
path = get_subfolder(
f"population{'_backup' if use_backup else ''}/storage/", "experiment6")
path = get_subfolder(path, "data")
csv_name = f"topology_{topology_id}"
path = f"{path}{csv_name}.csv"
# CSV exists, count number of rows
if os.path.exists(path):
with open(path, 'r') as f:
return sum(1 for _ in f), path
# CSV does not exist, create new
else:
with open(path, 'w', newline='') as f:
writer = csv.writer(f)
# Construct the CSV's head, all genomes have the full GRU-parameter suite
head = []
if topology_id in [1, 2, 3, 30]: # GRU populations
head += [
'bias_r', 'bias_z', 'bias_h', 'weight_xr', 'weight_xz',
'weight_xh', 'weight_hr', 'weight_hz', 'weight_hh'
]
elif topology_id in [22, 33]: # SRU populations
head += ['bias_h', 'weight_xh', 'weight_hh']
elif topology_id in [222]:
head += ['delay', 'scale', 'bias_h']
elif topology_id in [2222, 3333]:
head += [
'bias_z', 'bias_h', 'weight_xz', 'weight_xh', 'weight_hz',
'weight_hh'
]
elif topology_id in [22222]:
head += ['bias_h', 'weight']
else:
raise Exception(f"Topology ID '{topology_id}' not supported!")
if topology_id in [1]:
head += ['conn1', 'conn2']
elif topology_id in [2, 22, 222, 2222, 22222]:
head += ['bias_rw', 'conn2']
elif topology_id in [3, 30, 33, 3333]:
head += ['bias_rw', 'conn0', 'conn1', 'conn2']
else:
raise Exception(f"Topology ID '{topology_id}' not supported!")
head += ['fitness']
writer.writerow(head)
return 1, path
def elite_fitness(pop: Population, window: int = 5, show: bool = True):
"""
Visualize the elites of the given population. Each generation, the average fitness of the three stored elites is
taken.
:param pop: Population object
:param window: Window-size used in the function
:param show: Show the result
"""
f = get_subfolder(
f"population{'_backup' if pop.use_backup else ''}/storage/{pop.folder_name}/{pop}/",
'images')
f = get_subfolder(f, 'elites')
for func in [Forward, SMA, EMA]:
# Fetch name based on used function
name = f'{"EMA_" if func == EMA else "SMA_" if func == SMA else ""}gen_{pop.generation}'
# Load in the relevant data
history = sorted(pop.best_fitness.items(), key=lambda x: x[0])
generations, fitness = zip(*history)
# Create the figure
ax = plt.figure().gca()
plt.plot(generations, func(fitness, window))
if func == SMA:
plt.title(
f"Elite fitness in population: {pop}\nSimple Moving Average (window={window})"
)
elif func == EMA:
plt.title(
f"Elite fitness in population: {pop}\nExponential Moving Average (window={window})"
)
else:
plt.title(f"Elite fitness in population: {pop}")
plt.xlabel("generation")
plt.ylabel("fitness")
ax.xaxis.set_major_locator(
MaxNLocator(integer=True)) # Forces to use only integers
if max(fitness) <= 1:
plt.yticks([i / 10 for i in range(11)
]) # Fitness expressed in range of 0..1 (hops of 0.1)
plt.grid(axis='y')
plt.tight_layout()
# Save the result
plt.savefig(f'{f}{name}')
if show:
plt.show()
plt.close()
def get_csv_path(topology_id: int, use_backup: bool, batch_size: int):
"""Get the genome-key based on CSV-file's length."""
path = get_subfolder(f"population{'_backup' if use_backup else ''}/storage/", "experiment6")
path = get_subfolder(path, "data_neat")
csv_name = f"topology_{topology_id}"
path = f"{path}{csv_name}.csv"
# If CSV exists, check if not yet full
if os.path.exists(path):
with open(path, 'r') as f:
rows = sum(1 for _ in f) - 1 # Do not count header
if rows < batch_size: return path, csv_name, rows
# CSV does not yet exist, or is already full, create new CSV
else:
with open(path, 'w', newline='') as f:
writer = csv.writer(f)
# Construct the CSV's head
head = []
if topology_id in [1, 2, 3, 30]: # GRU populations
head += ['bias_r', 'bias_z', 'bias_h',
'weight_xr', 'weight_xz', 'weight_xh',
'weight_hr', 'weight_hz', 'weight_hh']
elif topology_id in [22, 33]: # SRU populations
head += ['bias_h', 'weight_xh', 'weight_hh']
elif topology_id in [222]:
head += ['delay', 'scale', 'resting']
elif topology_id in [2222, 3333]:
head += ['bias_z', 'bias_h',
'weight_xz', 'weight_xh',
'weight_hz', 'weight_hh']
elif topology_id in [22222]:
head += ['bias_h', 'weight']
else:
raise Exception(f"Topology ID '{topology_id}' not supported!")
if topology_id in [1]:
head += ['conn1', 'conn2']
elif topology_id in [2, 22, 222, 2222, 22222]:
head += ['bias_rw', 'conn2']
elif topology_id in [3, 30, 33, 3333]:
head += ['bias_rw', 'conn0', 'conn1', 'conn2']
else:
raise Exception(f"Topology ID '{topology_id}' not supported!")
head += ['fitness']
writer.writerow(head)
return path, csv_name, 0
def blueprint_genomes(self, pop: Population, parallel: bool = True):
"""
Create blueprints for all the requested mazes.
:param pop: Population object
:param parallel: Evaluate the population in parallel
"""
multi_env = get_multi_env(pop=pop, game_config=self.game_config)
if len(self.games) > 100:
raise Exception(
"It is not advised to evaluate on more than 100 at once")
multi_env.set_games(self.games, noise=False)
# Fetch the dictionary of genomes
genomes = list(iteritems(pop.population))
if parallel:
# Initialize the evaluation-pool
pool = mp.Pool(mp.cpu_count() - self.unused_cpu)
manager = mp.Manager()
return_dict = manager.dict()
# Evaluate the genomes
for genome in genomes:
pool.apply_async(func=multi_env.eval_genome,
args=(genome, return_dict))
pool.close() # Close the pool
pool.join() # Postpone continuation until everything is finished
else: # Evaluate sequentially
return_dict = dict()
for genome in genomes:
multi_env.eval_genome(genome, return_dict)
# Create blueprint of final result
game_objects = [get_game(g, cfg=self.game_config) for g in self.games]
path = get_subfolder(
f"population{'_backup' if pop.use_backup else ''}/storage/{pop.folder_name}/{pop}/",
'images')
path = get_subfolder(path, 'games')
create_blueprints(
final_observations=return_dict,
games=game_objects,
gen=pop.generation,
save_path=path,
)
def main(pop: Population, show: bool = True):
"""
Visualize the elites of the given population. Each generation, the average fitness of the three stored elites is
taken.
:param pop: Population object
:param show: Show the result
"""
# Determine the distance between the representatives
cache = GenomeDistanceCache(config=pop.config.genome)
temp = [(s.key, s.representative) for s in pop.species.species.values()]
species, representatives = zip(*sorted(temp, key=lambda x: x[0]))
distances = zeros((len(representatives), ) * 2) # Square matrix
for row, r1 in enumerate(representatives):
for col, r2 in enumerate(representatives):
distances[row, col] = cache(r1, r2)
# Create the figure
ax = sns.heatmap(distances,
linewidth=0.5,
annot=True,
xticklabels=species,
yticklabels=species)
bottom, top = ax.get_ylim()
ax.set_ylim(bottom + 0.5, top - 0.5)
plt.title(
f"Distance between elite representatives at generation {pop.generation}"
)
plt.xlabel("specie")
plt.ylabel("specie")
plt.tick_params(labelbottom='on',
labeltop='on',
labelleft='on',
labelright='on')
plt.tight_layout()
# Save the result
f = get_subfolder(
f"population{'_backup' if pop.use_backup else ''}/storage/{pop.folder_name}/{pop}/",
'images')
f = get_subfolder(f, 'species')
plt.savefig(f'{f}distances_gen_{pop.generation}')
if show:
plt.show()
plt.close()
def main(genome: Genome, show: bool = False):
"""Visualize the genome's network."""
cfg = Config()
path = get_subfolder(f'genomes_gru/images/', f"genome{genome.key}")
draw_net(config=cfg.genome,
genome=genome,
debug=True,
filename=f'{path}architecture',
view=show)
def save(self):
"""
Save the population as the current generation.
"""
# Create needed subfolder if not yet exist
f = get_subfolder(
f"population{'_backup' if self.use_backup else ''}/storage/",
f'{self.folder_name}')
if len(str(self).split("/")) > 1:
get_subfolder(f, f'{str(self).split("/")[0]}')
f = get_subfolder(f, f'{self}')
f = get_subfolder(f, 'generations')
# Save the population
store_pickle(self, f'{f}gen_{self.generation:05d}')
self.log(
f"Population '{self}' saved! Current generation: {self.generation}"
)
def specie_representatives(pop: Population,
show: bool = True,
del_cache: bool = True):
"""Show for each of the current species their representative's architecture."""
species = pop.species.species
elite_id = dict()
for sid, s in sorted(species.items()):
elite_id[sid] = s.representative.key
pop.visualize_genome(
debug=False, # Keep the networks simple
genome=s.representative,
show=False,
)
hor = min(len(elite_id), 5)
vert = max(len(elite_id) // 5 + 1, 1)
plt.figure(figsize=(5 * hor, 5 * vert))
plt.tight_layout()
path = get_subfolder(
f"population{'_backup' if pop.use_backup else ''}/storage/{pop.folder_name}/{pop}/",
"images")
path_architectures = get_subfolder(path, "architectures")
for i, (sid, eid) in enumerate(elite_id.items()):
plt.subplot(vert, hor, i + 1)
img = mpimg.imread(f'{path_architectures}genome_{eid}.png')
plt.imshow(img)
plt.title(f'Specie {sid}')
plt.axis('off')
# Save the result
path_species = get_subfolder(path, 'species')
plt.savefig(f'{path_species}representatives_gen{pop.generation}.png',
bbox_inches='tight')
if show:
plt.show()
plt.close()
if del_cache:
for eid in elite_id.values():
path = f'{path_architectures}genome_{eid}.png'
if os.path.exists(path): os.remove(path)
def visualize_bar(topology_id: int,
rounding: int = 2,
use_backup: bool = False):
"""Visualize a bar-plot of how many genomes obtained which fitness score"""
fitness = []
path_shared = get_subfolder(
f"population{'_backup' if use_backup else ''}/storage/", "experiment6")
path_data = get_subfolder(path_shared, "data")
path_images = get_subfolder(path_shared, 'images')
name = f"topology_{topology_id}"
csv_name = f"{path_data}{name}.csv"
# Read in the scores
total_size = 0
with open(csv_name, 'r') as f:
reader = csv.reader(f)
next(reader, None) # skip the headers
for row in reader:
fitness.append(round(float(row[-1]), rounding))
total_size += 1
# Count the scores
c = Counter()
for f in fitness:
c[f] += 1
# Plot the result
plt.figure(figsize=(10, 5))
x, y = zip(*sorted(c.items()))
i = 1
while i <= max(y):
plt.axhline(i, color="grey", linewidth=0.5)
i *= 10
plt.bar(x, y, width=1 / (10**rounding))
plt.yscale('log')
plt.title("Fitness-distribution of uniformly sampled genome-space")
plt.ylabel("Number of genomes")
plt.xlabel("Fitness score")
plt.savefig(f"{path_images}{name}.png")
# plt.show()
plt.close()
def evaluate_population(self,
pop,
game_ids=None): # TODO: Not used, remove?
"""
Evaluate the population on a set of games and create blueprints of the final positions afterwards.
:param pop: Population object
:param game_ids: List of game-ids
"""
if game_ids: self.set_games(game_ids)
if len(self.games) > 20:
raise Exception(
"It is not advised to evaluate a whole population on more than 20 games at once"
)
# Create the environment which is responsible for evaluating the genomes
multi_env = get_multi_env(pop=pop, game_config=self.game_config)
# Initialize the evaluation-pool
pool = mp.Pool(mp.cpu_count())
manager = mp.Manager()
return_dict = manager.dict()
# Fetch the dictionary of genomes
genomes = list(iteritems(pop.population))
# Progress bar during evaluation
pbar = tqdm(total=len(genomes), desc="parallel training")
def cb(*_):
"""Update progressbar after finishing a single genome's evaluation."""
pbar.update()
# Evaluate the genomes
for genome in genomes:
pool.apply_async(func=multi_env.eval_genome,
args=(genome, return_dict),
callback=cb)
pool.close() # Close the pool
pool.join() # Postpone continuation until everything is finished
pbar.close() # Close the progressbar
# Create blueprint of final result
game_objects = [get_game(g, cfg=self.game_config) for g in self.games]
create_blueprints(final_observations=return_dict,
games=game_objects,
gen=pop.generation,
save_path=get_subfolder(
f'population/storage/{pop.folder_name}/{pop}/',
'images'))
def visualize_genome(self, debug=False, genome=None, show: bool = True):
"""
Visualize the architecture of the given genome.
:param debug: Add excessive genome-specific details in the plot
:param genome: Genome that must be visualized, best genome is chosen if none
:param show: Directly visualize the architecture
"""
if not genome:
genome = self.best_genome if self.best_genome else list(
self.population.values())[0]
name = f"genome_{genome.key}"
sf = get_subfolder(
f"population{'_backup' if self.use_backup else ''}/"
f"storage/"
f"{self.folder_name}/"
f"{self}/", 'images')
sf = get_subfolder(sf, f'architectures{"_debug" if debug else ""}')
draw_net(config=self.config.genome,
genome=genome,
debug=debug,
filename=f'{sf}{name}',
view=show)
def trace_genomes(self, pop: Population, given_genome: Genome = None):
"""
Create blueprints that contain the walking-traces for all the requested mazes.
:param pop: Population object
:param given_genome: Single genomes for which the trace must be made
"""
multi_env = get_multi_env(pop=pop, game_config=self.game_config)
if len(self.games) > 20:
raise Exception(
"It is not advised to evaluate on more than 20 at once")
multi_env.set_games(self.games)
# Initialize the evaluation-pool
pool = mp.Pool(mp.cpu_count())
manager = mp.Manager()
return_dict = manager.dict()
# Fetch the dictionary of genomes
genomes = [(given_genome.key, given_genome)] if given_genome else list(
iteritems(pop.population))
# Progress bar during evaluation
pbar = tqdm(total=len(genomes), desc="parallel evaluating")
def cb(*_):
"""Update progressbar after finishing a single genome's evaluation."""
pbar.update()
# Evaluate the genomes
for genome in genomes:
pool.apply_async(func=multi_env.trace_genome,
args=(genome, return_dict),
callback=cb)
pool.close() # Close the pool
pool.join() # Postpone continuation until everything is finished
# Create blueprint of final result
game_objects = [get_game(g, cfg=self.game_config) for g in self.games]
create_traces(
traces=return_dict,
games=game_objects,
gen=pop.generation,
save_path=get_subfolder(
f'population/storage/{pop.folder_name}/{pop}/', 'images'),
save_name=f'trace_{given_genome.key}' if given_genome else 'trace',
)
def create_blueprints(final_observations: dict, games: list, gen: int,
save_path: str):
"""
Save images in the relative 'images/' subfolder of the population.
:param final_observations: Dictionary of all the final game observations made
:param games: List Game-objects used during evaluation
:param gen: Population's current generation
:param save_path: Path of 'images'-folder under which image must be saved
"""
genome_keys = list(final_observations.keys())
for g in games:
# Get the game's blueprint
g.get_blueprint()
# Add arrow to indicate initial direction of robot
x = g.player.init_pos[0]
y = g.player.init_pos[1]
dx = cos(g.player.init_angle)
dy = sin(g.player.init_angle)
plt.arrow(x, y, dx, dy, head_width=0.1, length_includes_head=True)
# Get all the final positions of the agents
positions = []
for gk in genome_keys:
positions += [
fo[D_POS] for fo in final_observations[gk]
if fo[D_GAME_ID] == g.id
]
# Plot the positions
dot_x = [p[0] for p in positions]
dot_y = [p[1] for p in positions]
plt.plot(dot_x, dot_y, 'ro')
# Add title
plt.title(f"Blueprint - Game {g.id:05d} - Generation {gen:05d}")
# Save figure
game_path = get_subfolder(save_path, 'game{id:05d}'.format(id=g.id))
plt.savefig(f'{game_path}blueprint_gen{gen:05d}')
plt.close()
def evaluate_fitness(pop: Population,
hops: float,
weight_range: float,
mutate_combine: bool = False,
mutate_bias: bool = True,
mutate_reset: bool = True,
mutate_update: bool = True,
mutate_candidate: bool = True):
"""Visualize the fitness-values of the population."""
# Initialization
r = int(weight_range / hops)
dim = 2 * r + 1
genome_key = 0
# Enroll the previous best genome
init_gru = pop.best_genome.nodes[2]
best_bias_genome = None
best_reset_genome = None
best_update_genome = None
best_candidate_genome = None
# Create genome-mutations
if mutate_bias:
bias_result = np.zeros((3, dim))
for i in range(3):
for a in range(dim):
g = pop.population[genome_key]
bias_result[i, a] = g.fitness
if best_bias_genome is None or g.fitness > best_bias_genome.fitness:
best_bias_genome = g
genome_key += 1
# Formalize the data
points = [[x, y] for x in range(3) for y in range(dim)]
points_normalized = [[p1, (p2 - r) * hops] for p1, p2 in points]
values = [bias_result[p[0], p[1]] for p in points]
grid_x, grid_y = np.mgrid[0:3:1, -r * hops:(r + 1) * hops:hops]
# Create the figure
plt.figure(figsize=(
10, 2.5)) # Rather horizontal plot due to limited number of rows
knn_data = griddata(points_normalized,
values, (grid_x, grid_y),
method='nearest')
ax = sns.heatmap(
knn_data,
annot=True,
fmt='.3g',
# vmin=0,
# vmax=1,
xticklabels=[round((i - r) * hops, 2) for i in range(dim)],
yticklabels=['r', 'z', 'h'],
cbar_kws={
"pad": 0.02,
"fraction": 0.05
},
)
ax.invert_yaxis()
plt.title('Bias mutation')
plt.xlabel(r'$\Delta bias_h$' + f' (init={list(init_gru.bias_h)!r})')
plt.ylabel(f'bias components')
plt.tight_layout()
path = f"population/storage/{pop.folder_name}/{pop}/"
path = get_subfolder(path, 'images')
path = get_subfolder(path, f'gen{pop.generation:05d}')
plt.savefig(f"{path}bias.png")
plt.close()
if mutate_reset:
reset_result = np.zeros((dim, dim))
for a in range(dim):
for b in range(dim):
g = pop.population[genome_key]
reset_result[a, b] = g.fitness
if best_reset_genome is None or g.fitness > best_reset_genome.fitness:
best_reset_genome = g
genome_key += 1
# Formalize the data
points = [[x, y] for x in range(dim) for y in range(dim)]
points_normalized = [[(p1 - r) * hops, (p2 - r) * hops]
for p1, p2 in points]
values = [reset_result[p[0], p[1]] for p in points]
grid_x, grid_y = np.mgrid[-r * hops:(r + 1) * hops:hops,
-r * hops:(r + 1) * hops:hops]
# Create the figure
plt.figure(
figsize=(15, 15)
) # Rather horizontal plot due to limited number of rows TODO set back to (5, 5)
knn_data = griddata(points_normalized,
values, (grid_x, grid_y),
method='nearest')
ax = sns.heatmap(
knn_data,
annot=True,
fmt='.3g',
# vmin=0,
# vmax=1,
xticklabels=[round((i - r) * hops, 2) for i in range(dim)],
yticklabels=[round((i - r) * hops, 2) for i in range(dim)],
cbar_kws={
"pad": 0.02,
"fraction": 0.05
},
)
ax.invert_yaxis()
plt.title('Reset-gate mutation')
plt.xlabel(r'$\Delta W_{hr}$' +
f' (init={round(init_gru.weight_hh[0, 0], 3)})')
plt.ylabel(r'$\Delta W_{xr}$' +
f' (init={round(init_gru.weight_xh[0, 0], 3)})')
plt.tight_layout()
path = f"population/storage/{pop.folder_name}/{pop}/"
path = get_subfolder(path, 'images')
path = get_subfolder(path, f'gen{pop.generation:05d}')
plt.savefig(f"{path}reset_gate.png")
plt.close()
if mutate_update:
update_result = np.zeros((dim, dim))
for a in range(dim):
for b in range(dim):
g = pop.population[genome_key]
update_result[a, b] = g.fitness
if best_update_genome is None or g.fitness > best_update_genome.fitness:
best_update_genome = g
genome_key += 1
# Formalize the data
points = [[x, y] for x in range(dim) for y in range(dim)]
points_normalized = [[(p1 - r) * hops, (p2 - r) * hops]
for p1, p2 in points]
values = [update_result[p[0], p[1]] for p in points]
grid_x, grid_y = np.mgrid[-r * hops:(r + 1) * hops:hops,
-r * hops:(r + 1) * hops:hops]
# Create the figure
plt.figure(
figsize=(15, 15)
) # Rather horizontal plot due to limited number of rows TODO set back to (5, 5)
knn_data = griddata(points_normalized,
values, (grid_x, grid_y),
method='nearest')
ax = sns.heatmap(
knn_data,
annot=True,
fmt='.3g',
# vmin=0,
# vmax=1,
xticklabels=[round((i - r) * hops, 2) for i in range(dim)],
yticklabels=[round((i - r) * hops, 2) for i in range(dim)],
cbar_kws={
"pad": 0.02,
"fraction": 0.05
},
)
ax.invert_yaxis()
plt.title('Update-gate mutation')
plt.xlabel(r'$\Delta W_{hz}$' +
f' (init={round(init_gru.weight_hh[1, 0], 3)})')
plt.ylabel(r'$\Delta W_{xz}$' +
f' (init={round(init_gru.weight_xh[1, 0], 3)})')
plt.tight_layout()
path = f"population/storage/{pop.folder_name}/{pop}/"
path = get_subfolder(path, 'images')
path = get_subfolder(path, f'gen{pop.generation:05d}')
plt.savefig(f"{path}update_gate.png")
plt.close()
if mutate_candidate:
candidate_result = np.zeros((dim, dim))
for a in range(dim):
for b in range(dim):
g = pop.population[genome_key]
candidate_result[a, b] = g.fitness
if best_candidate_genome is None or g.fitness > best_candidate_genome.fitness:
best_candidate_genome = g
genome_key += 1
# Formalize the data
points = [[x, y] for x in range(dim) for y in range(dim)]
points_normalized = [[(p1 - r) * hops, (p2 - r) * hops]
for p1, p2 in points]
values = [candidate_result[p[0], p[1]] for p in points]
grid_x, grid_y = np.mgrid[-r * hops:(r + 1) * hops:hops,
-r * hops:(r + 1) * hops:hops]
# Create the figure
plt.figure(
figsize=(15, 15)
) # Rather horizontal plot due to limited number of rows TODO set back to (5, 5)
knn_data = griddata(points_normalized,
values, (grid_x, grid_y),
method='nearest')
ax = sns.heatmap(
knn_data,
annot=True,
fmt='.3g',
# vmin=0,
# vmax=1,
xticklabels=[round((i - r) * hops, 2) for i in range(dim)],
yticklabels=[round((i - r) * hops, 2) for i in range(dim)],
cbar_kws={
"pad": 0.02,
"fraction": 0.05
},
)
ax.invert_yaxis()
plt.title('Candidate-state mutation')
plt.xlabel(r'$\Delta W_{hh}$' +
f' (init={round(init_gru.weight_hh[2, 0], 3)})')
plt.ylabel(r'$\Delta W_{xh}$' +
f' (init={round(init_gru.weight_xh[2, 0], 3)})')
plt.tight_layout()
path = f"population/storage/{pop.folder_name}/{pop}/"
path = get_subfolder(path, 'images')
path = get_subfolder(path, f'gen{pop.generation:05d}')
plt.savefig(f"{path}candidate_state.png")
plt.close()
# Set the most fit genome
pop.best_genome.fitness = 0
if mutate_bias and best_bias_genome.fitness > pop.best_genome.fitness:
pop.best_genome = copy.deepcopy(best_bias_genome)
if mutate_reset and best_reset_genome.fitness > pop.best_genome.fitness:
pop.best_genome = copy.deepcopy(best_reset_genome)
if mutate_update and best_update_genome.fitness > pop.best_genome.fitness:
pop.best_genome = copy.deepcopy(best_update_genome)
if mutate_candidate and best_candidate_genome.fitness > pop.best_genome.fitness:
pop.best_genome = copy.deepcopy(best_candidate_genome)
if mutate_combine:
pop.best_genome.nodes[2].bias_h = best_bias_genome.nodes[2].bias_h
pop.best_genome.nodes[2].weight_xh_full[
0, 0] = best_reset_genome.nodes[2].weight_xh_full[0, 0]
pop.best_genome.nodes[2].weight_xh_full[
1, 0] = best_update_genome.nodes[2].weight_xh_full[1, 0]
pop.best_genome.nodes[2].weight_xh_full[
2, 0] = best_candidate_genome.nodes[2].weight_xh_full[2, 0]
pop.best_genome.nodes[2].weight_hh[
0, 0] = best_reset_genome.nodes[2].weight_hh[0, 0]
pop.best_genome.nodes[2].weight_hh[
1, 0] = best_update_genome.nodes[2].weight_hh[1, 0]
pop.best_genome.nodes[2].weight_hh[
2, 0] = best_candidate_genome.nodes[2].weight_hh[2, 0]
def visualize_score(pop: Population, pop_cache: Population, genome: Genome, d: int, range_width: int):
"""Visualize the score of the evaluated population."""
pop_cache.log(f"{pop_cache.name} - Fetching fitness scores...")
dim = (2 * range_width + 1)
pbar = tqdm(range((dim ** 2) * 3), desc="Fetching fitness scores")
genome_key = 0
# Fetching scores of reset-mutations
reset_scores = np.zeros((dim, dim))
for a in range(dim):
for b in range(dim):
reset_scores[a, b] = pop_cache.population[genome_key].fitness
genome_key += 1
pbar.update()
# Fetching scores of update-mutations
update_scores = np.zeros((dim, dim))
for a in range(dim):
for b in range(dim):
update_scores[a, b] = pop_cache.population[genome_key].fitness
genome_key += 1
pbar.update()
# Fetching scores of candidate-mutations
candidate_scores = np.zeros((dim, dim))
for a in range(dim):
for b in range(dim):
candidate_scores[a, b] = pop_cache.population[genome_key].fitness
genome_key += 1
pbar.update()
pbar.close()
# Visualize the result
pop_cache.log(f"{pop_cache.name} - Visualizing the result...")
# GRU-node needed for labels
genome.update_rnn_nodes(config=pop.config.genome)
gru_node = None
for node in genome.nodes.values():
if type(node) == GruNodeGene:
gru_node = node
# Create the points and retrieve data for the plot
points = [[x, y] for x in range(dim) for y in range(dim)]
points_normalized = [[(p1 + -range_width) / d, (p2 + -range_width) / d] for p1, p2 in points]
values_reset = [reset_scores[p[0], p[1]] for p in points]
values_update = [update_scores[p[0], p[1]] for p in points]
values_candidate = [candidate_scores[p[0], p[1]] for p in points]
# K-nearest neighbours with hops of 0.01 is performed
grid_x, grid_y = np.mgrid[-range_width / d:range_width / d:0.01, -range_width / d:range_width / d:0.01]
# Perform k-NN
data_reset = griddata(points_normalized, values_reset, (grid_x, grid_y), method='nearest')
data_update = griddata(points_normalized, values_update, (grid_x, grid_y), method='nearest')
data_candidate = griddata(points_normalized, values_candidate, (grid_x, grid_y), method='nearest')
# Create the plots
plt.figure(figsize=(15, 5))
plt.subplot(131)
plt.imshow(data_reset.T,
vmin=0,
vmax=1,
extent=(-range_width / d, range_width / d, -range_width / d, range_width / d),
origin='lower')
plt.title('Reset-gate mutation')
plt.xlabel(r'$\Delta W_{hr}$' + f' (init={round(gru_node.weight_hh[0, 0], 3)})')
plt.ylabel(r'$\Delta W_{xr}$' + f' (init={round(gru_node.weight_xh[0, 0], 3)})')
plt.subplot(132)
plt.imshow(data_update.T,
vmin=0,
vmax=1,
extent=(-range_width / d, range_width / d, -range_width / d, range_width / d),
origin='lower')
plt.title('Update-gate mutation')
plt.xlabel(r'$\Delta W_{hz}$' + f' (init={round(gru_node.weight_hh[1, 0], 3)})')
plt.ylabel(r'$\Delta W_{xz}$' + f' (init={round(gru_node.weight_xh[1, 0], 3)})')
plt.subplot(133)
plt.imshow(data_candidate.T,
vmin=0,
vmax=1,
extent=(-range_width / d, range_width / d, -range_width / d, range_width / d),
origin='lower')
plt.title('Candidate-state mutation')
plt.xlabel(r'$\Delta W_{hh}$' + f' (init={round(gru_node.weight_hh[2, 0], 3)})')
plt.ylabel(r'$\Delta W_{xh}$' + f' (init={round(gru_node.weight_xh[2, 0], 3)})')
# Store the plot
plt.tight_layout()
path = f"population{'_backup' if pop.use_backup else ''}/storage/{pop.folder_name}/{pop}/"
path = get_subfolder(path, 'images')
path = get_subfolder(path, 'gru_analysis')
plt.savefig(f"{path}{genome.key}.png")
plt.savefig(f"{path}{genome.key}.eps", format="eps")
plt.close()
# Create overview
pop_cache.log("Overview of results:")
log = dict()
# Overview: reset-mutation
max_index_reset, max_value_reset, min_index_reset, min_value_reset = None, 0, None, 1
for index, x in np.ndenumerate(reset_scores):
if x < min_value_reset: min_index_reset, min_value_reset = index, x
if x > max_value_reset: max_index_reset, max_value_reset = index, x
pop_cache.log(f"\tReset-gate mutation:")
pop_cache.log(f"\t > Maximum fitness: {round(max_value_reset, 2)} for index {max_index_reset!r}")
pop_cache.log(f"\t > Average fitness: {round(np.average(reset_scores), 2)}")
pop_cache.log(f"\t > Minimum fitness: {round(min_value_reset, 2)} for index {min_index_reset!r}")
log['Reset-gate maximum fitness'] = f"{round(max_value_reset, 2)} for index {max_index_reset!r}"
log['Reset-gate average fitness'] = f"{round(np.average(reset_scores), 2)}"
log['Reset-gate minimum fitness'] = f"{round(min_value_reset, 2)} for index {min_index_reset!r}"
# Overview: update-mutation
max_index_update, max_value_update, min_index_update, min_value_update = None, 0, None, 1
for index, x in np.ndenumerate(update_scores):
if x < min_value_update: min_index_update, min_value_update = index, x
if x > max_value_update: max_index_update, max_value_update = index, x
pop_cache.log(f"\tUpdate-gate mutation:")
pop_cache.log(f"\t > Maximum fitness: {round(max_value_update, 2)} for index {max_index_update!r}")
pop_cache.log(f"\t > Average fitness: {round(np.average(update_scores), 2)}")
pop_cache.log(f"\t > Minimum fitness: {round(min_value_update, 2)} for index {min_index_update!r}")
log['Update-gate maximum fitness'] = f"{round(max_value_update, 2)} for index {max_index_update!r}"
log['Update-gate average fitness'] = f"{round(np.average(update_scores), 2)}"
log['Update-gate minimum fitness'] = f"{round(min_value_update, 2)} for index {min_index_update!r}"
# Overview: candidate-mutation
max_index_candidate, max_value_candidate, min_index_candidate, min_value_candidate = None, 0, None, 1
for index, x in np.ndenumerate(candidate_scores):
if x < min_value_candidate: min_index_candidate, min_value_candidate = index, x
if x > max_value_candidate: max_index_candidate, max_value_candidate = index, x
pop_cache.log(f"\tCandidate-state mutation:")
pop_cache.log(f"\t > Maximum fitness: {round(max_value_candidate, 2)} for index {max_index_candidate!r}")
pop_cache.log(f"\t > Average fitness: {round(np.average(candidate_scores), 2)}")
pop_cache.log(f"\t > Minimum fitness: {round(min_value_candidate, 2)} for index {min_index_candidate!r}")
log['Candidate-state maximum fitness'] = f"{round(max_value_candidate, 2)} for index {max_index_candidate!r}"
log['Candidate-state average fitness'] = f"{round(np.average(candidate_scores), 2)}"
log['Candidate-state minimum fitness'] = f"{round(min_value_candidate, 2)} for index {min_index_candidate!r}"
update_dict(f'{path}{genome.key}.txt', log)
def evaluate_populations(folder: str, pop_folder: str, max_v: int = 50):
"""
Evaluate the various populations against each other. Note that it is assumed that 'evaluate_generations' has ran
first.
"""
if folder[-1] != '/': folder += '/'
if pop_folder[-1] != '/': pop_folder += '/'
# Load in dummy population
print(
f"\n===> COMBINING POPULATION RESULTS OF FOLDER {folder}{pop_folder} <==="
)
pop = Population(
name=f'{pop_folder}v1',
folder_name=folder,
log_print=False,
use_backup=True,
)
max_gen = pop.generation
# Parse the results
fitness_dict = dict()
finished_dict = dict()
score_dict = dict()
distance_dict = dict()
time_dict = dict()
for g in range(0, max_gen + 1, HOPS):
fitness_dict[g] = []
finished_dict[g] = []
score_dict[g] = []
distance_dict[g] = []
time_dict[g] = []
for v in range(1, max_v + 1):
results: dict = load_dict(
f"population_backup/storage/{folder}{pop_folder}v{v}/evaluation/{g:05d}/results"
)
fitness_dict[g].append(
max([results[k][D_FITNESS] for k in results.keys()]))
finished_dict[g].append(
max([results[k][D_FINISHED] / 100 for k in results.keys()]))
score_dict[g].append(
max([results[k][D_SCORE_AVG] for k in results.keys()]))
distance_dict[g].append(
min([results[k][D_DISTANCE_AVG] for k in results.keys()]))
time_dict[g].append(
min([results[k][D_TIME_AVG] for k in results.keys()]))
# Save received data in evaluation subfolder of the population folder
path = get_subfolder(f'population_backup/storage/{folder}{pop_folder}',
'evaluation')
update_dict(f'{path}fitness', fitness_dict, overwrite=True)
update_dict(f'{path}finished', finished_dict, overwrite=True)
update_dict(f'{path}score', score_dict, overwrite=True)
update_dict(f'{path}distance', distance_dict, overwrite=True)
update_dict(f'{path}time', time_dict, overwrite=True)
# Visualize the data
path_images = get_subfolder(path, 'images')
plot_result(d=fitness_dict,
ylabel="fitness",
title="Average fitness",
save_path=f'{path_images}fitness')
plot_result(d=finished_dict,
ylabel="finished ratio",
title="Averaged finished ratio",
save_path=f'{path_images}finished')
plot_result(d=score_dict,
ylabel="score",
title="Average score",
save_path=f'{path_images}score')
plot_result(d=distance_dict,
ylabel="distance (m)",
title="Average final distance to target",
save_path=f'{path_images}distance')
plot_result(d=time_dict,
ylabel="time (s)",
title="Average simulation time",
save_path=f'{path_images}time')
def compute_complexity(
folder: str,
neat: bool = False,
neat_gru: bool = False,
neat_lstm: bool = False,
neat_sru: bool = False,
neat_sru_s: bool = False,
gen: int = 500,
max_v: int = 50,
):
"""Compute the complexity of the populations' elites."""
# Collect all the populations
populations = []
if neat: populations.append(D_NEAT)
if neat_gru: populations.append(D_NEAT_GRU)
if neat_lstm: populations.append(D_NEAT_LSTM)
if neat_sru: populations.append(D_NEAT_SRU)
if neat_sru_s: populations.append(D_NEAT_SRU_S)
if len(populations) == 0: return
# Go over all possibilities
print(f"\n===> COMPUTING POPULATION'S ELITE COMPLEXITY <===")
path = f"population_backup/storage/{folder}/"
genes_dict = dict()
for pop in populations:
path_eval = get_subfolder(f"{path}{pop}/", 'evaluation')
complexity = Counter()
genes = Counter()
genes_detailed = dict()
for v in range(1, max_v + 1):
population = Population(
name=f'{pop}/v{v}',
folder_name=folder,
use_backup=True,
)
if population.generation == 0:
raise Exception(f"Population {pop}/v{v} loaded incorrectly")
if population.generation != gen: population.load(gen=gen)
s = population.best_genome.size()
complexity[str(s)] += 1
c = str(s[0] + s[1])
genes[c] += 1
if c in genes_detailed:
genes_detailed[c].append(v)
else:
genes_detailed[c] = [v]
# Store results at populations themselves
update_dict(f'{path_eval}complexity_topology',
complexity,
overwrite=True)
update_dict(f'{path_eval}complexity_genes', genes, overwrite=True)
update_dict(f'{path_eval}complexity_genes_detailed',
genes_detailed,
overwrite=True)
# Update global dictionary
keys = list(genes.keys())
for k in keys:
genes[int(k)] = genes[k]
del genes[k]
genes_dict[pop] = list(sorted(genes.items()))
plt.figure(figsize=(10, 2.5))
max_x = max([max([a for a, _ in genes_dict[pop]]) for pop in populations])
min_x = min([min([a for a, _ in genes_dict[pop]]) for pop in populations])
for idx, pop in enumerate(populations):
keys = [a for a, _ in genes_dict[pop]]
for x in range(max_x):
if x not in keys: genes_dict[pop].append((x, 0))
x, y = zip(*genes_dict[pop])
width = 0.8 / len(populations)
plt.bar(x=np.asarray(x) - 0.4 + width / 2 + idx * width,
height=y,
width=width,
linewidth=2,
label=pop,
color=COLORS[pop])
# Beautify the plot
plt.xlim(min_x - .5, max_x + .5)
plt.xticks([i for i in range(min_x, max_x + 1)])
leg = plt.legend(loc='upper center',
bbox_to_anchor=(0.5, 1.18),
fancybox=True,
fontsize=10,
ncol=len(populations))
for line in leg.get_lines():
line.set_linewidth(4.0)
plt.grid(axis='y')
plt.tight_layout()
plt.xlabel("complexity expressed in #genes")
plt.ylabel("#elites")
plt.savefig(f"population_backup/storage/{folder}/images/complexity.png",
bbox_inches='tight',
pad_inches=0.02)
plt.savefig(f"population_backup/storage/{folder}/images/complexity.eps",
format='eps',
bbox_inches='tight',
pad_inches=0.02)
# plt.show()
plt.close()
# Also create a violin plot of the distribution if only two populations
if len(populations) == 2:
max_x = 0
min_x = float('inf')
df = pd.DataFrame()
palette = []
for idx, pop in enumerate(populations):
values = []
for a, b in genes_dict[pop]:
for _ in range(b):
values.append(a)
# Remove outliers
values = sorted(values)
q1 = min(values[int(round(1 / 4 * len(values)))],
values[int(round(3 / 4 * len(values)))])
q3 = max(values[int(round(1 / 4 * len(values)))],
values[int(round(3 / 4 * len(values)))])
iqr = q3 - q1
for i in range(len(values) - 1, -1, -1):
if (values[i] < (q1 - 1.5 * iqr)) or (values[i] >
(q3 + 1.5 * iqr)):
del values[i]
if min(values) < min_x: min_x = min(values)
if max(values) > max_x: max_x = max(values)
df = df.append(
pd.DataFrame({
'complexity': values,
'y': 'ignore',
'pop': pop
}))
palette.append(COLORS[pop])
# Create the plot
plt.figure(figsize=(10, 2.5))
sns.violinplot(data=df,
x="complexity",
y="y",
hue="pop",
palette=palette,
split=True,
inner="quartile")
plt.xlim(min_x - .5, max_x + .5)
plt.xticks([i for i in range(min_x, max_x + 1)])
plt.xlabel("complexity expressed in #genes")
plt.yticks([])
plt.ylabel('elite genome density')
leg = plt.legend(loc='upper center',
bbox_to_anchor=(0.5, 1.25),
fancybox=True,
fontsize=10,
ncol=len(populations))
for line in leg.get_lines():
line.set_linewidth(4.0)
plt.tight_layout()
plt.savefig(
f"population_backup/storage/{folder}/images/complexity_violin.png",
bbox_inches='tight',
pad_inches=0.02)
plt.savefig(
f"population_backup/storage/{folder}/images/complexity_violin.eps",
format='eps',
bbox_inches='tight',
pad_inches=0.02)
plt.show()
plt.close()
def plot_distribution(
folder: str,
neat: bool = False,
neat_gru: bool = False,
neat_lstm: bool = False,
neat_sru: bool = False,
neat_sru_s: bool = False,
gen: int = 500,
):
"""
Plot the one-dimensional distribution of all of the populations on each of the evaluation measures for the requested
generation. It is assumed that the evaluation-data has already been collected.
"""
# Collect all the populations
populations = []
if neat: populations.append(D_NEAT)
if neat_gru: populations.append(D_NEAT_GRU)
if neat_lstm: populations.append(D_NEAT_LSTM)
if neat_sru: populations.append(D_NEAT_SRU)
if neat_sru_s: populations.append(D_NEAT_SRU_S)
if len(populations) == 0: return
# Collect all the measure options
OPTIONS = ['distance', 'finished', 'fitness', 'score', 'time', 'training']
# Go over all possibilities
print(f"\n===> CREATING POPULATION DISTRIBUTIONS <===")
path = f"population_backup/storage/{folder}/"
path_images = get_subfolder(path, 'images')
for option in OPTIONS:
plt.figure(figsize=(10, 2.5))
min_val = float("inf")
max_val = -float("inf")
for pop in populations:
d = load_dict(f"{path}{pop}/evaluation/{option}")
dist = d[str(gen)]
if min(dist) < min_val: min_val = min(dist)
if max(dist) > max_val: max_val = max(dist)
# Remove outliers first
dist = sorted(dist)
q1 = min(dist[int(round(1 / 4 * len(dist)))],
dist[int(round(3 / 4 * len(dist)))])
q3 = max(dist[int(round(1 / 4 * len(dist)))],
dist[int(round(3 / 4 * len(dist)))])
iqr = q3 - q1
for i in range(len(dist) - 1, -1, -1):
if (dist[i] < (q1 - 1.5 * iqr)) or (dist[i] >
(q3 + 1.5 * iqr)):
del dist[i]
sns.distplot(
dist,
hist=False,
kde=True,
norm_hist=True,
bins=100,
color=COLORS[pop],
kde_kws={'linewidth': 2},
label=pop,
)
plt.xlim(min_val, max_val)
# plt.title(f"Probability density across populations for '{option}' at generation {gen}")
plt.xlabel(option)
# plt.yticks([])
plt.ylabel('probability density')
leg = plt.legend(loc='upper center',
bbox_to_anchor=(0.5, 1.2),
fancybox=True,
fontsize=8,
ncol=len(populations))
for line in leg.get_lines():
line.set_linewidth(4.0)
plt.tight_layout()
plt.savefig(f"{path_images}dist_{option}.png",
bbox_inches='tight',
pad_inches=0.02)
plt.savefig(f"{path_images}dist_{option}.eps",
format='eps',
bbox_inches='tight',
pad_inches=0.02)
# plt.show()
plt.close()
def main(population: Population,
game_id: int,
genome: Genome = None,
game_cfg: Config = None,
average: int = 1,
debug: bool = False):
"""
Monitor the genome on the following elements:
* Position
* Hidden state of SRU (Ht)
* Actuation of both wheels
* Distance
* Delta distance
"""
# Make sure all parameters are set
if not genome: genome = population.best_genome
if not game_cfg: game_cfg = pop.config
# Check if valid genome (contains at least one hidden SRU, first SRU is monitored) - also possible for fixed RNNs
a = len([n for n in genome.get_used_nodes().values() if type(n) == SimpleRnnNodeGene]) >= 1
b = len([n for n in genome.get_used_nodes().values() if type(n) == FixedRnnNodeGene]) >= 1
assert a or b
# Get the game
game = get_game(game_id, cfg=game_cfg, noise=False)
state = game.reset()[D_SENSOR_LIST]
step_num = 0
# Create the network
net = make_net(genome=genome,
genome_config=population.config.genome,
batch_size=1,
initial_read=state,
)
# Containers to monitor
actuation = []
distance = []
delta_distance = []
position = []
Ht = []
target_found = []
score = 0
# Initialize the containers
actuation.append([0, 0])
distance.append(state[0])
delta_distance.append(0)
position.append(game.player.pos.get_tuple())
Ht.append(net.rnn_state[0, 0, 0])
if debug:
print(f"Step: {step_num}")
print(f"\t> Actuation: {(round(actuation[-1][0], 5), round(actuation[-1][1], 5))!r}")
print(f"\t> Distance: {round(distance[-1], 5)} - Delta distance: {round(delta_distance[-1], 5)}")
print(f"\t> Position: {(round(position[-1][0], 2), round(position[-1][1], 2))!r}")
print(f"\t> SRU state: Ht={round(Ht[-1], 5)}")
# Start monitoring
while True:
# Check if maximum iterations is reached
if step_num == game_cfg.game.duration * game_cfg.game.fps: break
# Determine the actions made by the agent for each of the states
action = net(np.asarray([state]))
# Check if each game received an action
assert len(action) == 1
# Proceed the game with one step, based on the predicted action
obs = game.step(l=action[0][0], r=action[0][1])
finished = obs[D_DONE]
# Update the score-count
if game.score > score:
target_found.append(step_num)
score = game.score
# Update the candidate's current state
state = obs[D_SENSOR_LIST]
# Stop if agent reached target in all the games
if finished: break
step_num += 1
# Update the containers
actuation.append(action[0])
distance.append(state[0])
delta_distance.append(distance[-2] - distance[-1])
position.append(game.player.pos.get_tuple())
Ht.append(net.rnn_state[0, 0, 0])
if debug:
print(f"Step: {step_num}")
print(f"\t> Actuation: {(round(actuation[-1][0], 5), round(actuation[-1][1], 5))!r}")
print(f"\t> Distance: {round(distance[-1], 5)} - Delta distance: {round(delta_distance[-1], 5)}")
print(f"\t> Position: {(round(position[-1][0], 2), round(position[-1][1], 2))!r}")
print(f"\t> SRU state: Ht={round(Ht[-1], 5)}")
if average > 1:
# Average out the noise
x, y = zip(*actuation)
x = SMA(x, window=average)
y = SMA(y, window=average)
actuation = list(zip(x, y))
distance = SMA(distance, window=average)
delta_distance = SMA(delta_distance, window=average)
Ht = SMA(Ht, window=average)
# Resolve weird artifacts at the beginning
for i in range(average, 0, -1):
actuation[i - 1] = actuation[i]
distance[i - 1] = distance[i]
delta_distance[i - 1] = delta_distance[i]
Ht[i - 1] = Ht[i]
# Visualize the monitored values
path = get_subfolder(f"population{'_backup' if population.use_backup else ''}/"
f"storage/"
f"{population.folder_name}/"
f"{population}/", "images")
path = get_subfolder(path, f"monitor")
path = get_subfolder(path, f"{genome.key}")
path = get_subfolder(path, f"{game_id}")
visualize_actuation(actuation,
target_found=target_found,
game_cfg=game_cfg.game,
save_path=f"{path}actuation.png")
visualize_distance(distance,
target_found=target_found,
game_cfg=game_cfg.game,
save_path=f"{path}distance.png")
visualize_hidden_state(Ht,
target_found=target_found,
game_cfg=game_cfg.game,
save_path=f"{path}hidden_state.png")
visualize_position(position,
game=game,
save_path=f"{path}trace.png")
merge(f"Monitored genome={genome.key} on game={game.id}", path=path)
def main(population: Population,
game_id: int,
genome: Genome = None,
game_cfg: Config = None,
debug: bool = False):
"""
Monitor the genome on the following elements:
* Position
* Update gate (Zt)
* Hidden state of GRU (Ht)
* Actuation of both wheels
* Distance
"""
# Make sure all parameters are set
if not genome: genome = population.best_genome
if not game_cfg: game_cfg = pop.config
# Check if valid genome (contains at least one hidden GRU, first GRU is monitored)
assert len([
n for n in genome.get_used_nodes().values()
if type(n) == GruNoResetNodeGene
]) >= 1
# Get the game
game = get_game(game_id, cfg=game_cfg, noise=False)
state = game.reset()[D_SENSOR_LIST]
step_num = 0
# Create the network
net = make_net(
genome=genome,
genome_config=population.config.genome,
batch_size=1,
initial_read=state,
)
# Containers to monitor
actuation = []
distance = []
position = []
Ht = []
Ht_tilde = []
Zt = []
target_found = []
score = 0
# Initialize the containers
actuation.append([0, 0])
distance.append(state[0])
position.append(game.player.pos.get_tuple())
ht, ht_tilde, zt = get_gru_states(net=net, x=np.asarray([state]))
Ht.append(ht)
Ht_tilde.append(ht_tilde)
Zt.append(zt)
if debug:
print(f"Step: {step_num}")
print(
f"\t> Actuation: {(round(actuation[-1][0], 5), round(actuation[-1][1], 5))!r}"
)
print(f"\t> Distance: {round(distance[-1], 5)}")
print(
f"\t> Position: {(round(position[-1][0], 2), round(position[-1][1], 2))!r}"
)
print(f"\t> GRU states: "
f"\t\tHt={round(Ht[-1], 5)}"
f"\t\tHt_tilde={round(Ht_tilde[-1], 5)}"
f"\t\tZt={round(Zt[-1], 5)}")
# Start monitoring
while True:
# Check if maximum iterations is reached
if step_num == game_cfg.game.duration * game_cfg.game.fps: break
# Determine the actions made by the agent for each of the states
action = net(np.asarray([state]))
# Check if each game received an action
assert len(action) == 1
# Proceed the game with one step, based on the predicted action
obs = game.step(l=action[0][0], r=action[0][1])
finished = obs[D_DONE]
# Update the score-count
if game.score > score:
target_found.append(step_num)
score = game.score
# Update the candidate's current state
state = obs[D_SENSOR_LIST]
# Stop if agent reached target in all the games
if finished: break
step_num += 1
# Update the containers
actuation.append(action[0])
distance.append(state[0])
position.append(game.player.pos.get_tuple())
ht, ht_tilde, zt = get_gru_states(net=net, x=np.asarray([state]))
Ht.append(ht)
Ht_tilde.append(ht_tilde)
Zt.append(zt)
if debug:
print(f"Step: {step_num}")
print(
f"\t> Actuation: {(round(actuation[-1][0], 5), round(actuation[-1][1], 5))!r}"
)
print(f"\t> Distance: {round(distance[-1], 5)}")
print(
f"\t> Position: {(round(position[-1][0], 2), round(position[-1][1], 2))!r}"
)
print(f"\t> GRU states: "
f"\t\tHt={round(Ht[-1], 5)}"
f"\t\tHt_tilde={round(Ht_tilde[-1], 5)}"
f"\t\tZt={round(Zt[-1], 5)}")
# Visualize the monitored values
path = get_subfolder(
f"population{'_backup' if population.use_backup else ''}/"
f"storage/"
f"{population.folder_name}/"
f"{population}/", "images")
path = get_subfolder(path, f"monitor")
path = get_subfolder(path, f"{genome.key}")
path = get_subfolder(path, f"{game_id}")
visualize_actuation(actuation,
target_found=target_found,
game_cfg=game_cfg.game,
save_path=f"{path}actuation.png")
visualize_distance(distance,
target_found=target_found,
game_cfg=game_cfg.game,
save_path=f"{path}distance.png")
visualize_hidden_state(Ht,
target_found=target_found,
game_cfg=game_cfg.game,
save_path=f"{path}hidden_state.png")
visualize_candidate_hidden_state(
Ht_tilde,
target_found=target_found,
game_cfg=game_cfg.game,
save_path=f"{path}candidate_hidden_state.png")
visualize_update_gate(Zt,
target_found=target_found,
game_cfg=game_cfg.game,
save_path=f"{path}update_gate.png")
visualize_position(position, game=game, save_path=f"{path}trace.png")
merge(f"Monitored genome={genome.key} on game={game.id}", path=path)
def combine_all_populations(
folder: str,
max_v: int = None,
neat: bool = False,
neat_gru: bool = False,
neat_lstm: bool = False,
neat_sru: bool = False,
neat_sru_s: bool = False,
):
"""Combine the scores for all of the populations in a given folder."""
# Collect all the populations
populations = []
if neat: populations.append(D_NEAT)
if neat_gru: populations.append(D_NEAT_GRU)
if neat_lstm: populations.append(D_NEAT_LSTM)
if neat_sru: populations.append(D_NEAT_SRU)
if neat_sru_s: populations.append(D_NEAT_SRU_S)
if len(populations) == 0: return
# Collect all the measure options
OPTIONS = ['distance', 'finished', 'fitness', 'score', 'time', 'training']
# OPTIONS = ['fitness']
# Go over all possibilities
print(f"\n===> COMBINING POPULATIONS OF FOLDER {folder} <===")
path = f"population_backup/storage/{folder}/"
path_images = get_subfolder(path, 'images')
for option in OPTIONS:
plt.figure(figsize=(8, 2.5))
max_data = 0
max_gen = 0
for pop in populations:
# Load the dictionary
d = load_dict(f"{path}{pop}/evaluation/{option}")
size = len(list(d.values())[0])
if max_v: assert size == max_v
# Prepare the data containers
q1 = []
q2 = [] # Median
q3 = []
idx_q1 = int(round(1 / 4 * size))
idx_q2 = int(round(2 / 4 * size))
idx_q3 = int(round(3 / 4 * size))
# Loop over each iteration
x = sorted([int(k) for k in d.keys()])
for g in x:
if g > max_gen: max_gen = g
lst = sorted(d[str(g)]) # Sort values from low to high
q1.append(lst[idx_q1])
q2.append(lst[idx_q2])
q3.append(lst[idx_q3])
# Plot the results
plt.plot(x, q1, color=COLORS[pop], linestyle=":", linewidth=.5)
plt.plot(x, q3, color=COLORS[pop], linestyle=":", linewidth=.5)
plt.plot(x,
q2,
color=COLORS[pop],
linestyle="-",
linewidth=2,
label=pop)
plt.fill_between(x, q1, q3, color=COLORS[pop], alpha=0.2)
# Update the max-counter
if max(q3) > max_data: max_data = max(q3)
# Finalize the figure
leg = plt.legend(loc='upper center',
bbox_to_anchor=(0.5, 1.25),
fancybox=True,
fontsize=10,
ncol=len(populations))
for line in leg.get_lines():
line.set_linewidth(4.0)
# plt.xticks([i * 100 for i in range(11)]) # TODO
plt.xlabel("generation")
plt.xlim(0, max_gen)
# plt.yticks([i for i in range(7)]) # TODO
plt.ylabel(option)
plt.ylim(0, max(max_data * 1.05, 1.05))
# plt.ylim(0, 6) # TODO
plt.grid()
plt.tight_layout()
plt.savefig(f"{path_images}comb_{option}.png",
bbox_inches='tight',
pad_inches=0.02,
dpi=500)
# plt.savefig(f"{path_images}comb_{option}.eps", format="eps", bbox_inches='tight', pad_inches=0.02)
# plt.show()
plt.close()
def visualize_generations(name, experiment_id, folder=None, hops: int = 10):
"""Visualize the result of the 'evaluate_generations' script. Should only be used for debugging!"""
# Fetch population and evaluation games
folder = folder if folder else get_folder(experiment_id)
pop = Population(
name=name,
folder_name=folder,
log_print=False,
use_backup=True,
)
# Parse the results
fitness_dict = dict()
finished_dict = dict()
score_dict = dict()
distance_dict = dict()
time_dict = dict()
max_gen = pop.generation
for gen in tqdm(range(0, max_gen + 1, hops)):
results: dict = load_dict(
f"population{'_backup' if pop.use_backup else ''}/"
f"storage/"
f"{pop.folder_name}/"
f"{pop}/"
f"evaluation/"
f"{gen:05d}/"
f"results")
# Fitness
fitness = [results[k][D_FITNESS] for k in results.keys()]
fitness_dict[gen] = fitness
# Finished
finished = [results[k][D_FINISHED] / 100
for k in results.keys()] # Normalize to percentage (0..1)
finished_dict[gen] = finished
# Score
score = [results[k][D_SCORE_AVG] for k in results.keys()]
score_dict[gen] = score
# Distance
distance = [results[k][D_DISTANCE_AVG] for k in results.keys()]
distance_dict[gen] = distance
# Time
time = [results[k][D_TIME_AVG] for k in results.keys()]
time_dict[gen] = time
# Create visualizations for each of the results
sf = get_subfolder(
f"population{'_backup' if pop.use_backup else ''}/"
f"storage/"
f"{pop.folder_name}/"
f"{pop}/"
f"images/", 'evaluation')
plot_population(fitness_dict,
ylabel="Fitness score",
title="Fitness (higher is better)",
save_path=f'{sf}fitness_group.png')
plot_elite(fitness_dict,
f=max,
ylabel="Fitness score",
title="Fitness of population's elite (higher is better)",
save_path=f'{sf}fitness_elite.png')
plot_population(finished_dict,
ylabel="Percentage finished (%)",
title="Percentage finished (higher is better)",
save_path=f'{sf}finished_group.png')
plot_elite(
finished_dict,
f=max,
ylabel="Percentage finished (%)",
title="Percentage finished by population's elite (higher is better)",
save_path=f'{sf}finished_elite.png')
plot_population(score_dict,
ylabel="Score",
title="Final scores (higher is better)",
save_path=f'{sf}score_group.png')
plot_elite(score_dict,
f=max,
ylabel="Score",
title="Final score of population's elite (higher is better)",
save_path=f'{sf}score_elite.png')
plot_population(distance_dict,
ylabel="Distance (m)",
title="Distance from target (lower is better)",
save_path=f'{sf}distance_group.png')
plot_elite(
distance_dict,
f=min,
ylabel="Distance (m)",
title="Distance from target for population's elite (lower is better)",
save_path=f'{sf}distance_elite.png')
plot_population(time_dict,
ylabel="Time (s)",
title="Time needed to reach target (lower is better)",
save_path=f'{sf}time_group.png')
plot_elite(
time_dict,
f=min,
ylabel="Time (s)",
title=
"Time needed to reach target for population's elite (lower is better)",
save_path=f'{sf}time_elite.png')
