def _process_scores(self, score_per_generation_per_run):
score_means = []
score_stds = []
for index in range(len(score_per_generation_per_run[0])):
column = [row[index] for row in score_per_generation_per_run]
score_means.append(round_value(numpy.mean(column)))
score_stds.append(round_value(numpy.std(column)))
return score_means, score_stds
def _generate_overall_metrics_output(self, run_infos):
msg = "\n\n\n#################### OVERALL RESULTS ####################"
msg += "\n\n\n##### BEST TEAM METRICS"
score_per_run = []
for run in run_infos:
score_per_run.append(round_value(run.best_team.score_testset_))
self.best_scores_per_runs_ = score_per_run
msg += "\n\nBest Team Validation Score per Run: "+str(score_per_run)
msg += "\nmean: "+str(round_value(numpy.mean(score_per_run)))
msg += "\nstd. deviation: "+str(round_value(numpy.std(score_per_run)))
scores = [run.best_team.score_testset_ for run in run_infos]
best_run = run_infos[scores.index(max(scores))]
msg += "\nbest run: "+str(best_run.run_id)
score_means, score_stds = self._process_scores([run.train_score_per_validation for run in run_infos])
msg += "\n\nBest Team Train Score per Validation across Runs:"
msg += "\nmean: "+str(score_means)
msg += "\nstd. deviation: "+str(score_stds)
score_means, score_stds = self._process_scores([run.test_score_per_validation for run in run_infos])
msg += "\n\nBest Team Validation Score per Validation across Runs:"
msg += "\nmean: "+str(score_means)
msg += "\nstd. deviation: "+str(score_stds)
msg += "\n\n\n##### GLOBAL METRICS"
final_scores = [run.global_mean_validation_score_per_validation[-1] for run in run_infos]
msg += "\n\nGlobal Validation Score per Run: "+str(final_scores)
msg += "\nmean: "+str(round_value(numpy.mean(final_scores)))
msg += "\nstd. deviation: "+str(round_value(numpy.std(final_scores)))
best_run = run_infos[final_scores.index(max(final_scores))]
msg += "\nbest run: "+str(best_run.run_id)
score_means, score_stds = self._process_scores([run.global_mean_fitness_score_per_validation for run in run_infos])
msg += "\n\nGlobal Train Score per Validation across Runs:"
msg += "\nmean: "+str(score_means)
msg += "\nstd. deviation: "+str(score_stds)
score_means, score_stds = self._process_scores([run.global_mean_validation_score_per_validation for run in run_infos])
msg += "\n\nGlobal Validation Score per Validation across Runs:"
msg += "\nmean: "+str(score_means)
msg += "\nstd. deviation: "+str(score_stds)
for key in Config.RESTRICTIONS['used_diversities']:
score_means, score_stds = self._process_scores([run.global_diversity_per_validation[key] for run in run_infos])
msg += "\n\nMean Diversity per Validation across Runs ("+str(key)+"):"
msg += "\nmean: "+str(score_means)
msg += "\nstd. deviation: "+str(score_stds)
elapseds_per_run = [run.elapsed_time for run in run_infos]
msg += "\n\nFinished execution, total elapsed time: "+str(round_value(sum(elapseds_per_run)))+" mins "
msg += "(mean: "+str(round_value(numpy.mean(elapseds_per_run)))+", std: "+str(round_value(numpy.std(elapseds_per_run)))+")"
return msg
def _store_per_generation_metrics(self, run_info, teams_population):
older_teams = [team for team in teams_population if team.generation != self.current_generation_]
generation_info = []
for team in older_teams:
team_info = []
team_info.append(round_value(team.fitness_, round_decimals_to = 3))
team_info.append(round_value(team.score_testset_, round_decimals_to = 3))
for diversity in Config.RESTRICTIONS['used_diversities']:
if diversity in team.diversity_:
value = round_value(team.diversity_[diversity], round_decimals_to = 3)
else:
value = 0.0
team_info.append(value)
generation_info.append(team_info)
run_info.info_per_team_per_generation.append(generation_info)
mean_fitness = round_value(numpy.mean([team.fitness_ for team in older_teams]), 3)
run_info.global_mean_fitness_per_generation.append(mean_fitness)
run_info.global_max_fitness_per_generation.append(round_value(max([team.fitness_ for team in older_teams])))
for diversity in Config.RESTRICTIONS['used_diversities']:
run_info.global_diversity_per_generation[diversity].append(round_value(numpy.mean([t.diversity_[diversity] for t in older_teams]), 3))
if len(Config.RESTRICTIONS['used_diversities']) > 1 and self.selection.previous_diversity_:
run_info.global_fitness_per_diversity_per_generation[self.selection.previous_diversity_].append(mean_fitness)
run_info.novelty_type_per_generation.append(Config.RESTRICTIONS['used_diversities'].index(self.selection.previous_diversity_))
def run(self):
print "\n### Starting pSBB"
# initialize the environment and the selection algorithm
self.environment = self._initialize_environment()
self.selection = Selection(self.environment)
overall_info = ""
overall_info += "\n### CONFIG: "+str(Config.USER)+"\n"
overall_info += "\n### RESTRICTIONS: "+str(Config.RESTRICTIONS)+"\n"
overall_info += self.environment.metrics()
overall_info += "\nSeeds per run: "+str(self.seeds_per_run_)
overall_info += "\nDiversities: "+str(Config.RESTRICTIONS['used_diversities'])
print overall_info
run_infos = []
for run_id in range(Config.USER['training_parameters']['runs_total']):
start_time = time.time()
run_info = RunInfo(run_id+1, self.seeds_per_run_[run_id])
print "\nStarting run: "+str(run_info.run_id)
self._set_seed(run_info.seed)
# randomly initialize populations
self.current_generation_ = 0
teams_population, programs_population = self._initialize_populations()
self.environment.reset()
while not self._stop_criterion():
self.current_generation_ += 1
if self.current_generation_ == 1 or self.current_generation_ % Config.USER['training_parameters']['validate_after_each_generation'] == 0:
validation = True
else:
validation = False
# selection
teams_population, programs_population, pareto_front = self.selection.run(self.current_generation_, teams_population, programs_population, validation)
# final pruning
if self._stop_criterion():
older_teams = [team for team in teams_population if team.generation != self.current_generation_]
for team in older_teams:
team.prune_total()
# validation
if not validation:
print ".",
sys.stdout.flush()
self._store_per_generation_metrics(run_info, teams_population)
else:
best_team = self.environment.validate(self.current_generation_, teams_population)
self._store_per_generation_metrics(run_info, teams_population)
self._print_and_store_per_validation_metrics(run_info, best_team, teams_population, programs_population)
self._store_per_run_metrics(run_info, best_team, teams_population, pareto_front)
run_info.elapsed_time = round_value((time.time() - start_time)/60.0)
print("\nFinished run "+str(run_info.run_id)+", elapsed time: "+str(run_info.elapsed_time)+" mins")
run_infos.append(run_info)
sys.stdout.flush()
# finalize execution (get final metrics, print to output, print to file)
overall_info += self._generate_overall_metrics_output(run_infos)
print overall_info
sys.stdout.flush()
if Config.RESTRICTIONS['write_output_files']:
self.filepath_ = self._create_folder()
self._write_output_files(run_infos, overall_info)
self._save_teams_data_per_generation(run_infos)
def _print_and_store_per_validation_metrics(self, run_info, best_team, teams_population, programs_population):
print "\n\n>>>>> Generation: "+str(self.current_generation_)+", run: "+str(run_info.run_id)
run_info.train_score_per_validation.append(best_team.fitness_)
run_info.test_score_per_validation.append(best_team.score_testset_)
run_info.recall_per_validation.append(best_team.extra_metrics_['recall_per_action'])
print("\n### Best Team Metrics: "+best_team.metrics()+"\n")
older_teams = [team for team in teams_population if team.generation != self.current_generation_]
fitness_score_mean = round_value(numpy.mean([team.fitness_ for team in older_teams]))
validation_score_mean = round_value(numpy.mean([team.score_testset_ for team in older_teams]))
run_info.global_mean_validation_score_per_validation.append(validation_score_mean)
print
for key in best_team.diversity_:
run_info.global_diversity_per_validation[key].append(run_info.global_diversity_per_generation[key][-1])
print str(key)+": "+str(best_team.diversity_[key])+" (global: "+str(run_info.global_diversity_per_generation[key][-1])+")"
print "\n### Global Metrics:"
run_info.global_mean_fitness_score_per_validation.append(fitness_score_mean)
run_info.global_max_fitness_score_per_validation.append(round_value(max([team.fitness_ for team in older_teams])))
print "\nfitness (global): "+str(fitness_score_mean)
actions_distribution = Counter([p.action for p in programs_population])
print "\nactions distribution: "+str(actions_distribution)
actions_distribution_array = []
for action in range(Config.RESTRICTIONS['total_actions']):
if action in actions_distribution:
actions_distribution_array.append(actions_distribution[action])
else:
actions_distribution_array.append(0)
run_info.actions_distribution_per_validation.append(actions_distribution_array)
inputs_distribution_per_instruction = Counter()
inputs_distribution_per_team = Counter()
for team in older_teams:
inputs_distribution_per_instruction.update(team.inputs_distribution())
inputs_distribution_per_team.update(list(team.inputs_distribution()))
inputs_distribution_per_instruction_array = []
inputs_distribution_per_team_array = []
for value in range(Config.RESTRICTIONS['total_inputs']):
if value in inputs_distribution_per_instruction:
inputs_distribution_per_instruction_array.append(inputs_distribution_per_instruction[value])
else:
inputs_distribution_per_instruction_array.append(0)
if value in inputs_distribution_per_team:
inputs_distribution_per_team_array.append(inputs_distribution_per_team[value])
else:
inputs_distribution_per_team_array.append(0)
print "inputs distribution (global, per instruction): "+str(inputs_distribution_per_instruction_array)
print "inputs distribution (global, per team): "+str(inputs_distribution_per_team_array)
run_info.inputs_distribution_per_instruction_per_validation.append(inputs_distribution_per_instruction_array)
run_info.inputs_distribution_per_team_per_validation.append(inputs_distribution_per_team_array)
print
print "Global Fitness (last 10 gen.): "+str(run_info.global_mean_fitness_per_generation[-10:])
if len(Config.RESTRICTIONS['used_diversities']) > 0:
print "Global Diversity (last 10 gen.):"
for diversity in Config.RESTRICTIONS['used_diversities']:
print "- "+str(diversity)+": "+str(run_info.global_diversity_per_generation[diversity][-10:])
if len(Config.RESTRICTIONS['used_diversities']) > 1:
print "Diversity Type (last 10 gen.): "+str(run_info.novelty_type_per_generation[-10:])
avg_team_size = round_value(numpy.mean([len(team.programs) for team in older_teams]))
avg_program_with_intros_size = round_value(numpy.mean(flatten([[len(program.instructions) for program in team.programs] for team in older_teams])))
avg_program_without_intros_size = round_value(numpy.mean(flatten([[len(program.instructions_without_introns_) for program in team.programs] for team in older_teams])))
run_info.mean_team_size_per_validation.append(avg_team_size)
run_info.mean_program_size_with_introns_per_validation.append(avg_program_with_intros_size)
run_info.mean_program_size_without_introns_per_validation.append(avg_program_without_intros_size)
print "\nMean Team Sizes: "+str(run_info.mean_team_size_per_validation[-10:])
print "Mean Program Sizes (with introns): "+str(run_info.mean_program_size_with_introns_per_validation[-10:])
print "Mean Program Sizes (without introns): "+str(run_info.mean_program_size_without_introns_per_validation[-10:])
print "\n<<<<< Generation: "+str(self.current_generation_)+", run: "+str(run_info.run_id)
def run(self):
print "\n### Starting pSBB"
initial_info = self._generate_initial_message_output()
print initial_info
for run_id in range(Config.USER['training_parameters']['runs_total']):
run_info = RunInfo(run_id+1, self.environment_, self.seeds_per_run_[run_id])
print "\nStarting run: "+str(run_info.run_id)
self._set_seed(run_info.seed)
if Config.USER['advanced_training_parameters']['second_layer']['enabled']:
self._initialize_actions(run_info)
self.current_generation_ = 0
teams_population, programs_population = self._initialize_populations()
self.environment_.reset()
while not self._stop_criterion():
self.current_generation_ += 1
validation = False
if self._is_validation():
validation = True
teams_population, programs_population, pareto_front = self.selection_.run(
self.current_generation_, teams_population, programs_population)
if self._stop_criterion():
older_teams = [team for team in teams_population if team.generation != self.current_generation_]
for team in older_teams:
team.prune_total()
self.environment_.metrics_.store_per_generation_metrics(run_info, teams_population,
self.current_generation_, self.selection_.previous_diversity_)
if not validation:
print ".",
sys.stdout.flush()
else:
best_team = self.environment_.validate(self.current_generation_, teams_population)
self.environment_.metrics_.store_per_validation_metrics(run_info, best_team,
teams_population, programs_population, self.current_generation_)
print "\n\n>>>>> Generation: "+str(self.current_generation_)+", run: "+str(run_info.run_id)
self.environment_.metrics_.print_per_validation_metrics(run_info, best_team)
print "\n<<<<< Generation: "+str(self.current_generation_)+", run: "+str(run_info.run_id)
self.environment_.metrics_.store_per_run_metrics(run_info, best_team, teams_population, pareto_front,
self.current_generation_)
run_info.end()
print("\nFinished run "+str(run_info.run_id)+", elapsed time: "+str(run_info.elapsed_time_)+" mins")
self.run_infos_.append(run_info)
sys.stdout.flush()
# finalize execution (get final metrics, print to output, print to file)
msg = self.environment_.metrics_.generate_overall_metrics_output(self.run_infos_)
elapseds_per_run = [run.elapsed_time_ for run in self.run_infos_]
msg += "\n\nFinished execution, total elapsed time: "+str(round_value(sum(elapseds_per_run)))+" mins "
msg += "(mean: "+str(round_value(numpy.mean(elapseds_per_run)))+", std: "+str(round_value(numpy.std(elapseds_per_run)))+")"
initial_info += msg
self.best_scores_per_runs_ = [round_value(run.best_team_.score_champion_) for run in self.run_infos_]
print initial_info
sys.stdout.flush()
if Config.RESTRICTIONS['write_output_files']:
self._write_output_files(initial_info)