def _calculate_solution_value(self, solution, data_set, learner):
X = get_input_variables(data_set).as_matrix()
target = get_target_variable(data_set).as_matrix()
neural_network = FeedForwardNetwork.create(solution,
learner.configuration)
prediction = self._predict_neural_network(neural_network, X)
return self.metric.evaluate(prediction, target)
def phenotype(self,
genome: neat.DefaultGenome) -> neat.nn.FeedForwardNetwork:
"""
Creates and returns the neural network phenotype of the given genome.
:param genome: The genome encoding the network.
"""
return FeedForwardNetwork.create(genome, self.config)
def test_unconnected():
# Unconnected network with no inputs and one output neuron.
node_evals = [(0, activations.sigmoid_activation, sum, 0.0, 1.0, [])]
r = FeedForwardNetwork([], [0], node_evals)
assert r.values[0] == 0.0
result = r.activate([])
assert_almost_equal(r.values[0], 0.5, 0.001)
assert result[0] == r.values[0]
result = r.activate([])
assert_almost_equal(r.values[0], 0.5, 0.001)
assert result[0] == r.values[0]
def evaluate(self, pool_workers, genome, config):
if self.network_type == 'ffnn':
net = SimpleFeedForwardNetwork.create(genome, config)
elif self.network_type == 'cppn':
cppn_network = FeedForwardNetwork.create(genome, config)
net = PhenotypeNetwork.create(cppn_network, self.substrate)
jobs = []
for i in range(self.n_games):
# Create puzzles of varying difficulties
error_rate = self.error_rates[i % len(self.error_rates)]
# Determine the initial random error configuration
seed = self.sample_seeds[i]
jobs.append(
pool_workers.apply_async(self.get_fitness,
(net, error_rate, seed)))
fitness = {err: 0 for err in self.error_rates}
for job in jobs:
res, error_rate = job.get(timeout=None)
fitness[error_rate] += res
total_fitness = sum(fitness.values()) / self.n_games
details = [
fitness[err] * len(self.error_rates) / self.n_games
for err in self.error_rates
]
return total_fitness, details
def get_command(net: nn.FeedForwardNetwork, distance: int, size: int,
speed: int) -> str:
value = net.activate([distance, size, speed])[0]
# print('activation: {:.4f}'.format(value), end='\r')
if value >= 0.5:
return 'up'
return ''
def get(self, genome, config, puzzles_proportions):
if self.network_type == 'ffnn':
net = SimpleFeedForwardNetwork.create(genome, config)
elif self.network_type == 'cppn':
cppn_net = FeedForwardNetwork.create(genome, config)
net = PhenotypeNetwork.create(cppn_net, self.substrate)
fitness = {error_rate: 0 for error_rate in self.error_rates}
n_puzzles = {
error_rate: int(self.n_games * puzzles_proportions[error_rate])
for error_rate in self.error_rates
}
fail_count = {error_rate: 0 for error_rate in self.error_rates}
for error_rate in self.error_rates:
for i in range(n_puzzles[error_rate]):
result = self.game.play(net, error_rate, self.error_mode,
self.reward_mode,
GameMode["TRAINING"])["fitness"]
fitness[error_rate] += result
# Count the number of fails for the resampling learner
fail_count[error_rate] += 1 - result
return sum(fitness.values()) / len(
self.error_rates) / self.n_games, fail_count
def post_evaluate(self, config, population, species, best_genome):
if self._best_net is None or best_genome.fitness > self._best_fitness:
self._best_net = FeedForwardNetwork.create(best_genome, config)
self._best_fitness = best_genome.fitness
if self._best_ever is None or self._best_fitness > self._best_ever:
self._best_ever = self._best_fitness
print("Best fitness so far in this cycle", self._best_fitness,
", Best fitness ever", self._best_ever,
", Currently used agent:",
"random" if self._last_fitness is None else self._last_fitness)
self._generations += 1
if self._generations >= self._reset_number:
is_random = self._last_fitness is not None and self._best_fitness < self._last_fitness
print(
"Resetting opponent, last fitness was",
"random" if self._last_fitness is None else self._last_fitness,
"new fitness is",
"random" if is_random else self._best_fitness)
if is_random:
self._current_opponent = RandomAgent()
self._last_fitness = None
else:
self._current_opponent = self._best_net
self._last_fitness = self._best_fitness
self._best_fitness = None
self._best_net = None
if self._save_opponents:
with open('opponent-net-{}.pkl'.format(self._last_fitness),
'wb') as output:
pickle.dump(self._current_opponent, output, 1)
self._generations = 0
def save_genome(self, genome):
name = self.make_name(genome)
net = FeedForwardNetwork.create(genome, self)
pickle.dump((net, self.env_name), open('models/%s.dat' % name, 'wb'))
_GymNeatConfig.draw_net(net, 'visuals/%s-network' % name,
self.node_names)
def post_evaluate(self, config, population, species, best_genome):
if self._best_id is not None and self._best_id not in population:
self._best_net = None
self._best_id = None
self._best_fitness = None
self._generation_count += 1
if self._best_net is None or int(
best_genome.fitness) > self._best_fitness:
if config.genome_config.feed_forward:
self._best_net = FeedForwardNetwork.create(best_genome, config)
else:
self._best_net = RecurrentNetwork.create(best_genome, config)
self._best_fitness = int(best_genome.fitness)
self._best_generation = self._generation_count
self._best_id = best_genome.key
# FIXME: hard coded value
if self._generation_count - self._best_generation >= 50 and (
self._generation_count - self._best_generation) % 10 == 0:
print(
"Need training assistance, id is", self._best_id, "fitness is",
self._best_fitness, "last improvment in generation",
self._best_generation,
"({} ago)".format(self._generation_count -
self._best_generation))
saveNet(self._best_net,
"need-training-{}.net".format(self._best_fitness))
def test_basic():
# Very simple network with one connection of weight one to a single sigmoid output node.
node_evals = [(0, activations.sigmoid_activation, sum, 0.0, 1.0, [(-1, 1.0)
])]
r = FeedForwardNetwork([-1], [0], node_evals)
assert r.values[0] == 0.0
result = r.activate([0.2])
assert r.values[-1] == 0.2
assert_almost_equal(r.values[0], 0.731, 0.001)
assert result[0] == r.values[0]
result = r.activate([0.4])
assert r.values[-1] == 0.4
assert_almost_equal(r.values[0], 0.881, 0.001)
assert result[0] == r.values[0]
def create_new(self, genome_type, genome_config, num_genomes):
new_genomes = {}
for i in range(num_genomes):
key = self.genome_indexer.next()
g = genome_type(key)
g.configure_new(genome_config)
g.net = FeedForwardNetwork.create(g, self.config)
new_genomes[key] = g
self.ancestors[key] = tuple()
return new_genomes
def evaluate(self, genome: DefaultGenome,
config: Config) -> Tuple[float, int]:
network = FeedForwardNetwork.create(genome, config)
environment = gym.make(self._environment_name)
fitness, frames = 0., 0
for _ in range(self._runs_per_network):
fit, fr = self._run_episode(network, environment)
fitness += fit
frames += fr
return fitness / self._runs_per_network, frames
def new(self, genomes, config):
self.all_sprites = pygame.sprite.LayeredUpdates()
# self.all_sprites = pygame.sprite.Group()
self.birds = pygame.sprite.Group()
self.mobs = pygame.sprite.Group()
self.pipes = pygame.sprite.Group()
self.bird_pool = {}
self.nets = {}
for genome_id, genome in genomes:
genome.fitness = 0
self.bird_pool[genome_id] = sprites.Player(self)
self.nets[genome_id] = FeedForwardNetwork.create(genome, config)
# self.player = sprites.Player(self)
self.pipes_pool = [
sprites.PipePair(self, WIDTH / 2),
sprites.PipePair(self, WIDTH + 50)
]
self.pipe_gen_time = pygame.time.get_ticks()
self.base_pool = [sprites.Base(self, 0), sprites.Base(self, 336)]
def render_result(
environment: Env,
network: FeedForwardNetwork,
steps: int = 500,
):
frames = []
observation = environment.reset()
for _ in range(steps):
outputs = network.activate(observation)
action = handle_action(outputs, environment)
observation, _, done, _ = environment.step(action)
frame = environment.render(mode='rgb_array')
frames.append(frame)
if done:
break
environment.close()
return frames
def _run_episode(
self,
network: FeedForwardNetwork,
environment: Env,
) -> Tuple[float, int]:
observation = environment.reset()
fitness = 0.0
done = False
step = 0
while not done:
if step > self._max_steps:
break
output = network.activate(observation)
action = handle_action(output, environment)
observation, reward, done, _ = environment.step(action)
fitness += reward
step += 1
return fitness, step
def post_evaluate(self, config, population, species, best_genome):
if self._best_id is not None and self._best_id not in population:
print(
"Lost best fitness in generation {}, restarting, saving old net as best-{}-{}.net"
.format(self._generation_count, self._best_fitness,
self._best_id))
os.rename('best.net', 'best-{}.net'.format(self._best_fitness))
self._best_id = None
self._best_fitness = None
if self._best_fitness is None or best_genome.fitness > self._best_fitness:
print("New best fitness: {}, Size = {}, Generation: {}, id: {}".
format(best_genome.fitness, best_genome.size(),
self._generation_count, best_genome.key))
self._best_fitness = best_genome.fitness
self._best_id = best_genome.key
if config.genome_config.feed_forward:
net = FeedForwardNetwork.create(best_genome, config)
else:
net = RecurrentNetwork.create(best_genome, config)
saveNet(net, "best.net")
self._generation_count += 1
def get(self, genome, config):
'''
if self.network_type == 'ffnn':
genome_nn = genome # No need to copy in this case, since there is no modification
if self.network_type == 'cppn':
# Transform the CPPN genome to an ANN genome
genome_nn = convert_to_ANN_genome(genome, substrate, config)
net = SimpleFeedForwardNetwork.create(genome_nn, config)
'''
if self.network_type == 'ffnn':
net = SimpleFeedForwardNetwork.create(genome, config)
elif self.network_type == 'cppn':
cppn_net = FeedForwardNetwork.create(genome, config)
net = PhenotypeNetwork.create(cppn_net, self.substrate)
fitness = 0
#print(id(self.game))
for i in range(self.n_games):
# Create puzzles of varying difficulties
error_rate = self.error_rates[i % len(self.error_rates)]
fitness += self.game.play(net, error_rate, self.error_mode,
self.reward_mode,
GameMode["TRAINING"])["fitness"]
return fitness / self.n_games
def eval_genome(genome, config):
'''
The result of this function gets assigned to the genome's fitness.
'''
net = FeedForwardNetwork.create(genome, config)
return config.eval_net_mean(net, genome)
def reproduce(self, config, species, pop_size, generation, exp, syllabus):
all_fitnesses = []
for sid, s in iteritems(species.species):
all_fitnesses.extend(m.fitness for m in itervalues(s.members))
min_fitness = min(all_fitnesses)
max_fitness = max(all_fitnesses)
fitness_range = max(1.0, max_fitness - min_fitness)
num_remaining = 0
species_fitness = []
avg_adjusted_fitness = 0.0
for sid, s, stagnant in self.stagnation.update(species, generation):
if stagnant:
self.reporters.species_stagnant(sid, s)
else:
num_remaining += 1
msf = mean([m.fitness for m in itervalues(s.members)])
s.adjusted_fitness = (msf - min_fitness) / fitness_range
species_fitness.append((sid, s, s.fitness))
avg_adjusted_fitness += s.adjusted_fitness
if 0 == num_remaining:
species.species = {}
return []
avg_adjusted_fitness /= len(species_fitness)
self.reporters.info(
"Average adjusted fitness: {:.3f}".format(avg_adjusted_fitness))
spawn_amounts = []
for sid, s, sfitness in species_fitness:
spawn = len(s.members)
if sfitness > avg_adjusted_fitness:
spawn = max(spawn + 2, spawn * 1.1)
else:
spawn = max(spawn * 0.9, 2)
spawn_amounts.append(spawn)
total_spawn = sum(spawn_amounts)
norm = pop_size / total_spawn
spawn_amounts = [int(round(n * norm)) for n in spawn_amounts]
new_population = {}
species.species = {}
learning_function = None
if exp.learning_function == 'BP':
learning_function = backpropagation
elif exp.learning_function == 'BT':
learning_function = batch
has_learning = not learning_function is None
if has_learning:
(writer, lessons) = syllabus
if exp.syllabus_source == 'EXP':
lessons_to_learn = random.sample(
lessons, min(exp.syllabus_size, len(lessons)))
elif exp.syllabus_source == 'RND':
lessons_to_learn = lessons
for spawn, (sid, s, sfitness) in zip(spawn_amounts, species_fitness):
spawn = max(spawn, self.elitism)
if spawn <= 0:
continue
old_members = list(iteritems(s.members))
s.members = {}
species.species[sid] = s
old_members.sort(reverse=True, key=lambda x: x[1].fitness)
if self.elitism > 0:
for i, m in old_members[:self.elitism]:
new_population[i] = m
spawn -= 1
if spawn <= 0:
continue
repro_cutoff = int(
math.ceil(self.survival_threshold * len(old_members)))
repro_cutoff = max(repro_cutoff, 2)
old_members = old_members[:repro_cutoff]
if has_learning and (exp.learning_target == 'Ambos'
or exp.learning_target == 'Pais'):
for (_, g) in old_members:
if g != writer:
learning_function(g, lessons_to_learn,
exp.learning_rate)
while spawn > 0:
spawn -= 1
parent1_id, parent1 = random.choice(old_members)
parent2_id, parent2 = random.choice(old_members)
gid = self.genome_indexer.next()
child = config.genome_type(gid)
child.configure_crossover(parent1, parent2,
config.genome_config)
child.mutate(config.genome_config)
child.net = FeedForwardNetwork.create(child, self.config)
if has_learning and (exp.learning_target == 'Ambos'
or exp.learning_target == 'Filhos'):
if exp.children_inclusion == 'Inicial' or (
exp.children_inclusion == 'Tardia'
and generation >= exp.generations / 2):
learning_function(child, lessons_to_learn,
exp.learning_rate)
new_population[gid] = child
self.ancestors[gid] = (parent1_id, parent2_id)
return new_population
def evaluate(file,
error_rates,
error_mode,
n_games,
n_jobs,
verbose,
file_suffix='',
transfer_to_distance=None):
time_id = datetime.now()
# Load the corresponding config files
savedir = file[:file.rfind("/")]
if not os.path.exists("%s/config.json" % savedir):
raise ValueError("Configuration file does not exist (%s)." %
("%s/config.json" % savedir))
with open("%s/config.json" % savedir) as f:
config = json.load(f)
config = check_config(config)
# Load the genome to be evaluated
if not os.path.exists(file):
raise ValueError("Genome file does not exist.")
with open(file, "rb") as f:
genome = pickle.load(f)
if not os.path.exists("%s/population-config" % savedir):
raise ValueError("Population configuration file does not exist.")
population_config = neat.Config(neat.DefaultGenome,
neat.DefaultReproduction,
neat.DefaultSpeciesSet,
neat.DefaultStagnation,
"%s/population-config" % savedir)
if config["Training"]["network_type"] == 'ffnn':
if transfer_to_distance is None:
net = SimpleFeedForwardNetwork.create(genome, population_config)
code_distance = config["Physics"]["distance"]
elif transfer_to_distance > config["Physics"]["distance"]:
generate_config_file(savedir, config, transfer_to_distance)
pop_config_transferred = neat.Config(
neat.DefaultGenome, neat.DefaultReproduction,
neat.DefaultSpeciesSet, neat.DefaultStagnation, savedir +
"/population-config-temp-d" + str(transfer_to_distance))
new_genome = pop_config_transferred.genome_type(0)
new_genome.configure_new(pop_config_transferred.genome_config)
new_genome.connections = {}
new_genome.nodes = {}
transplantate(pop_config_transferred.genome_config, new_genome,
transfer_to_distance, genome,
config["Physics"]["distance"])
net = SimpleFeedForwardNetwork.create(new_genome,
pop_config_transferred)
code_distance = transfer_to_distance
elif config["Training"]["network_type"] == 'cppn':
# HyperNEAT: possibility of evaluating a CPPN trained on d=3 data on d>3 data
if transfer_to_distance is None:
code_distance = config["Physics"]["distance"]
connection_weight_scale = 1
elif transfer_to_distance > config["Physics"]["distance"]:
code_distance = transfer_to_distance
# As they are more connections, in larger codes, we need to scale down the connection weight by this factor
connection_weight_scale = config["Physics"][
"distance"]**2 / transfer_to_distance**2
#connection_weight_scale = 0.01
else:
raise ValueError(
"Transfer knwoledge can only be done to higher distance codes."
)
if config["Training"]["substrate_type"] == 0:
substrate = SubstrateType0(
code_distance,
config["Training"]["rotation_invariant_decoder"])
elif config["Training"]["substrate_type"] == 1:
substrate = SubstrateType1(code_distance)
#print(code_distance, connection_weight_scale)
cppn_network = FeedForwardNetwork.create(genome, population_config)
net = PhenotypeNetwork.create(cppn_network, substrate,
connection_weight_scale)
# DIRTY: To ensure that samples are generated according to transfer_to_distance
config["Physics"]["distance"] = code_distance
## (PARALLEL) EVALUATION LOOP
fitness = []
results = {
"fitness": [],
"error_rate": [],
"outcome": [],
"nsteps": [],
"initial_qubits_flips": []
}
# with statement to close properly the parallel processes
with Pool(n_jobs) as pool:
# Game evaluation
for error_rate in error_rates:
fitness.append(0)
jobs = []
for i in range(n_games):
#
jobs.append(
pool.apply_async(get_fitness,
(net, config, error_rate, error_mode)))
for job in jobs:
output, errors_id = job.get(timeout=None)
fitness[-1] += output["fitness"]
for k, v in output.items():
results[k].append(v)
results["initial_qubits_flips"].append(errors_id)
fitness[-1] /= n_games
print("Evaluation on error_rate=%.2f is done, %.2f success." %
(error_rate, fitness[-1]))
elapsed = datetime.now() - time_id
print("Total running time:", elapsed.seconds, ":",
elapsed.microseconds)
# Always overwrite the result of evaluation
# Synthesis report
if transfer_to_distance is not None:
file_suffix += ".transfered_distance%i" % transfer_to_distance
savefile = "%s_evaluation.ngames=%i.errormode=%i.%s.csv" % (
file.replace(".pkl", ""), n_games, error_mode, file_suffix)
if os.path.exists(savefile):
print("Deleting evaluation file %s" % savefile)
os.remove(savefile)
print([error_rates, fitness])
df = pd.DataFrame(list(zip(error_rates, fitness)),
columns=["error_rate", "mean_fitness"])
df.to_csv(savefile)
# Detailed report
savefile = "%s_detailed_results_evaluation.ngames=%i.%s.csv" % (
file.replace(".pkl", ""), n_games, file_suffix)
if os.path.exists(savefile):
print("Deleting evaluation file %s" % savefile)
os.remove(savefile)
pd.DataFrame.from_dict(results).to_csv(savefile)
return error_rates, fitness
ENV_NAME = 'BipedalWalker-v3'
LOAD_PATH = Path(f'./{ENV_NAME}.pkl')
CONFIG_PATH = NEAT_CONFIGS_PATH / 'config-bipedal-walker-v3'
config = neat.Config(
neat.DefaultGenome,
neat.DefaultReproduction,
neat.DefaultSpeciesSet,
neat.DefaultStagnation,
str(CONFIG_PATH),
)
with LOAD_PATH.open('rb') as file:
genome = pickle.load(file)
network = FeedForwardNetwork.create(genome, config)
frames = render_result(
environment=gym.make(ENV_NAME),
network=network,
steps=2000,
)
def save_frames_as_gif(frames, path='./', filename='gym_animation.gif'):
plt.tight_layout()
plt.figure(figsize=(frames[0].shape[1] / 72.0, frames[0].shape[0] / 72.0),
dpi=72)
patch = plt.imshow(frames[0])
plt.axis('off')
fitness = game.fitness()
last_change = 0
else:
last_change += 1
if last_change > 250: # Around 10seconds in game
break
if last_change > 125: # I guess around 5 seconds in game?
break
if __name__ == '__main__':
# TODO parse sysargs properly
net = None
if len(sys.argv) >= 3:
p = Checkpointer.restore_checkpoint(sys.argv[1])
genome = p.population[int(sys.argv[2])]
if p.config.genome_config.feed_forward:
net = FeedForwardNetwork.create(genome, p.config)
else:
net = RecurrentNetwork.create(genome, p.config)
elif len(sys.argv) >= 2:
with gzip.open(sys.argv[1]) as f:
net = pickle.load(f)
if type(net) == tuple:
p = Checkpointer.restore_checkpoint(sys.argv[1])
for key in p.population:
print(key)
print("No id selected")
sys.exit(0)
print("Fitness:", simulateGame(net))
