def __init__(self, organism_type, config={}, assessor_function=None, verbose=True):
self.organism_type = organism_type
self.check_organism_type()
self.verbose = verbose
self.assessor_function = assessor_function
self.config = dict(self.default_config(), **config)
self.population = Population(organism_type, self.config)
def test_minimal():
local_dir = os.path.dirname(__file__)
config_path = os.path.join(local_dir, 'test_configuration')
pop = Population(config_path)
pe = parallel.ParallelEvaluator(4, eval_fitness)
pop.run(pe.evaluate, 400)
def test_minimal():
# sample fitness function
def eval_fitness(population):
for individual in population:
individual.fitness = 1.0
# creates the population
local_dir = os.path.dirname(__file__)
config = Config(os.path.join(local_dir, 'test_configuration'))
config.save_best = True
pop = Population(config)
# run the simulation for up to 20 generations
pop.run(eval_fitness, 20)
# Test save_checkpoint with defaults
pop.save_checkpoint()
# get statistics
best_unique = pop.statistics.best_unique_genomes(1)
best_unique = pop.statistics.best_unique_genomes(2)
avg_fitness = pop.statistics.get_average_fitness()
assert len(avg_fitness) == 1
assert all(f == 1 for f in avg_fitness)
# Change fitness threshold and do another run.
config.max_fitness_threshold = 1.1
pop = Population(config)
# runs the simulation for 20 generations
pop.run(eval_fitness, 20)
# get statistics
avg_fitness = pop.statistics.get_average_fitness()
assert len(avg_fitness) == 20
assert all(f == 1 for f in avg_fitness)
def __init__(self, seed, setContextFunc, population=None, train=True):
BaseContext.__init__(self, setContextFunc)
self.seed = seed
self.pop = Population(seed, 100) if population is None else population
if train:
self.worlds = []
for net in sorted(self.pop.current_generation, key=lambda x: x.fitness, reverse=True):
nWorld = NeuronalWorld(self.pop.seed, net)
nWorld.renderer = RenderNeuronalWorld(nWorld)
self.worlds.append(nWorld)
nWorld.generatePlatform()
else:
best_nn = max((n for n in self.pop.current_generation), key=lambda x: x.fitness)
nWorld = NeuronalWorld(self.pop.seed, best_nn)
nWorld.renderer = RenderNeuronalWorld(nWorld)
self.worlds = [nWorld]
self.drawmode = 0
self._train = train
# the gui overlays
self._overlays = texhandler.adjustedSurface(texhandler.Textures.overlays, height=48)
fontObj = SysFont("Monospace", 30, bold=True)
# mode switch buttons
self.addElements({
"bNext": GuiButton(constants.screenWidth - 100, constants.screenHeight - 70, fontObj, "->",
width=70).connect(self.buttonModeSwitch, 1),
"bPrevious": GuiButton(30, constants.screenHeight - 70, fontObj, "<-", width=70).connect(
self.buttonModeSwitch, -1)
})
def __init__(self, config, stats):
Population.__init__(self, config)
self.config = config
self.stats = stats
def test_minimal():
# sample fitness function
def eval_fitness(population):
for individual in population:
individual.fitness = 1.0
# creates the population
config = Config('test_configuration')
pop = Population(config)
# runs the simulation for 250 epochs
pop.epoch(eval_fitness, 250)
def test_minimal():
# sample fitness function
def eval_fitness(population):
for individual in population:
individual.fitness = 1.0
# creates the population
local_dir = os.path.dirname(__file__)
config = Config(os.path.join(local_dir, 'test_configuration'))
pop = Population(config)
# runs the simulation for 250 epochs
pop.epoch(eval_fitness, 250)
def __init__(self, config: Config):
self.config = config
self.population = Population(self.config)
self.observers = [] # type: List[AbstractObserver]
self.best_genotype = None
self.best_genotype_score = None
self.best_genotypes = []
self.min_score_history = []
self.avg_score_history = []
self.max_score_history = []
self.max_score_history_gens = []
def evolve(config, evalfun, seed, task_name, ngen, checkpoint):
'''
NEAT evolution with parallel evaluator
'''
# Set random seed (including None)
random.seed(seed)
# Make directories for saving results
os.makedirs('models', exist_ok=True)
os.makedirs('visuals', exist_ok=True)
# Create an ordinary population or a population for NoveltySearch
pop = _NoveltyPopulation(config) if config.is_novelty() else Population(
config)
# Add a stdout reporter to show progress in the terminal.
pop.add_reporter(_StdOutReporter(show_species_detail=False))
stats = neat.StatisticsReporter()
pop.add_reporter(stats)
# Add a reporter (which can also checkpoint the best)
pop.add_reporter(_SaveReporter(task_name, checkpoint))
# Create a parallel fitness evaluator
pe = neat.ParallelEvaluator(mp.cpu_count(), evalfun)
# Run for number of generations specified in config file
winner = pop.run(pe.evaluate) if ngen is None else pop.run(
pe.evaluate, ngen)
# Save winner
config.save_genome(winner)
def restore_checkpoint(filename):
"""Resumes the simulation from a previous saved point."""
with gzip.open(filename) as f:
generation, config, population, species_set, rndstate = pickle.load(
f)
random.setstate(rndstate)
return Population(config, (population, species_set, generation))
def restore_checkpoint(filename):
"""Resumes the simulation from a previous saved point."""
with open(filename, 'rb') as f:
generation, config, population, species_set, best_genome, rndstate = dill.load(
f)
random.setstate(rndstate)
return Population(config, (population, species_set, generation))
def _epoch(self):
if self.current_generation == 0:
self.population = Population(self.configuration)
else:
# Create the next generation from the current generation.
self.population.population = self.population.reproduction.reproduce(self.population.config, self.population.species,
self.population.config.pop_size, self.population.generation)
# Check for complete extinction.
if not self.population.species.species:
self.population.reporters.complete_extinction()
# If requested by the user, create a completely new population,
# otherwise raise an exception.
if self.population.config.reset_on_extinction:
self.population.population = self.population.reproduction.create_new(self.population.config.genome_type,
self.population.config.genome_config,
self.population.config.pop_size)
else:
raise CompleteExtinctionException()
# Evaluate all genomes using the user-provided function.
self._eval_genomes(list(iteritems(self.population.population)), self.population.config)
# Gather and report statistics.
best = None
for g in itervalues(self.population.population):
if best is None or g.fitness > best.fitness:
best = g
# Track the best genome ever seen.
if self.population.best_genome is None or best.fitness > self.population.best_genome.fitness:
self.population.best_genome = best
self.champion = self.population.best_genome
# Divide the new population into species.
self.population.species.speciate(self.population.config, self.population.population, self.population.generation)
return self.stopping_criterion.evaluate(self)
def __init__(self, networkContext, setContextFunc, popFileName):
BaseContext.__init__(self, setContextFunc)
self._networkContext = networkContext
self._popFileName = popFileName
self._pop = Population.load_from_file(
constants.res_loc("networks") + popFileName)
self._background = texturehandler.fillSurface(
pygame.Surface(constants.screenSize),
random.choice(texturehandler.blocks), (64, 64))
best_fitness = max(n.fitness for n in self._pop.current_generation)
fontObj = Font(None, 40)
self.addElements({
"lCaption":
GuiLabel.createCentered(10, Font(None, 60), self._pop.name),
"lSeed":
GuiLabel(240, 90, fontObj, "Current seed:"),
"tfSeed":
GuiNumberTextfield(450,
87,
SysFont("Monospace", 24, bold=True),
width=140,
text=str(self._pop.seed)),
"bSeed":
GuiButton(600, 87, fontObj, "Set Seed", width=200,
height=32).connect(self.buttonSetSeed),
"lSize":
GuiLabel(
240, 130, fontObj, "Population size: {}".format(
len(self._pop.current_generation))),
"lFitness":
GuiLabel(240, 170, fontObj,
"Highest fitness: {0:.2f}".format(best_fitness)),
"lGeneration":
GuiLabel(240, 210, fontObj,
"Generation: {}".format(self._pop.generation_count)),
"bDelete":
GuiButton(390,
470,
fontObj,
"Delete (hold CTRL)",
width=300,
height=40,
startColor=(255, 50, 50),
endColor=(255, 100, 100)).connect(self.buttonDelete),
"bShowResult":
GuiButton(240, 530, fontObj, "Show Result",
width=285).connect(self.buttonShowResult),
"bResumeTraining":
GuiButton(555, 530, fontObj, "Resume Training",
width=285).connect(self.buttonResumeTraining),
"bBack":
GuiButton(240, 600, fontObj, "Back").connect(self.buttonBack)
})
# enable key repeats
pygame.key.set_repeat(500, 50)
def main():
try:
pop = Population.load_from_file(constants.res_loc("networks") + pop_name + ".pop")
except:
seed = random.randint(0, 1000)
pop = Population(seed, 100)
pool = Pool(number_of_processes)
while True:
worlds = []
for net in pop.current_generation:
nWorld = NeuronalWorld(pop.seed, net)
worlds.append(nWorld)
nWorld.generatePlatform()
# evaluate all neuronal worlds
fitnesses = pool.map(evaluate, worlds)
# set the fitness (because multiprocessing)
for world, fit in zip(worlds, fitnesses):
world.nn.fitness = fit
path = constants.res_loc("networks") + pop.name + ".pop"
pop.save_to_file(path)
best_fitness = max(nn.fitness for nn in pop.current_generation)
print("best fitness:", best_fitness)
pop.create_next_generation()
pop.generation_count += 1
def test_checkpoint():
# sample fitness function
def eval_fitness(population):
for individual in population:
individual.fitness = 0.99
# creates the population
local_dir = os.path.dirname(__file__)
config = Config(os.path.join(local_dir, 'test_configuration'))
pop = Population(config)
pop.run(eval_fitness, 20)
t = tempfile.NamedTemporaryFile(delete=False)
t.close()
pop.save_checkpoint(t.name)
pop2 = Population(config)
pop2.load_checkpoint(t.name)
def test_config_options():
# sample fitness function
def eval_fitness(population):
for individual in population:
individual.fitness = 1.0
local_dir = os.path.dirname(__file__)
config = Config(os.path.join(local_dir, 'test_configuration'))
for hn in (0, 1, 2):
config.hidden_nodes = hn
for fc in (0, 1):
config.fully_connected = fc
for act in activation_functions.functions.keys():
config.allowed_activation = [act]
for ff in (0, 1):
config.feedforward = ff
pop = Population(config)
pop.run(eval_fitness, 250)
def test_config_options():
# sample fitness function
def eval_fitness(population):
for individual in population:
individual.fitness = 1.0
local_dir = os.path.dirname(__file__)
config = Config(os.path.join(local_dir, 'test_configuration'))
for hn in (0, 1, 2):
config.hidden_nodes = hn
for fc in (0, 1):
config.fully_connected = fc
for act in activation_functions.functions.keys():
config.allowed_activation = [act]
for ff in (0, 1):
config.feedforward = ff
pop = Population(config)
pop.run(eval_fitness, 250)
def __init__(self, env, n_pops=150, offline=False):
self.env = env
self.n_trials = env.spec.trials
self.reward_threshold = env.spec.reward_threshold
genesis = Creature(env.observation_space.shape[0], env.action_space.n)
self.population = Population(genesis, n_pops)
self.fitness_history = []
self.snapshot_i = 0
run_id_hash = hashlib.sha1()
run_id_hash.update(str(time()).encode('utf-8'))
self.run_id = run_id_hash.hexdigest()[:16]
self.offline = offline
if not self.offline:
r = requests.request('POST',
NeatAlgorithm.api_url + '/runs',
json=dict(id=self.run_id))
if r.status_code != 201:
print('WARNING: Could not add run data to database.')
print(r.json())
def restore_checkpoint(filename):
"""Resumes the simulation from a previous saved point."""
with gzip.open(filename) as f:
generation, config, population, species_set, rndstate = pickle.load(
f)
random.setstate(rndstate)
# print('generation: ', generation)
# print('config: ', config)
pop = list(population.keys())
pop.sort()
# print('population: ', population)
# print('-'*20)
# print('species_set: ', species_set)
# print('rndstate: ', rndstate)
return Population(config, (population, species_set, generation))
def test_checkpoint():
# sample fitness function
def eval_fitness(population):
for individual in population:
individual.fitness = 0.99
# creates the population
local_dir = os.path.dirname(__file__)
config = Config(os.path.join(local_dir, 'test_configuration'))
pop = Population(config)
pop.run(eval_fitness, 20)
t = tempfile.NamedTemporaryFile(delete=False)
t.close()
pop.save_checkpoint(t.name)
pop2 = Population(config)
pop2.load_checkpoint(t.name)
def from_json(config, offline=False):
"""Load an instance of the NEAT algorithm from JSON.
Arguments:
config: the JSON dictionary loaded from file.
offline: Whether the NEAT instance should be started 'offline'.
Returns: an instance of the NEAT algorithm.
"""
env = gym.make(config['env'])
algo = NeatAlgorithm(env, offline=offline)
algo.run_id = config['run_id']
algo.n_trials = env.spec.trials
algo.reward_threshold = env.spec.reward_threshold
algo.population = Population.from_json(config['population'])
NeatAlgorithm.set_config(config['settings'])
return algo
def restore_checkpoint(filename):
"""Resumes the simulation from a previous saved point."""
with gzip.open(filename) as f:
generation, config, population, species_set, rndstate = pickle.load(
f)
# Truncates the fitplot data file to the current generation:
rebuilt = []
with open('fitness_population', 'r') as f:
lines = f.readlines()
for line in lines:
if line.startswith(str(generation)):
break
else:
rebuilt.append(line)
if rebuilt:
with open('fitness_population', 'w') as f:
f.writelines(rebuilt)
random.setstate(rndstate)
return Population(config, (population, species_set, generation))
time_out = config.fitness_threshold
proper_game = play(game=proper_tetris, brain=brain, time_out=time_out)
return proper_game.score
evaluator = ParallelEvaluator(
num_workers=CPU_COUNT,
eval_function=fitness_f,
)
if __name__ == '__main__':
print('CPU_COUNT:', CPU_COUNT)
p = Population(MAIN_CONFIG)
p.add_reporter(StdOutReporter(True))
stats = StatisticsReporter()
p.add_reporter(stats)
p.add_reporter(Checkpointer(50))
try:
winner = p.run(evaluator.evaluate, 1000)
except KeyboardInterrupt:
pass
stats.save()
print(stats.best_genome())
class NNTraningContext(BaseContext):
"""
Context for training neuronal networks
"""
def __init__(self, seed, setContextFunc, population=None, train=True):
BaseContext.__init__(self, setContextFunc)
self.seed = seed
self.pop = Population(seed, 100) if population is None else population
if train:
self.worlds = []
for net in sorted(self.pop.current_generation, key=lambda x: x.fitness, reverse=True):
nWorld = NeuronalWorld(self.pop.seed, net)
nWorld.renderer = RenderNeuronalWorld(nWorld)
self.worlds.append(nWorld)
nWorld.generatePlatform()
else:
best_nn = max((n for n in self.pop.current_generation), key=lambda x: x.fitness)
nWorld = NeuronalWorld(self.pop.seed, best_nn)
nWorld.renderer = RenderNeuronalWorld(nWorld)
self.worlds = [nWorld]
self.drawmode = 0
self._train = train
# the gui overlays
self._overlays = texhandler.adjustedSurface(texhandler.Textures.overlays, height=48)
fontObj = SysFont("Monospace", 30, bold=True)
# mode switch buttons
self.addElements({
"bNext": GuiButton(constants.screenWidth - 100, constants.screenHeight - 70, fontObj, "->",
width=70).connect(self.buttonModeSwitch, 1),
"bPrevious": GuiButton(30, constants.screenHeight - 70, fontObj, "<-", width=70).connect(
self.buttonModeSwitch, -1)
})
def calculateDelta(self, clock):
if self._train:
return constants.UPS
else:
return BaseContext.calculateDelta(self, clock)
def update(self, t):
BaseContext.update(self, t)
done = True
for world in self.worlds:
if world.update(constants.UPS):
done = False
if done and self._train:
self.pop.save_to_file(constants.res_loc("networks") + self.pop.name + ".pop")
self.pop.create_next_generation()
self.pop.generation_count += 1
self.worlds = []
for net in self.pop.current_generation:
nWorld = NeuronalWorld(self.pop.seed, net)
nWorld.renderer = RenderNeuronalWorld(nWorld)
self.worlds.append(nWorld)
nWorld.generatePlatform()
def draw(self, screen):
if self.worlds:
[self.drawSimple, self.drawNetwork, self.drawSummary, self.drawOverview][self.drawmode](screen)
BaseContext.draw(self, screen)
# draw current generation
renderedGen = SysFont("Monospace", 40, bold=True).render(str(self.pop.generation_count), 1, (50, 50, 50))
# draw generation overlay (dinosaur egg)
screen.blit(self._overlays, (485, 650), (336, 0, 48, 48))
screen.blit(renderedGen, (540, 655))
def drawSimple(self, screen):
"""
only draws the first network
"""
self.worlds[0].renderer.render(screen)
def drawNetwork(self, screen):
"""
draws the first network with it's network-graph
"""
# world = max(self.worlds, key=lambda w: w.nn.fitness)
# draw the world
world = self.worlds[0]
world.renderer.render(screen)
networkSurface = pygame.Surface((750, 180)).convert_alpha()
networkSurface.fill((0, 0, 0, 0))
# draw the minimap and network
networkrenderer.render_network(networkSurface, world.nn, world.minimapValues)
screen.blit(networkSurface, (10, 60))
def drawSummary(self, screen):
"""
draws a summary
"""
x = 30
y = 50
fontObj = SysFont("Monospace", 18, bold=True)
time = 0
rowHeight = fontObj.get_height() + 2
for world in self.worlds:
time = max(time, world.time)
renderedFitness = fontObj.render("Fitness: {0:.2f}".format(world.nn.fitness), 1, (255, 255, 255))
if (y + rowHeight > (constants.screenHeight - 95)):
y = 50
x += 260
if x + renderedFitness.get_width() > constants.screenWidth:
break
# draw vertical line
pygame.draw.line(screen, (100, 100, 100), (x - 10, 45), (x - 10, constants.screenHeight - 100), 2)
# draw horizontal line (if still on first column)
elif x < 100:
pygame.draw.line(screen, (100, 100, 100), (25, y + rowHeight - 3),
(constants.screenWidth - 25, y + rowHeight - 3), 2)
screen.blit(renderedFitness, (x, y))
y += rowHeight
# draw the time
renderedTime = pygame.font.SysFont("Monospace", 34, bold=True).render("Time: {0:.2f}".format(time), 1,
(255, 255, 255))
screen.blit(renderedTime, ((constants.screenWidth - renderedTime.get_width()) // 2, 10))
def drawOverview(self, screen):
"""
draws the best 9 networks
"""
if len(self.worlds) >= 9:
partWidth = constants.screenWidth // 3
partHeight = constants.screenHeight // 3
bestWords = sorted(self.worlds, key=lambda x: -x.nn.fitness)[:9]
for y in range(0, 3):
for x in range(0, 3):
surface = pygame.Surface(constants.screenSize)
bestWords[3 * y + x].renderer.render(surface)
surface = pygame.transform.scale(surface, (partWidth, partHeight))
screen.blit(surface, (partWidth * x, partHeight * y))
pygame.draw.rect(screen, (0, 0, 0), (partWidth * x, partHeight * y, partWidth, partHeight), 1)
else:
self.drawSimple(screen)
def handleEvent(self, event):
if BaseContext.handleEvent(self, event):
return True
if event.type == pygame.KEYDOWN:
if event.key == pygame.K_ESCAPE:
from context.gamepausecontext import GamePauseContext
self._setContextFunc(GamePauseContext(self, self._setContextFunc))
elif event.key == pygame.K_TAB:
self.drawmode = (self.drawmode + 1) % 4
return False
"""
button functions
"""
def buttonModeSwitch(self, inc):
self.drawmode = (self.drawmode + inc) % 4
def execute(config):
population = Population(config)
population.evaluate(eval_xor, 600)
def main():
"""
The program's starting point and main logic
"""
# Initialize pygame, the screen and the framerate clock
pygame.init()
screen = pygame.display.set_mode((SCREEN_WIDTH, SCREEN_HEIGHT))
frame_clock = pygame.time.Clock()
# Lists to keep track of fitness and best network for every generation
fitness_history = []
network_history = []
# Parse the map description and generate walls and checkpoints accordingly
start_pos, walls, checkpoints = game_map.gen_map('assets/track.png')
# Instantiate a new game simulation
game = Game(CARS_PER_GENERATION, start_pos, walls, checkpoints)
# Generate an initial population (with networks which have 4 inputs and 4 outputs)
population = Population(CARS_PER_GENERATION, 4, 4)
cur_generation = 0
# Compute the usable neural network for each genome in the initial population
networks = [
organism.genome.as_neural_network()
for organism in population.organisms
]
# The car sensor data that the neural networks use is the values sensed last frame, so we need
# to save them
last_car_sensors = None
# We track the total runtime of the generation simulation so we can stop if the cars get stuck
# but don't die
running_time = 0
# We also track the fitness from the last simulation update so we can detect if there was a
# positivedelta in fitness, i.e. the cars made progress
last_fitness = None
while True:
# Handle pygame events
for event in pygame.event.get():
if event.type == pygame.QUIT:
# Exit if the close button was pressed
sys.exit()
elif event.type == pygame.KEYDOWN:
# Keyboard shortcuts
if event.key == pygame.K_f:
# If the letter `f` was pressed, print the fitness of all simulated cars
print(game.get_cars_fitness())
elif event.key == pygame.K_s:
# If the letter `s` was pressed, dump the history
print('fitness_history =', fitness_history)
print('network_history =', network_history)
fitness = game.get_cars_fitness()
if last_fitness is not None:
# We go through each car and check if its fitness improved by some epsilon, to detect if
# any of the cars made progress
had_progress = False
for last, cur in zip(last_fitness, fitness):
if cur > last + 0.05:
had_progress = True
break
# If some car did make progress, we don't want to end the simulation early, so we push
# back the runtime by 100ms
if had_progress:
running_time = max(0, running_time - 100)
last_fitness = fitness
# If all the cars have died by colliding with a wall, or the cars did not make sufficient
# progress in over 1 second (1000ms), we end the simulation
if all(game.dead) or running_time > 1000:
print(f'Finished generation {cur_generation}')
print(
f'Average: {sum(fitness)/len(fitness):5.2f} Max: {max(fitness):5.2f}'
)
# Record the fitness scores of each car in the generation
fitness_history.append(fitness)
# Find and record the genome of the most fit organism fo this generation
best_idx = 0
for i in range(1, CARS_PER_GENERATION):
if fitness[i] > fitness[best_idx]:
best_idx = i
network_history.append(
population.organisms[best_idx].genome.connections)
# Update NEAT's view of the fitness scores of the organisms so the genetic algorithm
# can proceed. A tiny epsilon is added because a fitness score of zero does not work
# well when the relative fitness is calculated
for i in range(CARS_PER_GENERATION):
population.organisms[i].fitness = 0.00001 + fitness[i]
# Go through all of the genetic algorithm steps, creating the next generation
population.epoch()
# Keep track of the current generation
cur_generation += 1
# Reset the running time
running_time = 0
# Recompute the usable neural networks for the new organisms
networks = [
organism.genome.as_neural_network()
for organism in population.organisms
]
# Reset the game simulation
game = Game(CARS_PER_GENERATION, start_pos, walls, checkpoints)
# Reset the last frame data
last_car_sensors = None
last_fitness = None
# Background color
screen.fill((57, 57, 57))
# We need to choose which car the game camera tracks: We pick the most fit car which is not
# dead
best_car = None
for i in range(0, CARS_PER_GENERATION):
if game.dead[i]: continue
if best_car is None or fitness[i] > fitness[best_car]:
best_car = i
# If all cars are dead we just default to the first one
if best_car is None:
best_car = 0
game.track_car(best_car)
# If this is the first frame, we don't yet have sensor data to base our decision on, so we
# just don't move
if last_car_sensors is None:
controls = [{
'forward': False,
'left': False,
'backward': False,
'right': False
}] * CARS_PER_GENERATION
else:
# We compute the controls for each car
controls = []
for i in range(CARS_PER_GENERATION):
# We run the sensor data from last frame through the car's network
network_output = networks[i].evaluate_input(
last_car_sensors[i])
# We then decide whether to enable that control based on a simple threshold
control = {
'forward': network_output[0] >= 0,
'left': network_output[1] >= 0,
'backward': network_output[2] >= 0,
'right': network_output[3] >= 0
}
controls.append(control)
# We make a simulation update step
last_car_sensors = game.update(frame_clock.get_time() / 1000, controls)
# We ask the game to draw the current state to the screen
game.draw_scene(screen)
# We draw the speedometer on top of the game, showing the speed of the tracked car
ui.draw_speedometer(screen, game.cars[best_car].get_normalized_speed())
# We display the drawn frame
pygame.display.flip()
running_time += frame_clock.tick(30) # Limit framerate to 30 FPS
class Neat:
def __init__(self, organism_type, config={}, assessor_function=None, verbose=True):
self.organism_type = organism_type
self.check_organism_type()
self.verbose = verbose
self.assessor_function = assessor_function
self.config = dict(self.default_config(), **config)
self.population = Population(organism_type, self.config)
def run(self, generations=None):
while True:
if self.population.next(self.assessor_function):
print("\nSolver Organism/s found.\n")
return
if generations is not None and self.population.generation > generations:
break
def get_solvers(self):
return self.population.solvers
def check_organism_type(self):
if not issubclass(self.organism_type, Organism):
raise Exception("organism_type must be a subclass of Organism")
if not callable(getattr(self.organism_type, 'calculate_fitness', None)):
raise Exception("organism_type must implement the calculate_fitness function")
def default_config(self):
return {
'population_size' : 100,
'num_inputs' : 1,
'num_outputs' : 1,
'add_node_prob' : 0.01,
'add_conn_prob' : 0.1,
'mutate_weight_prob' : 0.8,
'replace_weight_prob' : 0.1,
'weight_init' : 'gauss',
'weight_init_params' : [0, 1],
'weight_mutate_step' : 0.01,
'activations' : ['sigmoid'],
'activation_weights' : [1],
'output_activation' : 'sigmoid',
'weight_min' : -3,
'weight_max' : 3,
'compat_disjoint_coeff' : 1.0,
'compat_weight_coeff' : 3.0,
'compat_threshold' : 4.0,
'survival_threshold': 0.2,
'champion' : True,
'min_species_size' : 2,
'elitists' : 2,
'custom_print_fields' : [],
'stats_file' : None,
'dup_parent' : 0.25,
'population_stag' : 20,
'species_stag' : 15
}
if self.isRendered:
screen.fill(colors.black)
ship.draw(screen)
for bullet in bullets:
bullet.draw(screen)
for asteroid in asteroids:
asteroid.draw(screen)
pygame.display.flip()
clock.tick(60)
fitness = self.score
if self.shoots > 0: fitness += self.score / self.shoots * 68
fitness += self.time / 4000 * 68
return [fitness, self.score]
p = Population(300)
for i in range(100):
for genome in p.genomes:
for j in range(5):
result = Game.forAI(genome, False).run()
genome.fitness += result[0] / 5
if result[1] > genome.score: genome.score = result[1]
print "======================================================="
print "best fitness: %s" % max([g.fitness for g in p.genomes])
p.naturalSelection()
while True:
raw_input("Press Enter")
Game.forAI(p.champion).run()
class Neat:
def __init__(self, config: Config):
self.config = config
self.population = Population(self.config)
self.observers = [] # type: List[AbstractObserver]
self.best_genotype = None
self.best_genotype_score = None
self.best_genotypes = []
self.min_score_history = []
self.avg_score_history = []
self.max_score_history = []
self.max_score_history_gens = []
def start(self):
print(self.config.generation < self.config.max_generations)
print(self.config.evals < self.config.max_evals)
print(not self.config.done)
print("run",
self.config.generation < self.config.max_generations
and self.config.evals < self.config.max_evals
and not self.config.done,
end="; ")
while self.config.generation < self.config.max_generations and self.config.evals < self.config.max_evals and not self.config.done:
print("start notify", end="; ")
self._notify_start_generation()
print("next generation", end="; ")
self._next_generation()
print("generation done", end="; ")
self.config.generation += 1
self._notify_end_generation()
print("end notify", end="; ")
print()
def _next_generation(self) -> None:
print("gen started", end="; ")
if self.config.generation % self.config.seed_next == 0 and self.config.generation > 0:
print("seed", end="; ")
print("SEED OLD", self.population.seed)
self.population.next_seed()
print("SEED NEW", self.population.seed)
if self.config.generation % 50 == 0 and self.config.generation > 0:
min_score, avg_score, max_score, current_best_genotype = self.population.test(
)
self.min_score_history.append(min_score)
self.avg_score_history.append(avg_score)
self.max_score_history.append(max_score)
self.max_score_history_gens.append(self.config.generation)
if self.best_genotype is None or self.best_genotype_score < max_score:
self.best_genotype = current_best_genotype.copy()
self.best_genotype_score = max_score
self.best_genotypes.append(self.best_genotype)
if max_score >= self.config.max_score_termination:
print("SOLVED ENDING...")
self.config.done = True
self.config.done_genotype = current_best_genotype.copy()
self.config.done_genotype.score = max_score
self.population.adapt_params_start()
print("speciate", end="; ")
start = time.time()
self.population.speciate()
end = time.time()
print("TIME speciate end", end - start)
print("archivate", end="; ")
start = time.time()
self.population.archivate()
end = time.time()
print("TIME archivate end", end - start)
# print("crossover", end="; ")
# start = time.time()
# self.population.crossover_mutate_ray()
# end = time.time()
# print("\n\nTIME crossover_mutate_ray end", end - start)
print("crossover", end="; ")
start = time.time()
self.population.crossover()
end = time.time()
print("TIME crossover end", end - start)
print("mutate topology", end="; ")
start = time.time()
self.population.mutate_topology()
end = time.time()
print("TIME mutate end", end - start)
print("evaluate offspring", end="; ")
start = time.time()
self.population.evaluate_offsprings()
end = time.time()
print("TIME EVAL OFFSPRINGS end", end - start)
print("merge", end="; ")
start = time.time()
self.population.merge()
end = time.time()
print("TIME MERGE end", end - start)
# self.population.mutate_weights()
print("adapt params", end="; ")
self.population.adapt_params_end()
print(
"BEST DISABLED EDGES",
sum(1 for e in self.population.get_best_member().edges
if not e.enabled))
def add_observer(self, observer: AbstractObserver) -> None:
self.observers.append(observer)
# Autosave observer as last, otherwise changes in other observers wont be saved
new_observers = [
observer for observer in self.observers
if type(observer) == AutosaveObserver
]
for observer in [
observer for observer in self.observers
if type(observer) != AutosaveObserver
]:
new_observers.append(observer)
self.observers = new_observers
def _notify_start_generation(self):
for observer in self.observers:
observer.start_generation(self.config.generation)
def _notify_end_generation(self):
for observer in self.observers:
observer.end_generation(self)
@staticmethod
def open(path: str) -> "Neat":
print("Openning: " + path)
with open(path, 'rb') as file:
neat = pickle.load(file) # type: Neat
neat.config.next_tcp_port()
for obs in neat.observers:
if type(obs) == AutosaveObserver:
obs.init_walltime(neat.config.walltime_sec)
return neat
def __init__(self, config):
Population.__init__(self, config)
class NeatAlgorithm:
"""An implementation of the NEAT algorithm based off the original paper."""
api_url = 'http://localhost:5000/api'
def __init__(self, env, n_pops=150, offline=False):
self.env = env
self.n_trials = env.spec.trials
self.reward_threshold = env.spec.reward_threshold
genesis = Creature(env.observation_space.shape[0], env.action_space.n)
self.population = Population(genesis, n_pops)
self.fitness_history = []
self.snapshot_i = 0
run_id_hash = hashlib.sha1()
run_id_hash.update(str(time()).encode('utf-8'))
self.run_id = run_id_hash.hexdigest()[:16]
self.offline = offline
if not self.offline:
r = requests.request('POST',
NeatAlgorithm.api_url + '/runs',
json=dict(id=self.run_id))
if r.status_code != 201:
print('WARNING: Could not add run data to database.')
print(r.json())
def train(self,
n_episodes=100,
n_steps=200,
n_pso_episodes=5,
debug_mode=False):
"""Train species of individuals.
Arguments:
n_episodes: The number of episodes to trian for.
n_steps: The maximum number of steps per individual per episode.
n_pso_episodes: The number of episodes
debug_mode: If set to True, some features that aren't intended for
testing environments and such are disabled.
"""
sim_start = time()
episode_complete_msg_format = "\r{:03d}/{:03d} - " \
"mean fitness: {:.2f} - " \
"median fitness: {:.2f} - " \
"mean time per creature: {:02.4f}s - " \
"total time: {:.4f}s"
for episode in range(n_episodes):
print('Episode {:02d}/{:02d}'.format(episode + 1, n_episodes))
episode_start = time()
self.fitness_history.append([])
if n_pso_episodes > 0:
print('Acquiring Collective Intelligence...')
for species in self.population.species:
pso = PSO(self.env, species.members)
pso.train(n_episodes=n_pso_episodes, n_steps=n_steps)
pso.apply()
print('\nCollective Intelligence acquired in {:.4f}s.'.format(
time() - episode_start))
print('Evaluating Population Goodness...')
for pop_i, creature in enumerate(self.population.creatures):
observation = self.env.reset()
for step in range(n_steps):
action = creature.get_action(observation)
observation, reward, done, _ = self.env.step(action)
if done:
creature.fitness = step + 1
break
else:
creature.fitness = n_steps
self.fitness_history[episode].append(creature.fitness)
mean_fitness = np.mean(self.fitness_history[episode])
median_fitness = np.median(self.fitness_history[episode])
episode_time = time() - episode_start
mean_time_per_creature = episode_time / (pop_i + 1)
print(episode_complete_msg_format.format(
pop_i + 1, self.population.n_pops, mean_fitness,
median_fitness, mean_time_per_creature, episode_time),
end='')
if episode >= self.n_trials and \
np.mean(self.fitness_history[episode - self.n_trials:episode]) \
>= self.reward_threshold:
print('\nSolved in %d episodes :)' % (episode + 1))
break
print()
if not debug_mode:
self.make_snapshot()
self.population.next_generation()
print('Total episode time: %.4fs.\n' % (time() - episode_start))
else:
print('Could not solve in %d episodes :(' % n_episodes)
print('Total run time: {:.2f}s'.format(time() - sim_start))
print()
if not self.offline:
r = requests.request(
'PATCH',
'%s/runs/%s/finished' % (NeatAlgorithm.api_url, self.run_id))
if r.status_code != 204:
print("WARNING: Was not able to update run finished status.")
print(r.json())
self.post_training_stuff(n_steps, debug_mode)
def post_training_stuff(self, n_steps, debug_mode=False):
"""Do post training stuff."""
print('Here are the species that made it to the end and the number of '
'creatures in each of them:')
self.population.list_species()
best_species = self.population.best_species
print()
oldest_creature = self.population.oldest_creature
print('The oldest creature was %s, who lived for %d generations.' %
(oldest_creature, oldest_creature.age))
print('Out of these species, the best species was %s.' % best_species)
print(
'The overall champion was %s who had %d nodes and %d '
'connections in its neural network.' %
(best_species.champion, len(best_species.champion.phenotype.nodes),
len(best_species.champion.phenotype.connections)))
print()
print('Checking if %s makes the grade...' % best_species.champion,
end='')
makes_the_grade = self.makes_the_grade(best_species.champion, n_steps)
print(('\r%s makes the grade :)' if makes_the_grade else
'\r%s doesn\'t make the grade :(') % best_species.champion)
print()
if not debug_mode:
print("Recording %s" % best_species.champion)
self.record_video(best_species.champion, n_steps=n_steps)
self.dump()
self.env.close()
def makes_the_grade(self, creature, n_steps):
"""Check if the creature 'passes' the environment.
Returns: True if the creature passes, False otherwise.
"""
avg_reward = 0
env = self.env
for episode in range(self.n_trials):
observation = env.reset()
episode_reward = 0
for step in range(n_steps):
action = creature.get_action(observation)
observation, reward, done, _ = env.step(action)
episode_reward += reward
if done:
break
avg_reward += episode_reward
return (avg_reward / self.n_trials) >= self.reward_threshold
def record_video(self, creature, n_episodes=20, n_steps=200):
"""Record a video of the creature trying to solve the problem.
Arguments:
creature: the creature to record.
n_episodes: how many episodes to record.
n_steps: how many steps to run each episode for.
"""
env = wrappers.Monitor(self.env, './data/videos/%s' % time())
for i_episode in range(n_episodes):
observation = env.reset()
for step in range(n_steps):
env.render()
action = creature.get_action(observation)
observation, _, done, _ = env.step(action)
if done:
print("Episode finished after {} timesteps".format(step +
1))
break
def dump(self, path='data/training/', filename=None):
"""Save training data to file."""
if path[-1] != '/':
path += '/'
path += self.run_id + '/'
fullpath = path + (filename if filename else 'dump.json')
os.makedirs(path, exist_ok=True)
with open(fullpath, 'w') as f:
json.dump(self.to_json(), f)
print('Saved training data to: %s.' % fullpath)
def make_snapshot(self, path='data/training/', filename=None):
"""Save training data to file."""
if path[-1] != '/':
path += '/'
path += self.run_id + '/'
self.snapshot_i += 1
fullpath = path + (filename if filename else 'snapshot-%02d.json' %
self.snapshot_i)
os.makedirs(path, exist_ok=True)
with open(fullpath, 'w') as f:
json.dump(self.population.to_json(), f)
def to_json(self):
"""Encode the current state of the algorithm as JSON.
This saves pretty much everything from parameters to individual
creatures.
Returns: the generated JSON.
"""
return dict(
run_id=self.run_id,
env=self.env.unwrapped.spec.id,
population=self.population.to_json(),
settings=dict(
survival_threshold=Population.survival_threshold,
compatibility_threshold=Species.compatibility_threshold,
p_interspecies_mating=Species.p_interspecies_mating,
disjointedness_importance=Creature.disjointedness_importance,
excessivity_importance=Creature.excessivity_importance,
weight_unsameness_importance=Creature.
weight_unsameness_importance,
p_mate_only=Creature.p_mate_only,
p_mutate_only=Creature.p_mutate_only,
p_mate_average=Genome.p_mate_average,
p_mate_choose=Genome.p_mate_choose,
p_add_node=Genome.p_add_node,
p_add_connection=Genome.p_add_connection,
p_re_enable_connection=Genome.p_re_enable_connection,
p_perturb=Genome.p_perturb,
perturb_range=Genome.perturb_range))
@staticmethod
def from_json(config, offline=False):
"""Load an instance of the NEAT algorithm from JSON.
Arguments:
config: the JSON dictionary loaded from file.
offline: Whether the NEAT instance should be started 'offline'.
Returns: an instance of the NEAT algorithm.
"""
env = gym.make(config['env'])
algo = NeatAlgorithm(env, offline=offline)
algo.run_id = config['run_id']
algo.n_trials = env.spec.trials
algo.reward_threshold = env.spec.reward_threshold
algo.population = Population.from_json(config['population'])
NeatAlgorithm.set_config(config['settings'])
return algo
@staticmethod
def set_config(config):
"""Set the parameters of the NEAT algorithm.
Arguments:
config: A dictionary containing the key-value pairs for the
algorithm parameters.
"""
Population.survival_threshold = config['survival_threshold']
Species.compatibility_threshold = config['compatibility_threshold']
Species.p_interspecies_mating = config['p_interspecies_mating']
Creature.disjointedness_importance = config[
'disjointedness_importance']
Creature.excessivity_importance = config['excessivity_importance']
Creature.p_mate_only = config['p_mate_only']
Creature.p_mutate_only = config['p_mutate_only']
Genome.p_mate_average = config['p_mate_average']
Genome.p_mate_choose = config['p_mate_choose']
Genome.p_add_node = config['p_add_node']
Genome.p_add_connection = config['p_add_connection']
Genome.p_re_enable_connection = config['p_re_enable_connection']
Genome.p_perturb = config['p_perturb']
Genome.perturb_range = config['perturb_range']
def population():
pop_config = PopulationConfiguration(pop_size, inputs, outputs)
population = Population(pop_config, Innovations(), MutationRates())
return population
