def test_DistributedEvaluator_mode():
"""Tests for the mode determination of DistributedEvaluator"""
# test auto mode setting
# we also test that the mode is not
# automatically determined when explicitly given.
tests = (
# (hostname or ip, mode to pass, expected mode)
("localhost", MODE_PRIMARY, MODE_PRIMARY),
("0.0.0.0", MODE_PRIMARY, MODE_PRIMARY),
("localhost", MODE_SECONDARY, MODE_SECONDARY),
("example.org", MODE_PRIMARY, MODE_PRIMARY),
(socket.gethostname(), MODE_SECONDARY, MODE_SECONDARY),
("localhost", MODE_AUTO, MODE_PRIMARY),
(socket.gethostname(), MODE_AUTO, MODE_PRIMARY),
(socket.getfqdn(), MODE_AUTO, MODE_PRIMARY),
("example.org", MODE_AUTO, MODE_SECONDARY),
)
for hostname, mode, expected in tests:
addr = (hostname, 8022)
try:
de = neat.DistributedEvaluator(
addr,
authkey=b"abcd1234",
eval_function=eval_dummy_genome_nn,
mode=mode,
)
except EnvironmentError:
print("test_DistributedEvaluator_mode(): Error with hostname " +
"{!r}".format(hostname))
raise
result = de.mode
assert result == expected, "Mode determination failed! Hostname: {h}; expected: {e}; got: {r!r}!".format(
h=hostname, e=expected, r=result)
if result == MODE_AUTO:
raise Exception(
"DistributedEvaluator.__init__(mode=MODE_AUTO) did not automatically determine its mode!"
)
elif (result == MODE_PRIMARY) and (not de.is_primary()):
raise Exception(
"DistributedEvaluator.is_primary() returns False even if the evaluator is in primary mode!"
)
elif (result == MODE_SECONDARY) and de.is_primary():
raise Exception(
"DistributedEvaluator.is_primary() returns True even if the evaluator is in secondary mode!"
)
# test invalid mode error
try:
de = neat.DistributedEvaluator(
addr,
authkey=b"abcd1234",
eval_function=eval_dummy_genome_nn,
mode="#invalid MODE!",
)
de.start()
except ValueError:
pass
else:
raise Exception("Passing an invalid mode did not cause an exception to be raised on start()!")
def run_secondary(addr, authkey, num_workers=1):
# Determine path to configuration file. This path manipulation is
# here so that the script will run successfully regardless of the
# current working directory.
local_dir = os.path.dirname(__file__)
config_path = os.path.join(local_dir, 'test_configuration2')
# Load configuration.
config = neat.Config(neat.DefaultGenome, neat.DefaultReproduction,
neat.DefaultSpeciesSet, neat.DefaultStagnation,
config_path)
# Create the population, which is the top-level object for a NEAT run.
p = neat.Population(config)
# Add a stdout reporter to show progress in the terminal.
p.add_reporter(neat.StdOutReporter(True))
stats = neat.StatisticsReporter()
p.add_reporter(stats)
# Run for the specified number of generations.
de = neat.DistributedEvaluator(
addr,
authkey=authkey,
eval_function=eval_genome_distributed,
mode=MODE_SECONDARY,
num_workers=num_workers,
)
try:
de.start(secondary_wait=3, exit_on_stop=True, reconnect=True)
except SystemExit:
pass
else:
raise Exception("DistributedEvaluator in secondary mode did not try to exit!")
def test_DistributedEvaluator_primary_restrictions():
"""Tests that some primary-exclusive methods fail when called by the secondaries"""
secondary = neat.DistributedEvaluator(
("localhost", 8022),
authkey=b"abcd1234",
eval_function=eval_dummy_genome_nn,
mode=MODE_SECONDARY,
)
try:
secondary.stop()
except ModeError:
# only ignore ModeErrors
# a RuntimeError should only be raised when in primary mode.
pass
else:
raise Exception(
"A DistributedEvaluator in secondary mode could call stop()!")
try:
secondary.evaluate(None,
None) # we do not need valid values for this test
except ModeError:
# only ignore ModeErrors
# other errors should only be raised when in primary mode.
pass
else:
raise Exception(
"A DistributedEvaluator in secondary mode could call evaluate()!")
def run_secondary(addr, authkey, num_workers=1):
"""Starts a DistributedEvaluator in secondary mode."""
# Load configuration.
local_dir = os.path.dirname(__file__)
config_path = os.path.join(local_dir, 'test_configuration')
config = neat.Config(neat.DefaultGenome, neat.DefaultReproduction,
neat.DefaultSpeciesSet, neat.DefaultStagnation,
config_path)
# Create the population, which is the top-level object for a NEAT run.
p = neat.Population(config)
# Add a stdout reporter to show progress in the terminal.
p.add_reporter(neat.StdOutReporter(True))
stats = neat.StatisticsReporter()
p.add_reporter(stats)
# Run for the specified number of generations.
de = neat.DistributedEvaluator(
addr,
authkey=authkey,
eval_function=eval_dummy_genome_nn,
mode=MODE_SECONDARY,
num_workers=num_workers,
)
try:
de.start(secondary_wait=3, exit_on_stop=True)
except SystemExit:
pass
else:
raise Exception("DistributedEvaluator in secondary mode did not try to exit!")
def test_DistributedEvaluator_state_error4():
"""Tests that attempts to use an unstarted manager for get_namespace cause an error."""
primary = neat.DistributedEvaluator(
("localhost", 8022),
authkey=b"abcd1234",
eval_function=eval_dummy_genome_nn,
mode=MODE_PRIMARY,
)
try:
ignored = primary.em.get_namespace()
except RuntimeError:
pass
else:
raise Exception("primary.em.get_namespace() with unstarted manager did not raise a RuntimeError!")
def test_DistributedEvaluator_state_error1():
"""Tests that attempts to use an unstarted manager for set_secondary_state cause an error."""
primary = neat.DistributedEvaluator(
("localhost", 8022),
authkey=b"abcd1234",
eval_function=eval_dummy_genome_nn,
mode=MODE_PRIMARY,
)
try:
primary.em.set_secondary_state(_STATE_RUNNING)
except RuntimeError:
pass
else:
raise Exception("primary.em.set_secondary_state with unstarted manager did not raise a RuntimeError!")
def test_DistributedEvaluator_state_error5():
"""Tests that attempts to set an invalid state cause an error."""
primary = neat.DistributedEvaluator(
("localhost", 8022),
authkey=b"abcd1234",
eval_function=eval_dummy_genome_nn,
mode=MODE_PRIMARY,
)
primary.start()
try:
primary.em.set_secondary_state(-1)
except ValueError:
pass
else:
raise Exception("primary.em.set_secondary_state(-1) did not raise a ValueError!")
def run_primary(addr, authkey, generations):
"""Starts a DistributedEvaluator in primary mode."""
# Load configuration.
local_dir = os.path.dirname(__file__)
config_path = os.path.join(local_dir, 'test_configuration')
config = neat.Config(neat.DefaultGenome, neat.DefaultReproduction,
neat.DefaultSpeciesSet, neat.DefaultStagnation,
config_path)
# Create the population, which is the top-level object for a NEAT run.
p = neat.Population(config)
# Add a stdout reporter to show progress in the terminal.
p.add_reporter(neat.StdOutReporter(True))
stats = neat.StatisticsReporter()
p.add_reporter(stats)
p.add_reporter(neat.Checkpointer(max(1,int(generations/3)), 5))
# Run for the specified number of generations.
de = neat.DistributedEvaluator(
addr,
authkey=authkey,
eval_function=eval_dummy_genome_nn,
mode=MODE_PRIMARY,
secondary_chunksize=15,
)
print("Starting DistributedEvaluator")
sys.stdout.flush()
de.start()
print("Running evaluate")
sys.stdout.flush()
p.run(de.evaluate, generations)
print("Evaluated")
sys.stdout.flush()
de.stop(wait=5)
print("Did de.stop")
sys.stdout.flush()
stats.save()
def run_primary(addr, authkey, generations):
# Determine path to configuration file. This path manipulation is
# here so that the script will run successfully regardless of the
# current working directory.
local_dir = os.path.dirname(__file__)
config_path = os.path.join(local_dir, 'test_configuration2')
# Load configuration.
config = neat.Config(neat.DefaultGenome, neat.DefaultReproduction,
neat.DefaultSpeciesSet, neat.DefaultStagnation,
config_path)
# Create the population, which is the top-level object for a NEAT run.
p = neat.Population(config)
# Add a stdout reporter to show progress in the terminal.
p.add_reporter(neat.StdOutReporter(True))
stats = neat.StatisticsReporter()
p.add_reporter(stats)
checkpointer = neat.Checkpointer(max(1, int(generations / 4)), 10)
p.add_reporter(checkpointer)
# Run for the specified number of generations.
winner = None
de = neat.DistributedEvaluator(
addr,
authkey=authkey,
eval_function=eval_genome_distributed,
secondary_chunksize=15,
mode=MODE_PRIMARY,
)
de.start()
winner = p.run(de.evaluate, generations)
print("===== stopping DistributedEvaluator =====")
de.stop(wait=3, shutdown=True, force_secondary_shutdown=False)
print("===== DistributedEvaluator stopped. =====")
if winner:
# Display the winning genome.
print('\nBest genome:\n{!s}'.format(winner))
# Show output of the most fit genome against training data.
print('\nOutput:')
winner_net = neat.nn.FeedForwardNetwork.create(winner, config)
for xi, xo in zip(XOR_INPUTS, XOR_OUTPUTS):
output = winner_net.activate(xi)
print("input {!r}, expected output {!r}, got {!r}".format(xi, xo, output))
if (checkpointer.last_generation_checkpoint >= 0) and (checkpointer.last_generation_checkpoint < 100):
filename = 'neat-checkpoint-{0}'.format(checkpointer.last_generation_checkpoint)
print("Restoring from {!s}".format(filename))
p2 = neat.checkpoint.Checkpointer.restore_checkpoint(filename)
p2.add_reporter(neat.StdOutReporter(True))
stats2 = neat.StatisticsReporter()
p2.add_reporter(stats2)
winner2 = None
time.sleep(3)
de.start()
winner2 = p2.run(de.evaluate, (100-checkpointer.last_generation_checkpoint))
winner2 = p2.run(de.evaluate, (100 - checkpointer.last_generation_checkpoint))
print("===== stopping DistributedEvaluator (forced) =====")
de.stop(wait=3, shutdown=True, force_secondary_shutdown=True)
print("===== DistributedEvaluator stopped. =====")
if winner2:
if not winner:
raise Exception("Had winner2 without first-try winner")
elif winner:
raise Exception("Had first-try winner without winner2")
def run(config_file, addr, authkey, mode, workers):
# Load configuration.
config = neat.Config(neat.DefaultGenome, neat.DefaultReproduction,
neat.DefaultSpeciesSet, neat.DefaultStagnation,
config_file)
# Create the population, which is the top-level object for a NEAT run.
p = neat.Population(config)
# Add a stdout reporter to show progress in the terminal.
p.add_reporter(neat.StdOutReporter(True))
stats = neat.StatisticsReporter()
p.add_reporter(stats)
# setup an DistributedEvaluator
de = neat.DistributedEvaluator(
addr, # connect to addr
authkey, # use authkey to authenticate
eval_genome, # use eval_genome() to evaluate a genome
secondary_chunksize=4, # send 4 genomes at once
num_workers=workers, # when in secondary mode, use this many workers
worker_timeout=10, # when in secondary mode and workers > 1,
# wait at most 10 seconds for the result
mode=mode, # wether this is the primary or a secondary node
# in most case you can simply pass
# 'neat.distributed.MODE_AUTO' as the mode.
# This causes the DistributedEvaluator to
# determine the mode by checking if address
# points to the localhost.
)
# start the DistributedEvaluator
de.start(
exit_on_stop=True, # if this is a secondary node, call sys.exit(0) when
# when finished. All code after this line will only
# be executed by the primary node.
secondary_wait=
3, # when a secondary, sleep this many seconds before continuing
# this is useful when the primary node may need more time
# to start than the secondary nodes.
)
# Run for up to 500 generations.
winner = p.run(de.evaluate, 500)
# stop evaluator
de.stop()
# Display the winning genome.
print('\nBest genome:\n{!s}'.format(winner))
# Show output of the most fit genome against training data.
print('\nOutput:')
winner_net = neat.nn.FeedForwardNetwork.create(winner, config)
for xi, xo in zip(xor_inputs, xor_outputs):
output = winner_net.activate(xi)
print("input {!r}, expected output {!r}, got {!r}".format(
xi, xo, output))
node_names = {-1: 'A', -2: 'B', 0: 'A XOR B'}
visualize.draw_net(config, winner, True, node_names=node_names)
visualize.plot_stats(stats, ylog=False, view=True)
visualize.plot_species(stats, view=True)
def run(config_file, addr, authkey, mode, workers, chunk):
print('Chunk for', chunk, 'genomes')
# Load configuration.
config = neat.Config(neat.DefaultGenome, neat.DefaultReproduction,
neat.DefaultSpeciesSet, neat.DefaultStagnation,
config_file)
# Create the population, which is the top-level object for a NEAT run.
p = neat.Population(config)
# Add a stdout reporter to show progress in the terminal.
p.add_reporter(neat.StdOutReporter(True))
stats = neat.StatisticsReporter()
p.add_reporter(stats)
p.add_reporter(
neat.Checkpointer(generation_interval=100,
filename_prefix='checkpoints_ctrnn/chk_'))
# setup an DistributedEvaluator
de = neat.DistributedEvaluator(
addr, # connect to addr
authkey, # use authkey to authenticate
eval_genome, # use eval_genome() to evaluate a genome
secondary_chunksize=chunk, # send 4 genomes at once
num_workers=workers, # when in secondary mode, use this many workers
worker_timeout=10, # when in secondary mode and workers > 1,
# wait at most 10 seconds for the result
mode=mode, # wether this is the primary or a secondary node
# in most case you can simply pass
# 'neat.distributed.MODE_AUTO' as the mode.
# This causes the DistributedEvaluator to
# determine the mode by checking if address
# points to the localhost.
)
# start the DistributedEvaluator
de.start(
exit_on_stop=True, # if this is a secondary node, call sys.exit(0) when
# when finished. All code after this line will only
# be executed by the primary node.
secondary_wait=
3, # when a secondary, sleep this many seconds before continuing
# this is useful when the primary node may need more time
# to start than the secondary nodes.
)
winner = p.run(de.evaluate)
# stop evaluator
de.stop()
# Display the winning genome.
print('\nBest genome:\n{!s}'.format(winner))
# Save the winner.
with open('winner-ctrnn', 'wb') as f:
pickle.dump(winner, f)
visualize.plot_stats(stats,
view=False,
ylog=True,
filename="pictures_ctrnn/feedforward-fitness.svg")
visualize.plot_species(
stats,
view=False,
filename="pictures_ctrnn/feedforward-speciation.svg")
node_names = {
-1: 'ext',
-2: 'eyt',
-3: 'sf',
-4: 'sl',
-5: 'sr',
0: 'ux',
1: 'uy',
2: 'wz'
}
visualize.draw_net(config,
winner,
False,
node_names=node_names,
filename='pictures_ctrnn/Digraph.gv')
visualize.draw_net(config,
winner,
view=False,
node_names=node_names,
filename="pictures_ctrnn/winner-feedforward.gv")
visualize.draw_net(config,
winner,
view=False,
node_names=node_names,
filename="pictures_ctrnn/winner-feedforward-enabled.gv",
show_disabled=False)
