def __init__(self, number_of_islands=None, *args, **kwargs):
assert isinstance(number_of_islands, int)
super(AbstractParallelObserver, self).__init__()
self.queue = RedisQueue(name=str(uuid4()), namespace=self.__name__)
self.clients = {}
self.run = True
for i in range(number_of_islands):
self._create_client(i)
class MultiprocessingMigrator(object):
"""Migrate among processes on the same machine.
This callable class allows individuals to migrate from one process
to another on the same machine. It maintains a queue of migrants
whose maximum length can be fixed via the ``max_migrants``
parameter in the constructor. If the number of migrants in the queue
reaches this value, new migrants are not added until earlier ones
are consumed. The unreliability of a multiprocessing environment
makes it difficult to provide guarantees. However, migrants are
theoretically added and consumed at the same rate, so this value
should determine the "freshness" of individuals, where smaller
queue sizes provide more recency.
An optional keyword argument in ``args`` requires the migrant to be
evaluated by the current evolutionary computation before being inserted
into the population. This can be important when different populations
use different evaluation functions and you need to be able to compare
"apples with apples," so to speak.
The migration takes the current individual *I* out of the queue, if
one exists. It then randomly chooses an individual *E* from the population
to insert into the queue. Finally, if *I* exists, it replaces *E* in the
population (re-evaluating fitness if necessary). Otherwise, *E* remains in
the population and also exists in the queue as a migrant.
Optional keyword arguments in args:
- *evaluate_migrant* -- should new migrants be evaluated before
adding them to the population (default False)
"""
def __init__(self, max_migrants=1):
self.max_migrants = max_migrants
self.migrants = RedisQueue(uuid4())
self.__name__ = self.__class__.__name__
def __call__(self, random, population, args):
evaluate_migrant = args.setdefault('evaluate_migrant', False)
migrant_index = random.randint(0, len(population) - 1)
old_migrant = population[migrant_index]
try:
migrant = self.migrants.get(block=False)
if evaluate_migrant:
fit = args["_ec"].evaluator([migrant.candidate], args)
migrant.fitness = fit[0]
args["_ec"].num_evaluations += 1
population[migrant_index] = migrant
except six.moves.queue.Empty:
pass
try:
self.migrants.put(old_migrant, block=False)
except six.moves.queue.Full:
pass
return population
def test_queue_size(self):
queue = RedisQueue("test-queue-size-1", maxsize=1)
queue.put(1)
self.assertRaises(six.moves.queue.Full, queue.put, 1)
queue = RedisQueue("test-queue-size-2", maxsize=2)
queue.put(1)
queue.put(1)
self.assertRaises(six.moves.queue.Full, queue.put, 1)
queue.get()
queue.get()
self.assertRaises(six.moves.queue.Empty, queue.get_nowait)
def test_queue_size(self):
print(REDIS_HOST)
print(os.getenv('REDIS_PORT_6379_TCP_ADDR'))
queue = RedisQueue("test-queue-size-1", maxsize=1, host=REDIS_HOST)
queue.put(1)
self.assertRaises(six.moves.queue.Full, queue.put, 1)
queue = RedisQueue("test-queue-size-2", maxsize=2, host=REDIS_HOST)
queue.put(1)
queue.put(1)
self.assertRaises(six.moves.queue.Full, queue.put, 1)
queue.get()
queue.get()
self.assertRaises(six.moves.queue.Empty, queue.get_nowait)
def test_queue_objects(self):
queue = RedisQueue("test-queue", maxsize=100, host=REDIS_HOST)
# put int
queue.put(1)
v = queue.get_nowait()
self.assertEqual(v, 1)
self.assertIsInstance(v, int)
# put str
queue.put("a")
v = queue.get_nowait()
self.assertEqual(v, "a")
self.assertIsInstance(v, str)
# put float
queue.put(1.)
v = queue.get_nowait()
self.assertEqual(v, 1.)
self.assertIsInstance(v, float)
# put list
queue.put([1, 3, 4, 5, "a", "b", "c", 1., 2., 3.])
v = queue.get_nowait()
self.assertEqual(v, [1, 3, 4, 5, "a", "b", "c", 1., 2., 3.])
self.assertIsInstance(v, list)
# put dict
queue.put({"x": "y"})
v = queue.get_nowait()
self.assertEqual(v, {"x": "y"})
self.assertIsInstance(v, dict)
def test_queue_objects(self):
queue = RedisQueue("test-queue", maxsize=100, host=REDIS_HOST)
# put int
queue.put(1)
v = queue.get_nowait()
assert v == 1
assert isinstance(v, int)
# put str
queue.put("a")
v = queue.get_nowait()
assert v == "a"
assert isinstance(v, str)
# put float
queue.put(1.)
v = queue.get_nowait()
assert v == 1.
assert isinstance(v, float)
# put list
queue.put([1, 3, 4, 5, "a", "b", "c", 1., 2., 3.])
v = queue.get_nowait()
assert v == [1, 3, 4, 5, "a", "b", "c", 1., 2., 3.]
assert isinstance(v, list)
# put dict
queue.put({"x": "y"})
v = queue.get_nowait()
assert v == {"x": "y"}
assert isinstance(v, dict)
def test_queue_size(self):
print(REDIS_HOST)
print(os.getenv('REDIS_PORT_6379_TCP_ADDR'))
queue = RedisQueue("test-queue-size-1", maxsize=1, host=REDIS_HOST)
queue.put(1)
with pytest.raises(six.moves.queue.Full):
queue.put(1)
queue = RedisQueue("test-queue-size-2", maxsize=2, host=REDIS_HOST)
queue.put(1)
queue.put(1)
with pytest.raises(six.moves.queue.Full):
queue.put(1)
queue.get()
queue.get()
with pytest.raises(six.moves.queue.Empty):
queue.get_nowait()
def test_queue_len(self):
queue = RedisQueue("test-queue-len", maxsize=100, host=REDIS_HOST)
self.assertEqual(queue.length, 0)
queue.put(1)
self.assertEqual(queue.length, 1)
queue.put(1)
self.assertEqual(queue.length, 2)
queue.put(1)
self.assertEqual(queue.length, 3)
queue.get_nowait()
self.assertEqual(queue.length, 2)
queue.get_nowait()
self.assertEqual(queue.length, 1)
queue.get_nowait()
self.assertEqual(queue.length, 0)
def test_queue_len(self):
queue = RedisQueue("test-queue-len", maxsize=100, host=REDIS_HOST)
assert queue.length == 0
queue.put(1)
assert queue.length == 1
queue.put(1)
assert queue.length == 2
queue.put(1)
assert queue.length == 3
queue.get_nowait()
assert queue.length == 2
queue.get_nowait()
assert queue.length == 1
queue.get_nowait()
assert queue.length == 0
class AbstractParallelObserver(object):
def __init__(self, number_of_islands=None, *args, **kwargs):
assert isinstance(number_of_islands, int)
super(AbstractParallelObserver, self).__init__()
self.queue = RedisQueue(name=str(uuid4()), namespace=self.__name__)
self.clients = {}
self.run = True
self.t = None
for i in range(number_of_islands):
self._create_client(i)
def _create_client(self, i):
raise NotImplementedError
def _listen(self):
print("Start %s" % self.__name__)
while self.run:
try:
message = self.queue.get_nowait()
self._process_message(message)
except Empty:
pass
except Exception as e:
print(e)
print("Exit %s" % self.__name__)
def _process_message(self, message):
raise NotImplementedError
def start(self):
"""
Starts the observer. It is called internally before the optimization starts.
The observer will not report anything until start has been called.
"""
self.run = True
self.t = Thread(target=self._listen)
self.t.start()
def finish(self):
"""
Stops the observer. The observer will not report anything else from the optimization.
"""
self.run = False
class AbstractParallelObserver(object):
def __init__(self, number_of_islands=None, *args, **kwargs):
assert isinstance(number_of_islands, int)
super(AbstractParallelObserver, self).__init__()
self.queue = RedisQueue(name=str(uuid4()), namespace=self.__name__)
self.clients = {}
self.run = True
self.t = None
for i in range(number_of_islands):
self._create_client(i)
def _create_client(self, i):
raise NotImplementedError
def _listen(self):
print("Start %s" % self.__name__)
while self.run:
try:
message = self.queue.get_nowait()
self._process_message(message)
except Empty:
pass
except Exception as e:
print(e)
print("Exit %s" % self.__name__)
def _process_message(self, message):
raise NotImplementedError
def start(self):
"""
Starts the observer. It is called internally before the optimization starts.
The observer will not report anything until start has been called.
"""
self.run = True
self.t = Thread(target=self._listen)
self.t.start()
def finish(self):
"""
Stops the observer. The observer will not report anything else from the optimization.
"""
self.run = False
class AbstractParallelObserver(object):
def __init__(self, number_of_islands=None, *args, **kwargs):
assert isinstance(number_of_islands, int)
super(AbstractParallelObserver, self).__init__()
self.queue = RedisQueue(name=str(uuid4()), namespace=self.__name__)
self.clients = {}
self.run = True
for i in range(number_of_islands):
self._create_client(i)
def _create_client(self, i):
raise NotImplementedError
def _listen(self):
print("Start %s" % self.__name__)
while self.run:
try:
message = self.queue.get_nowait()
self._process_message(message)
except Empty:
pass
except Exception as e:
print(e)
print("Exit %s" % self.__name__)
def _process_message(self, message):
raise NotImplementedError
def start(self):
self.run = True
self.t = Thread(target=self._listen)
self.t.start()
def finish(self):
self.run = False
def __init__(self, max_migrants=1, **connection_kwargs):
self.max_migrants = max_migrants
self.migrants = RedisQueue(uuid4(), **connection_kwargs)
self.__name__ = self.__class__.__name__
def __init__(self, max_migrants=1):
self.max_migrants = max_migrants
self.migrants = RedisQueue(uuid4())
self.__name__ = self.__class__.__name__
def __init__(self, max_migrants=1, **connection_kwargs):
self.max_migrants = max_migrants
self.migrants = RedisQueue(uuid4(), **connection_kwargs)
self.__name__ = self.__class__.__name__
class MultiprocessingMigrator(object):
"""Migrate among processes on multiple machines. This is possible by
having a Queue implemented on redis.
This callable class allows individuals to migrate from one process
to another. It maintains a queue of migrants whose maximum length
can be fixed via the ``max_migrants`` parameter in the constructor.
If the number of migrants in the queue reaches this value, new migrants
are not added until earlier ones are consumed. The unreliability of a
multiprocessing environment makes it difficult to provide guarantees.
However, migrants are theoretically added and consumed at the same rate,
so this value should determine the "freshness" of individuals, where
smaller queue sizes provide more recency.
An optional keyword argument in ``args`` requires the migrant to be
evaluated by the current evolutionary computation before being inserted
into the population. This can be important when different populations
use different evaluation functions and you need to be able to compare
"apples with apples," so to speak.
The migration takes the current individual *I* out of the queue, if
one exists. It then randomly chooses an individual *E* from the population
to insert into the queue. Finally, if *I* exists, it replaces *E* in the
population (re-evaluating fitness if necessary). Otherwise, *E* remains in
the population and also exists in the queue as a migrant.
Optional keyword arguments in args:
- *evaluate_migrant* -- should new migrants be evaluated before
adding them to the population (default False)
Arguments
---------
max_migrants: int
Number of migrants in the queue at the same time.
connection_kwargs: keyword arguments:
see parallel.RedisQueue
"""
def __init__(self, max_migrants=1, **connection_kwargs):
self.max_migrants = max_migrants
self.migrants = RedisQueue(uuid4(), **connection_kwargs)
self.__name__ = self.__class__.__name__
def __call__(self, random, population, args):
evaluate_migrant = args.setdefault('evaluate_migrant', False)
migrant_index = random.randint(0, len(population) - 1)
old_migrant = population[migrant_index]
try:
migrant = self.migrants.get(block=False)
logger.debug("Robinson Crusoe arrives on an island")
if evaluate_migrant:
fit = args["_ec"].evaluator([migrant.candidate], args)
migrant.fitness = fit[0]
args["_ec"].num_evaluations += 1
population[migrant_index] = migrant
except six.moves.queue.Empty:
logger.debug("Empty queue")
try:
logger.debug("Robinson Crusoe leaves an island")
self.migrants.put_nowait(old_migrant)
except six.moves.queue.Full:
logger.debug("Full queue")
return population
