def test_bidir_chain():
'''
Tests the migration on a one way chain.
'''
from sabaody.topology import TopologyFactory
def make_problem():
import pygmo as pg
return pg.problem(pg.rosenbrock(3))
def make_algorithm():
return pg.de(gen=10)
domain_qual = partial(getQualifiedName,
'com.how2cell.sabaody.test_bidir_chain_migration')
topology_factory = TopologyFactory(make_problem, domain_qual, 'localhost',
11211)
topology = topology_factory.createBidirChain(make_algorithm, 5)
assert len(topology.island_ids) == 5
assert len(topology.endpoints) == 2
from sabaody.migration_central import CentralMigrator, start_migration_service
from sabaody.migration import BestSPolicy, FairRPolicy, sort_by_fitness
import pygmo as pg
try:
process = start_migration_service()
sleep(2)
migrator = CentralMigrator('http://localhost:10100',
BestSPolicy(migration_rate=1),
FairRPolicy())
from collections import OrderedDict
for k in (1, 2):
islands = OrderedDict((i.id,
pg.island(algo=i.algorithm_constructor(),
prob=i.problem_constructor(),
size=i.size))
for i in topology.islands)
for island_id in islands.keys():
migrator.defineMigrantPool(island_id, 3)
if k == 1:
# test forward migration
seed_first(islands, topology)
else:
# test reverse migration
seed_last(islands, topology)
for n in range(1, 5 + 1):
assert count_hits(islands.values()) == n
# perform migration
for island_id, i in islands.items():
migrator.sendMigrants(island_id, i, topology)
for island_id, i in islands.items():
deltas, src_ids = migrator.receiveMigrants(
island_id, i, topology)
migrator.purgeAll()
finally:
process.terminate()
def run_island(island, topology):
import pygmo as pg
from multiprocessing import cpu_count
from pymemcache.client.base import Client
from sabaody.migration import BestSPolicy, FairRPolicy
from sabaody.migration_central import CentralMigrator
mc_client = Client((island.mc_host, island.mc_port))
migrator = CentralMigrator('http://luna:10100')
algorithm = pg.de(gen=10)
problem = island.problem_constructor()
# TODO: configure pop size
i = pg.island(algo=algorithm, prob=problem, size=20)
mc_client.set(island.domain_qualifier('island', str(island.id), 'status'),
'Running', 10000)
mc_client.set(island.domain_qualifier('island', str(island.id), 'n_cores'),
str(cpu_count()), 10000)
rounds = 10
migration_log = []
for x in range(rounds):
i.evolve()
i.wait()
# perform migration
migrator.sendMigrants(island.id, i, topology)
deltas, src_ids = migrator.receiveMigrants(island.id, i, topology)
"""
For Kafka Migration Enable below
"""
#migrator.send_migrants(island.id,i,topology,generation=x)
#deltas,src_ids = migrator.receive_migrants(island.id,i,topology,generation=x)
migration_log.append((float(pop.champion_f[0]), deltas, src_ids))
import socket
hostname = socket.gethostname()
ip = [
l for l in ([
ip for ip in socket.gethostbyname_ex(socket.gethostname())[2]
if not ip.startswith("127.")
][:1], [[(s.connect(('8.8.8.8', 53)), s.getsockname()[0], s.close())
for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]
][0][1]]) if l
][0][0]
return (ip, hostname, island.id, migration_log,
i.get_population().problem.get_fevals())
def run(self, sc):
from sabaody.migration_central import CentralMigrator
migrator = CentralMigrator('http://luna:10100')
for island_id in self.island_ids:
migrator.defineMigrantPool(island_id, 5) # FIXME: hardcoded
#islands = sc.parallelize(self.island_ids).map(lambda u: Island(u, self.problem_factory, self.domain_qualifier, self.mc_host, self.mc_port))
islands = [
Island(u,
problem_factory=self.problem_factory,
domain_qualifier=self.domain_qualifier,
mc_host=self.mc_host,
mc_port=self.mc_port) for u in self.island_ids
]
#print(islands.map(lambda i: i.id).collect())
#print(islands.map(lambda i: i.run()).collect())
#from .worker import run_island
return sc.parallelize(topology.islands).map(run_island).collect()
def select_migrator(self, migrator_name, migration_policy,
selection_policy, replacement_policy):
if migrator_name == 'central' or migrator_name == 'central-migrator':
from sabaody.migration_central import CentralMigrator
# central migrator process must be running
return CentralMigrator(migration_policy, selection_policy,
replacement_policy,
'http://luna:10100') # FIXME: hardcoded
elif migrator_name == 'kafka' or migrator_name == 'kafka-migrator':
from sabaody.kafka_migration_service import KafkaMigrator, KafkaBuilder
# Kafka must be running
return KafkaMigrator(selection_policy, replacement_policy,
KafkaBuilder('luna',
9092)) # FIXME: hardcoded
else:
raise RuntimeError('Migration scheme undefined')
def test_migration_client(mocker):
'''
Test the client methods, including defineMigrantPool and pushMigrant.
requests.post is patched so post requests are never actually sent.
'''
mocker.patch('requests.post')
from sabaody.migration_central import CentralMigrator
# url doesn't matter, requests never sent
m = CentralMigrator(None, None, None, 'http://www.schneierfacts.com:10100')
island_id = uuid4()
m.defineMigrantPool(island_id, 4)
from requests import post
if sys.version_info >= (3, 6):
post.assert_called_once()
post.reset_mock()
m._migrate(island_id, array([1., 2., 3., 4.]), 1.)
if sys.version_info >= (3, 6):
post.assert_called_once()
post.reset_mock()
def test_migration_replacement_policy_integration():
'''
Test migration replacement policy.
'''
from sabaody.migration_central import CentralMigrator, start_migration_service
from sabaody.migration import BestSPolicy, FairRPolicy, sort_by_fitness
from sabaody.utils import arrays_equal
from pygmo import population, rosenbrock
try:
process = start_migration_service()
sleep(2)
m = CentralMigrator('http://localhost:10100', None, FairRPolicy())
island1 = uuid4()
island2 = uuid4()
m.defineMigrantPool(island1, 3)
m.defineMigrantPool(island2, 3)
# migrants to island 1
m.pushMigrant(island1, array([1., 1., 1.]), 1., 'manual1')
m.pushMigrant(island1, array([2., 2., 2.]), 2., 'manual1')
# population for island 1
p1 = population(prob=rosenbrock(3), size=0, seed=0)
p1.push_back(array([9., 0., 1.]), array([3.]))
p1.push_back(array([9., 0., 2.]), array([4.]))
# migrants to island 2
m.pushMigrant(island2, array([3., 3., 3.]), 3.5, 'manual2')
m.pushMigrant(island2, array([4., 4., 4.]), 4.5, 'manual2')
# population for island 2
p2 = population(prob=rosenbrock(3), size=0, seed=0)
p2.push_back(array([9., 9., 1.]), array([3.]))
p2.push_back(array([9., 9., 2.]), array([4.]))
migrants, fitness, src_island_id = m.pullMigrants(island1)
assert array_equal(migrants, array([
[2., 2., 2.],
[1., 1., 1.],
]))
assert array_equal(fitness, array([[2.], [1.]]))
assert src_island_id == ['manual1', 'manual1']
# some parts of the code use loops like this
# (zip-type looping with 2d arrays), so make sure it works
for candidate, f in zip(migrants, fitness):
assert float(f) in (1., 2.)
assert array_equal(candidate, array([2., 2., 2.])) or array_equal(
candidate, array([1., 1., 1.]))
# re-push the migrants
m.pushMigrant(island1, array([1., 1., 1.]), 1.)
m.pushMigrant(island1, array([2., 2., 2.]), 2.)
deltas, src_ids = m.replace(island1, p1)
assert array_equal(
sort_by_fitness(p1)[0], array([[1., 1., 1.], [2., 2., 2.]]))
assert deltas == [-3., -1.]
# test island 2
deltas, src_ids = m.replace(island2, p2)
assert array_equal(
sort_by_fitness(p2)[0], array([[9., 9., 1.], [3., 3., 3.]]))
finally:
process.terminate()
def test_one_way_ring_migration():
'''
Tests the migration on a one way ring.
'''
from sabaody.topology import TopologyFactory
def make_problem():
import pygmo as pg
return pg.problem(pg.rosenbrock(3))
def make_algorithm():
return pg.de(gen=10)
domain_qual = partial(getQualifiedName,
'com.how2cell.sabaody.test_one_way_ring_migration')
topology_factory = TopologyFactory(make_problem, domain_qual, 'localhost',
11211)
topology = topology_factory.createOneWayRing(make_algorithm, 5)
assert len(topology.island_ids) == 5
from sabaody.migration_central import CentralMigrator, start_migration_service
from sabaody.migration import BestSPolicy, FairRPolicy, sort_by_fitness
import pygmo as pg
try:
process = start_migration_service()
sleep(2)
migrator = CentralMigrator('http://localhost:10100',
BestSPolicy(migration_rate=1),
FairRPolicy())
from collections import OrderedDict
islands = OrderedDict((i.id,
pg.island(algo=i.algorithm_constructor(),
prob=i.problem_constructor(),
size=i.size))
for i in topology.islands)
for island_id in islands.keys():
migrator.defineMigrantPool(island_id, 3)
# seed solution in one island
seed_first(islands, topology)
assert count_hits(islands.values()) == 1
for n in range(1, 5 + 1):
assert count_hits(islands.values()) == n
# perform migration
for island_id, i in islands.items():
migrator.sendMigrants(island_id, i, topology)
for island_id, i in islands.items():
deltas, src_ids = migrator.receiveMigrants(
island_id, i, topology)
assert n == 5
# reset & differentiate from chain
islands = OrderedDict((i.id,
pg.island(algo=i.algorithm_constructor(),
prob=i.problem_constructor(),
size=i.size))
for i in topology.islands)
seed_predecessor(islands, topology)
# perform migration
for island_id, i in islands.items():
migrator.sendMigrants(island_id, i, topology)
for island_id, i in islands.items():
deltas, src_ids = migrator.receiveMigrants(island_id, i, topology)
assert tuple(islands.values())[0].get_population().champion_f[0] == 0.
assert count_hits(islands.values()) == 2
finally:
process.terminate()
# Sabaody
# Copyright 2018 J Kyle Medley
from __future__ import print_function, division, absolute_import
from sabaody.migration_central import CentralMigrator
from toolz import partial
from numpy import array
from pygmo import population, rosenbrock
from uuid import uuid4
m = CentralMigrator('http://localhost:10100')
island1 = uuid4()
island2 = uuid4()
m.defineMigrantPool(island1, 3)
m.defineMigrantPool(island2, 3)
# migrants to island 1
m.pushMigrant(island1, array([1., 1., 1.]), 1.)
m.pushMigrant(island1, array([2., 2., 2.]), 2.)
# population for island 1
p1 = population(prob=rosenbrock(3), size=0, seed=0)
p1.push_back(array([9., 0., 1.]), array([3.]))
p1.push_back(array([9., 0., 2.]), array([4.]))
# migrants to island 2
m.pushMigrant(island2, array([3., 3., 3.]), 3.)
m.pushMigrant(island2, array([4., 4., 4.]), 4.)