def test_non_population_based_migration():
import pygmo as pg
from pygmo import de, nlopt, rosenbrock
from sabaody.topology import TopologyFactory
topology_factory = TopologyFactory(island_size=5, migrant_pool_size=5)
topology = topology_factory.createBidirChain(de, number_of_islands=2)
assert len(topology.island_ids) == 2
assert len(topology.endpoints) == 2
de_island = pg.island(algo=de(gen=10),
prob=rosenbrock(3),
size=topology.islands[0].size)
nm = nlopt('neldermead')
nm.selection = 'random'
nm.replacement = 'random'
nm_island = pg.island(algo=nm,
prob=rosenbrock(3),
size=topology.islands[1].size)
from sabaody.migration import BestSPolicy, FairRPolicy
selection_policy = BestSPolicy(migration_rate=2)
replacement_policy = FairRPolicy()
from sabaody.migration import MigrationPolicyEachToAll
migration_policy = MigrationPolicyEachToAll()
# get the candidates from the de island
p_de = de_island.get_population()
candidates, candidate_f = selection_policy.select(p_de)
# try migrating to the nelder mead island
p_nm = nm_island.get_population()
replacement_policy.replace(p_nm, candidates, candidate_f)
nm_island.set_population(p_nm)
# finally, try to evolve it
new_pop = nm_island.evolve(n=10)
def run_island(island):
import pygmo as pg
from multiprocessing import cpu_count
from pymemcache.client.base import Client
mc_client = Client((island.mc_host, island.mc_port))
#udp = island.problem_factory()
algorithm = pg.algorithm(pg.de())
#problem = pg.problem(udp)
problem = island.problem_factory()
# TODO: configure pop size
i = pg.island(algo=algorithm, prob=problem, size=20)
mc_client.set(island.domain_qualifier('island', str(island.id), 'status'),
'Running', 10000)
mc_client.set(island.domain_qualifier('island', str(island.id), 'n_cores'),
str(cpu_count()), 10000)
#print('Starting island {} with {} cpus'.format(str(i.id), str(cpu_count())))
i.evolve()
i.wait()
import socket
hostname = socket.gethostname()
ip = [
l for l in ([
ip for ip in socket.gethostbyname_ex(socket.gethostname())[2]
if not ip.startswith("127.")
][:1], [[(s.connect(('8.8.8.8', 53)), s.getsockname()[0], s.close())
for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]
][0][1]]) if l
][0][0]
return (ip, hostname, i.get_population().problem.get_fevals())
def island_run(self):
algo = pg.algorithm(pg.pso(gen=self.generation))
pop = pg.population(self.problem, self.pop_size)
island = pg.island(algo=algo, pop=pop, udi=pg.mp_island())
island.evolve()
island.wait()
pop = island.get_population()
return pop.champion_f, pop.champion_x
def test_bidir_chain():
'''
Tests the migration on a one way chain.
'''
from sabaody.topology import TopologyFactory
def make_problem():
import pygmo as pg
return pg.problem(pg.rosenbrock(3))
def make_algorithm():
return pg.de(gen=10)
domain_qual = partial(getQualifiedName,
'com.how2cell.sabaody.test_bidir_chain_migration')
topology_factory = TopologyFactory(make_problem, domain_qual, 'localhost',
11211)
topology = topology_factory.createBidirChain(make_algorithm, 5)
assert len(topology.island_ids) == 5
assert len(topology.endpoints) == 2
from sabaody.migration_central import CentralMigrator, start_migration_service
from sabaody.migration import BestSPolicy, FairRPolicy, sort_by_fitness
import pygmo as pg
try:
process = start_migration_service()
sleep(2)
migrator = CentralMigrator('http://localhost:10100',
BestSPolicy(migration_rate=1),
FairRPolicy())
from collections import OrderedDict
for k in (1, 2):
islands = OrderedDict((i.id,
pg.island(algo=i.algorithm_constructor(),
prob=i.problem_constructor(),
size=i.size))
for i in topology.islands)
for island_id in islands.keys():
migrator.defineMigrantPool(island_id, 3)
if k == 1:
# test forward migration
seed_first(islands, topology)
else:
# test reverse migration
seed_last(islands, topology)
for n in range(1, 5 + 1):
assert count_hits(islands.values()) == n
# perform migration
for island_id, i in islands.items():
migrator.sendMigrants(island_id, i, topology)
for island_id, i in islands.items():
deltas, src_ids = migrator.receiveMigrants(
island_id, i, topology)
migrator.purgeAll()
finally:
process.terminate()
def test_pygmo_single_generation(two_reservoir_wrapper):
""" Simple pygmo wrapper test. """
wrapper = two_reservoir_wrapper
prob = pg.problem(wrapper)
algo = pg.algorithm(pg.moead(gen=1))
pg.mp_island.init_pool(2)
isl = pg.island(algo=algo, prob=prob, size=50, udi=pg.mp_island())
isl.evolve(1)
def addIsland(arc, algo):
isl = pg.island(
udi=pg.mp_island(),
algo=algo,
prob=prob,
#b=pg.mp_bfe(),
b=pg.bfe(udbfe=multi_bfre2()),
size=16)
arc.push_back(isl)
def get_island(evaluate, params, hooks):
# config
# D = 8 decision space dimension
# N0 = 100 initial population
# Ng = 100 total generation
D, Ng, N0 = itemgetter('D', 'Ng', 'N0')(params)
algo = pg.algorithm(pg.de(gen=Ng))
algo.set_verbosity(int(Ng / 10))
prob = pg.problem(evaluate_wrapper(D, evaluate))
island = pg.island(algo=algo, prob=prob, size=N0, udi=pg.mp_island())
return island
def run_island(island, topology):
import pygmo as pg
from multiprocessing import cpu_count
from pymemcache.client.base import Client
from sabaody.migration import BestSPolicy, FairRPolicy
from sabaody.migration_central import CentralMigrator
mc_client = Client((island.mc_host, island.mc_port))
migrator = CentralMigrator('http://luna:10100')
algorithm = pg.de(gen=10)
problem = island.problem_constructor()
# TODO: configure pop size
i = pg.island(algo=algorithm, prob=problem, size=20)
mc_client.set(island.domain_qualifier('island', str(island.id), 'status'),
'Running', 10000)
mc_client.set(island.domain_qualifier('island', str(island.id), 'n_cores'),
str(cpu_count()), 10000)
rounds = 10
migration_log = []
for x in range(rounds):
i.evolve()
i.wait()
# perform migration
migrator.sendMigrants(island.id, i, topology)
deltas, src_ids = migrator.receiveMigrants(island.id, i, topology)
"""
For Kafka Migration Enable below
"""
#migrator.send_migrants(island.id,i,topology,generation=x)
#deltas,src_ids = migrator.receive_migrants(island.id,i,topology,generation=x)
migration_log.append((float(pop.champion_f[0]), deltas, src_ids))
import socket
hostname = socket.gethostname()
ip = [
l for l in ([
ip for ip in socket.gethostbyname_ex(socket.gethostname())[2]
if not ip.startswith("127.")
][:1], [[(s.connect(('8.8.8.8', 53)), s.getsockname()[0], s.close())
for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]
][0][1]]) if l
][0][0]
return (ip, hostname, island.id, migration_log,
i.get_population().problem.get_fevals())
def archipelago():
udp = solo_mgar_udp([7000, 8000])
#uda = pg.sga(gen = 6000)
uda = pg.sade(memory=True, variant=1, gen=6000)
# instantiate an unconnected archipelago
for _ in range(1000):
archi = pg.archipelago(t=pg.topologies.unconnected())
for _ in range(32):
alg = pg.algorithm(uda)
#alg.set_verbosity(1)
prob = pg.problem(udp)
pop = pg.population(prob, 20)
isl = pg.island(algo=alg, pop=pop)
archi.push_back(isl)
archi.evolve()
archi.wait_check()
def archipelago():
print('interferometer sga archipelago')
uda = pg.sga(gen=50000)
# instantiate an unconnected archipelago
archi = pg.archipelago(t=pg.topologies.unconnected())
t = time()
for _ in range(8):
alg = pg.algorithm(uda)
#alg.set_verbosity(1)
prob = pg.problem(udp)
pop = pg.population(prob, 20)
isl = pg.island(algo=alg, pop=pop)
archi.push_back(isl)
archi.evolve()
archi.wait_check()
print(f'archi: {time() - t:0.3f}s')
def pygmo_main(harmonic=False):
import pygmo as pg
from pywr.optimisation.pygmo import PygmoWrapper
def update_archive(pop, archive=None):
if archive is None:
combined_f = pop.get_f()
else:
combined_f = np.r_[archive, pop.get_f()]
indices = pg.select_best_N_mo(combined_f, 50)
new_archive = combined_f[indices, :]
new_archive = np.unique(new_archive.round(4), axis=0)
return new_archive
wrapper = PygmoWrapper(get_model_data(harmonic=harmonic))
prob = pg.problem(wrapper)
print(prob)
algo = pg.algorithm(pg.moead(gen=1))
# algo = pg.algorithm(pg.nsga2(gen=1))
pg.mp_island.init_pool(2)
isl = pg.island(algo=algo, prob=prob, size=50, udi=pg.mp_island())
ref_point = [216500, 4000]
pop = isl.get_population()
print("Evolving!")
archive = update_archive(pop)
hv = pg.hypervolume(archive)
vol = hv.compute(ref_point)
hvs = [
vol,
]
print(
"Gen: {:03d}, Hypervolume: {:6.4g}, Archive size: {:d}".format(
0, vol, len(archive)
)
)
for gen in range(20):
isl.evolve(1)
isl.wait_check()
pop = isl.get_population()
archive = update_archive(pop, archive)
hv = pg.hypervolume(archive)
vol = hv.compute(ref_point)
print(
"Gen: {:03d}, Hypervolume: {:6.4g}, Archive size: {:d}".format(
gen + 1, vol, len(archive)
)
)
hvs.append(vol)
hvs = pd.Series(hvs)
print("Finished!")
plt.scatter(archive[:, 0], archive[:, 1])
objectives = wrapper.model_objectives
plt.xlabel(objectives[0].name)
plt.ylabel(objectives[1].name)
plt.grid(True)
title = "Harmonic Control Curve" if harmonic else "Monthly Control Curve"
plt.savefig("{} Example ({}).pdf".format("pygmo", title), format="pdf")
fig, ax = plt.subplots()
ax.plot(hvs / 1e6, marker="o")
ax.grid(True)
ax.set_ylabel("Hypervolume")
ax.set_xlabel("Generation")
plt.savefig("{} Example Hypervolume ({}).pdf".format("pygmo", title), format="pdf")
plt.show()
def run_island(id):
class B2_UDP:
def __init__(self, lb, ub):
# type: (Evaluator, array, array) -> None
'''
Inits the problem with an objective evaluator
(implementing the method evaluate), the parameter
vector lower bound (a numpy array) and upper bound.
Both bounds must have the same dimension.
'''
from sabaody.utils import check_vector, expect
check_vector(lb)
check_vector(ub)
expect(len(lb) == len(ub), 'Bounds mismatch')
self.lb = lb
self.ub = ub
from b2problem import B2Problem
self.evaluator = B2Problem('b2.xml')
def fitness(self, x):
return (self.evaluator.evaluate(x), )
def get_bounds(self):
return (self.lb, self.ub)
def get_name(self):
return 'Sabaody udp'
def get_extra_info(self):
return 'Sabaody extra info'
def __getstate__(self):
return {'lb': self.lb, 'ub': self.ub}
def __setstate__(self, state):
self.lb = state['lb']
self.ub = state['ub']
from b2problem import B2Problem
self.evaluator = B2Problem('b2.xml')
import pygmo as pg
from pymemcache.client.base import Client
mc_client = Client(('luna', 11211))
algorithm = pg.algorithm(pg.de(gen=1000))
from params import getLowerBound, getUpperBound
problem = pg.problem(B2_UDP(getLowerBound(), getUpperBound()))
i = pg.island(algo=algorithm, prob=problem, size=20)
#mc_client.set(id.domain_qualifier('island', str(id), 'status'), 'Running', 10000)
i.evolve()
i.wait()
import socket
hostname = socket.gethostname()
ip = [
l for l in ([
ip for ip in socket.gethostbyname_ex(socket.gethostname())[2]
if not ip.startswith("127.")
][:1], [[(s.connect(('8.8.8.8', 53)), s.getsockname()[0], s.close())
for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]
][0][1]]) if l
][0][0]
return (ip, hostname)
def create_archipelago(
unknowns: list,
optimizers: list,
optimizers_kwargs: list,
pg_problem: pygmo.problem,
rel_pop_size: float,
archipelago_kwargs: dict,
log_each_nth_gen: int,
report_level: int,
) -> PyfoombArchipelago:
"""
Helper method for parallelized estimation using the generalized island model.
Creates the archipelago object for running several rounds of evolutions.
Arguments
---------
unknowns : list
The unknowns, sorted alphabetically and case-insensitive.
optimizers : list
A list of optimizers to be used on individual islands.
optimizers_kwargs : list
A list of corresponding kwargs.
pg_problem : pygmo.problem
An pygmo problem instance.
archipelago_kwargs : dict
Additional kwargs for archipelago creation.
log_each_nth_gen : int
Specifies at which each n-th generation the algorithm stores logs.
report_level : int
Prints information on the archipelago creation for values >= 1.
Returns
-------
archipelago : PyfoombArchipelago
"""
_cpus = joblib.cpu_count()
# There is one optimizer with a set of kwargs
if len(optimizers) == 1 and len(optimizers_kwargs) == 1:
optimizers = optimizers * _cpus
optimizers_kwargs = optimizers_kwargs * _cpus
# Several optimizers with the same kwargs
elif len(optimizers) > 1 and len(optimizers_kwargs) == 1:
optimizers_kwargs = optimizers_kwargs * len(optimizers)
# Several kwargs for the same optimizer
elif len(optimizers) == 1 and len(optimizers_kwargs) > 1:
optimizers = optimizers * len(optimizers_kwargs)
elif len(optimizers) != len(optimizers_kwargs):
raise ValueError(
'Number of optimizers does not match number of corresponding kwarg dicts'
)
# Get the optimizer intances
algos = [
PygmoOptimizers.get_optimizer_algo_instance(
name=_optimizers, kwargs=_optimizers_kwargs) for _optimizers,
_optimizers_kwargs in zip(optimizers, optimizers_kwargs)
]
# Update number of islands
n_islands = len(algos)
if report_level >= 1:
print(
f'Creating archipelago with {n_islands} islands. May take some time...'
)
pop_size = int(numpy.ceil(rel_pop_size * len(unknowns)))
prop_create_args = ((pg_problem, pop_size,
seed * numpy.random.randint(0, 1e4))
for seed, pop_size in enumerate([pop_size] *
n_islands))
try:
parallel_verbose = 0 if report_level == 0 else 1
with joblib.parallel_backend('loky', n_jobs=n_islands):
pops = joblib.Parallel(verbose=parallel_verbose)(map(
joblib.delayed(
ArchipelagoHelpers.parallel_create_population),
prop_create_args))
except Exception as ex:
print(
f'Parallelized archipelago creation failed, falling back to sequential\n{ex}'
)
pops = (
ArchipelagoHelpers.parallel_create_population(prop_create_arg)
for prop_create_arg in prop_create_args)
# Now create the empyty archipelago
if not 't' in archipelago_kwargs.keys():
archipelago_kwargs['t'] = pygmo.fully_connected()
archi = PyfoombArchipelago(**archipelago_kwargs)
archi.set_migrant_handling(pygmo.migrant_handling.preserve)
# Add the populations to the archipelago and wait for its construction
with contextlib.redirect_stdout(io.StringIO()):
for _pop, _algo in zip(pops, algos):
if log_each_nth_gen is not None:
_algo.set_verbosity(int(log_each_nth_gen))
_island = pygmo.island(algo=_algo,
pop=_pop,
udi=pygmo.mp_island())
archi.push_back(_island)
archi.wait_check()
return archi
from numpy import array, save, savez, mean, std, nan_to_num, vstack
from tempfile import gettempdir
from os.path import join
import json
single_champions = []
N = 100
for i in range(N):
with RBRun('luna', 11211) as run:
print('Started run {} of {}'.format(i + 1, N))
from rbsetup import make_problem
algorithm = pg.de(gen=10)
problem = make_problem()
i = pg.island(algo=algorithm, prob=problem, size=20)
rounds = 10
c = []
for x in range(rounds):
i.evolve()
i.wait()
c.append(float(i.get_population().champion_f[0]))
print('round {}'.format(x))
print(i.get_population().champion_f)
print(i.get_population().get_x()[0, :])
single_champions.append(array(c))
single_champions_stack = vstack(single_champions)
single_champions_mean = mean(single_champions_stack, axis=0)
single_champions_std = std(single_champions_stack, axis=0)
import pygmo as pg
"""
Okay, so the lesson learned here is: not sure :-D
"""
if __name__ == "__main__":
prob = pg.problem(pg.rosenbrock(dim=5))
pool_creator = pg.mp_island()
# pool_creator.resize_pool(1)
pool_creator.init_pool(1)
island = pg.island(udi=pool_creator,
algo=pg.sga(gen=200),
pop=pg.population(prob, 200))
island.evolve()
island.wait()
print("island: ***** \n", island)
import uda_basic
import pygmo
import pickle
ub = pygmo.algorithm(uda_basic.uda_basic())
assert pickle.dumps(pickle.loads(pickle.dumps(ub))) == pickle.dumps(ub)
isl = pygmo.island(algo=ub, prob=pygmo.rosenbrock(), size=20)
risl = repr(isl)
assert "Thread" in risl
isl.evolve()
isl.wait_check()
assert risl == repr(isl)
class py_udp(object):
def get_bounds(self):
return ([0, 0], [1, 1])
def fitness(self, a):
return [42]
if __name__ == '__main__':
isl = pygmo.island(algo=ub, prob=py_udp(), size=20)
isl.evolve()
isl.wait_check()
pygmo.mp_island.shutdown_pool()
pygmo.mp_bfe.shutdown_pool()
print("All good!")
def test_one_way_ring_migration():
'''
Tests the migration on a one way ring.
'''
from sabaody.topology import TopologyFactory
def make_problem():
import pygmo as pg
return pg.problem(pg.rosenbrock(3))
def make_algorithm():
return pg.de(gen=10)
domain_qual = partial(getQualifiedName,
'com.how2cell.sabaody.test_one_way_ring_migration')
topology_factory = TopologyFactory(make_problem, domain_qual, 'localhost',
11211)
topology = topology_factory.createOneWayRing(make_algorithm, 5)
assert len(topology.island_ids) == 5
from sabaody.migration_central import CentralMigrator, start_migration_service
from sabaody.migration import BestSPolicy, FairRPolicy, sort_by_fitness
import pygmo as pg
try:
process = start_migration_service()
sleep(2)
migrator = CentralMigrator('http://localhost:10100',
BestSPolicy(migration_rate=1),
FairRPolicy())
from collections import OrderedDict
islands = OrderedDict((i.id,
pg.island(algo=i.algorithm_constructor(),
prob=i.problem_constructor(),
size=i.size))
for i in topology.islands)
for island_id in islands.keys():
migrator.defineMigrantPool(island_id, 3)
# seed solution in one island
seed_first(islands, topology)
assert count_hits(islands.values()) == 1
for n in range(1, 5 + 1):
assert count_hits(islands.values()) == n
# perform migration
for island_id, i in islands.items():
migrator.sendMigrants(island_id, i, topology)
for island_id, i in islands.items():
deltas, src_ids = migrator.receiveMigrants(
island_id, i, topology)
assert n == 5
# reset & differentiate from chain
islands = OrderedDict((i.id,
pg.island(algo=i.algorithm_constructor(),
prob=i.problem_constructor(),
size=i.size))
for i in topology.islands)
seed_predecessor(islands, topology)
# perform migration
for island_id, i in islands.items():
migrator.sendMigrants(island_id, i, topology)
for island_id, i in islands.items():
deltas, src_ids = migrator.receiveMigrants(island_id, i, topology)
assert tuple(islands.values())[0].get_population().champion_f[0] == 0.
assert count_hits(islands.values()) == 2
finally:
process.terminate()
GA_based_optimize(pm,
cm,
prob_var.left_ind,
prob_var.mid_ind,
prob_var.right_ind,
gen=max_gen,
seed=rnd_seed,
log_lvl=log_lvl))
# Generating isl_num initial populations
pop_lst = [pg.population(prob) for _ in range(isl_num)]
[[p.push_back(next(GenerateIndividual)) for p in pop_lst]
for _ in range(pop_size)]
# Generating pm_lst × cm_lst algorithms with different crossover and
# mutation probability
alg_lst = [
pg.algorithm(
GA_based_optimize(p,
c,
prob_var.left_ind,
prob_var.mid_ind,
prob_var.right_ind,
gen=max_gen,
seed=rnd_seed,
log_lvl=log_lvl)) for c in cm_lst for p in pm_lst
]
# Creating islands
islands = [
pg.island(algo=a, pop=p, udi=pg.mp_island())
for a, p in zip(alg_lst, pop_lst)
]
import uda_basic
import udp_basic
import pygmo
import pickle
ub = pygmo.problem(udp_basic.udp_basic())
assert pickle.dumps(pickle.loads(pickle.dumps(ub))) == pickle.dumps(ub)
isl = pygmo.island(algo=uda_basic.uda_basic(), prob=ub, size=20)
risl = repr(isl)
assert "Thread" in risl
isl.evolve()
isl.wait_check()
assert risl == repr(isl)
class py_uda(object):
def evolve(self, pop):
return pop
if __name__ == '__main__':
isl = pygmo.island(algo=py_uda(), prob=ub, size=20)
isl.evolve()
isl.wait_check()
pygmo.mp_island.shutdown_pool()
pygmo.mp_bfe.shutdown_pool()
print("All good!")
