def test_non_population_based_migration(): import pygmo as pg from pygmo import de, nlopt, rosenbrock from sabaody.topology import TopologyFactory topology_factory = TopologyFactory(island_size=5, migrant_pool_size=5) topology = topology_factory.createBidirChain(de, number_of_islands=2) assert len(topology.island_ids) == 2 assert len(topology.endpoints) == 2 de_island = pg.island(algo=de(gen=10), prob=rosenbrock(3), size=topology.islands[0].size) nm = nlopt('neldermead') nm.selection = 'random' nm.replacement = 'random' nm_island = pg.island(algo=nm, prob=rosenbrock(3), size=topology.islands[1].size) from sabaody.migration import BestSPolicy, FairRPolicy selection_policy = BestSPolicy(migration_rate=2) replacement_policy = FairRPolicy() from sabaody.migration import MigrationPolicyEachToAll migration_policy = MigrationPolicyEachToAll() # get the candidates from the de island p_de = de_island.get_population() candidates, candidate_f = selection_policy.select(p_de) # try migrating to the nelder mead island p_nm = nm_island.get_population() replacement_policy.replace(p_nm, candidates, candidate_f) nm_island.set_population(p_nm) # finally, try to evolve it new_pop = nm_island.evolve(n=10)
def run_island(island): import pygmo as pg from multiprocessing import cpu_count from pymemcache.client.base import Client mc_client = Client((island.mc_host, island.mc_port)) #udp = island.problem_factory() algorithm = pg.algorithm(pg.de()) #problem = pg.problem(udp) problem = island.problem_factory() # TODO: configure pop size i = pg.island(algo=algorithm, prob=problem, size=20) mc_client.set(island.domain_qualifier('island', str(island.id), 'status'), 'Running', 10000) mc_client.set(island.domain_qualifier('island', str(island.id), 'n_cores'), str(cpu_count()), 10000) #print('Starting island {} with {} cpus'.format(str(i.id), str(cpu_count()))) i.evolve() i.wait() import socket hostname = socket.gethostname() ip = [ l for l in ([ ip for ip in socket.gethostbyname_ex(socket.gethostname())[2] if not ip.startswith("127.") ][:1], [[(s.connect(('8.8.8.8', 53)), s.getsockname()[0], s.close()) for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)] ][0][1]]) if l ][0][0] return (ip, hostname, i.get_population().problem.get_fevals())
def island_run(self): algo = pg.algorithm(pg.pso(gen=self.generation)) pop = pg.population(self.problem, self.pop_size) island = pg.island(algo=algo, pop=pop, udi=pg.mp_island()) island.evolve() island.wait() pop = island.get_population() return pop.champion_f, pop.champion_x
def test_bidir_chain(): ''' Tests the migration on a one way chain. ''' from sabaody.topology import TopologyFactory def make_problem(): import pygmo as pg return pg.problem(pg.rosenbrock(3)) def make_algorithm(): return pg.de(gen=10) domain_qual = partial(getQualifiedName, 'com.how2cell.sabaody.test_bidir_chain_migration') topology_factory = TopologyFactory(make_problem, domain_qual, 'localhost', 11211) topology = topology_factory.createBidirChain(make_algorithm, 5) assert len(topology.island_ids) == 5 assert len(topology.endpoints) == 2 from sabaody.migration_central import CentralMigrator, start_migration_service from sabaody.migration import BestSPolicy, FairRPolicy, sort_by_fitness import pygmo as pg try: process = start_migration_service() sleep(2) migrator = CentralMigrator('http://localhost:10100', BestSPolicy(migration_rate=1), FairRPolicy()) from collections import OrderedDict for k in (1, 2): islands = OrderedDict((i.id, pg.island(algo=i.algorithm_constructor(), prob=i.problem_constructor(), size=i.size)) for i in topology.islands) for island_id in islands.keys(): migrator.defineMigrantPool(island_id, 3) if k == 1: # test forward migration seed_first(islands, topology) else: # test reverse migration seed_last(islands, topology) for n in range(1, 5 + 1): assert count_hits(islands.values()) == n # perform migration for island_id, i in islands.items(): migrator.sendMigrants(island_id, i, topology) for island_id, i in islands.items(): deltas, src_ids = migrator.receiveMigrants( island_id, i, topology) migrator.purgeAll() finally: process.terminate()
def test_pygmo_single_generation(two_reservoir_wrapper): """ Simple pygmo wrapper test. """ wrapper = two_reservoir_wrapper prob = pg.problem(wrapper) algo = pg.algorithm(pg.moead(gen=1)) pg.mp_island.init_pool(2) isl = pg.island(algo=algo, prob=prob, size=50, udi=pg.mp_island()) isl.evolve(1)
def addIsland(arc, algo): isl = pg.island( udi=pg.mp_island(), algo=algo, prob=prob, #b=pg.mp_bfe(), b=pg.bfe(udbfe=multi_bfre2()), size=16) arc.push_back(isl)
def get_island(evaluate, params, hooks): # config # D = 8 decision space dimension # N0 = 100 initial population # Ng = 100 total generation D, Ng, N0 = itemgetter('D', 'Ng', 'N0')(params) algo = pg.algorithm(pg.de(gen=Ng)) algo.set_verbosity(int(Ng / 10)) prob = pg.problem(evaluate_wrapper(D, evaluate)) island = pg.island(algo=algo, prob=prob, size=N0, udi=pg.mp_island()) return island
def run_island(island, topology): import pygmo as pg from multiprocessing import cpu_count from pymemcache.client.base import Client from sabaody.migration import BestSPolicy, FairRPolicy from sabaody.migration_central import CentralMigrator mc_client = Client((island.mc_host, island.mc_port)) migrator = CentralMigrator('http://luna:10100') algorithm = pg.de(gen=10) problem = island.problem_constructor() # TODO: configure pop size i = pg.island(algo=algorithm, prob=problem, size=20) mc_client.set(island.domain_qualifier('island', str(island.id), 'status'), 'Running', 10000) mc_client.set(island.domain_qualifier('island', str(island.id), 'n_cores'), str(cpu_count()), 10000) rounds = 10 migration_log = [] for x in range(rounds): i.evolve() i.wait() # perform migration migrator.sendMigrants(island.id, i, topology) deltas, src_ids = migrator.receiveMigrants(island.id, i, topology) """ For Kafka Migration Enable below """ #migrator.send_migrants(island.id,i,topology,generation=x) #deltas,src_ids = migrator.receive_migrants(island.id,i,topology,generation=x) migration_log.append((float(pop.champion_f[0]), deltas, src_ids)) import socket hostname = socket.gethostname() ip = [ l for l in ([ ip for ip in socket.gethostbyname_ex(socket.gethostname())[2] if not ip.startswith("127.") ][:1], [[(s.connect(('8.8.8.8', 53)), s.getsockname()[0], s.close()) for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)] ][0][1]]) if l ][0][0] return (ip, hostname, island.id, migration_log, i.get_population().problem.get_fevals())
def archipelago(): udp = solo_mgar_udp([7000, 8000]) #uda = pg.sga(gen = 6000) uda = pg.sade(memory=True, variant=1, gen=6000) # instantiate an unconnected archipelago for _ in range(1000): archi = pg.archipelago(t=pg.topologies.unconnected()) for _ in range(32): alg = pg.algorithm(uda) #alg.set_verbosity(1) prob = pg.problem(udp) pop = pg.population(prob, 20) isl = pg.island(algo=alg, pop=pop) archi.push_back(isl) archi.evolve() archi.wait_check()
def archipelago(): print('interferometer sga archipelago') uda = pg.sga(gen=50000) # instantiate an unconnected archipelago archi = pg.archipelago(t=pg.topologies.unconnected()) t = time() for _ in range(8): alg = pg.algorithm(uda) #alg.set_verbosity(1) prob = pg.problem(udp) pop = pg.population(prob, 20) isl = pg.island(algo=alg, pop=pop) archi.push_back(isl) archi.evolve() archi.wait_check() print(f'archi: {time() - t:0.3f}s')
def pygmo_main(harmonic=False): import pygmo as pg from pywr.optimisation.pygmo import PygmoWrapper def update_archive(pop, archive=None): if archive is None: combined_f = pop.get_f() else: combined_f = np.r_[archive, pop.get_f()] indices = pg.select_best_N_mo(combined_f, 50) new_archive = combined_f[indices, :] new_archive = np.unique(new_archive.round(4), axis=0) return new_archive wrapper = PygmoWrapper(get_model_data(harmonic=harmonic)) prob = pg.problem(wrapper) print(prob) algo = pg.algorithm(pg.moead(gen=1)) # algo = pg.algorithm(pg.nsga2(gen=1)) pg.mp_island.init_pool(2) isl = pg.island(algo=algo, prob=prob, size=50, udi=pg.mp_island()) ref_point = [216500, 4000] pop = isl.get_population() print("Evolving!") archive = update_archive(pop) hv = pg.hypervolume(archive) vol = hv.compute(ref_point) hvs = [ vol, ] print( "Gen: {:03d}, Hypervolume: {:6.4g}, Archive size: {:d}".format( 0, vol, len(archive) ) ) for gen in range(20): isl.evolve(1) isl.wait_check() pop = isl.get_population() archive = update_archive(pop, archive) hv = pg.hypervolume(archive) vol = hv.compute(ref_point) print( "Gen: {:03d}, Hypervolume: {:6.4g}, Archive size: {:d}".format( gen + 1, vol, len(archive) ) ) hvs.append(vol) hvs = pd.Series(hvs) print("Finished!") plt.scatter(archive[:, 0], archive[:, 1]) objectives = wrapper.model_objectives plt.xlabel(objectives[0].name) plt.ylabel(objectives[1].name) plt.grid(True) title = "Harmonic Control Curve" if harmonic else "Monthly Control Curve" plt.savefig("{} Example ({}).pdf".format("pygmo", title), format="pdf") fig, ax = plt.subplots() ax.plot(hvs / 1e6, marker="o") ax.grid(True) ax.set_ylabel("Hypervolume") ax.set_xlabel("Generation") plt.savefig("{} Example Hypervolume ({}).pdf".format("pygmo", title), format="pdf") plt.show()
def run_island(id): class B2_UDP: def __init__(self, lb, ub): # type: (Evaluator, array, array) -> None ''' Inits the problem with an objective evaluator (implementing the method evaluate), the parameter vector lower bound (a numpy array) and upper bound. Both bounds must have the same dimension. ''' from sabaody.utils import check_vector, expect check_vector(lb) check_vector(ub) expect(len(lb) == len(ub), 'Bounds mismatch') self.lb = lb self.ub = ub from b2problem import B2Problem self.evaluator = B2Problem('b2.xml') def fitness(self, x): return (self.evaluator.evaluate(x), ) def get_bounds(self): return (self.lb, self.ub) def get_name(self): return 'Sabaody udp' def get_extra_info(self): return 'Sabaody extra info' def __getstate__(self): return {'lb': self.lb, 'ub': self.ub} def __setstate__(self, state): self.lb = state['lb'] self.ub = state['ub'] from b2problem import B2Problem self.evaluator = B2Problem('b2.xml') import pygmo as pg from pymemcache.client.base import Client mc_client = Client(('luna', 11211)) algorithm = pg.algorithm(pg.de(gen=1000)) from params import getLowerBound, getUpperBound problem = pg.problem(B2_UDP(getLowerBound(), getUpperBound())) i = pg.island(algo=algorithm, prob=problem, size=20) #mc_client.set(id.domain_qualifier('island', str(id), 'status'), 'Running', 10000) i.evolve() i.wait() import socket hostname = socket.gethostname() ip = [ l for l in ([ ip for ip in socket.gethostbyname_ex(socket.gethostname())[2] if not ip.startswith("127.") ][:1], [[(s.connect(('8.8.8.8', 53)), s.getsockname()[0], s.close()) for s in [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)] ][0][1]]) if l ][0][0] return (ip, hostname)
def create_archipelago( unknowns: list, optimizers: list, optimizers_kwargs: list, pg_problem: pygmo.problem, rel_pop_size: float, archipelago_kwargs: dict, log_each_nth_gen: int, report_level: int, ) -> PyfoombArchipelago: """ Helper method for parallelized estimation using the generalized island model. Creates the archipelago object for running several rounds of evolutions. Arguments --------- unknowns : list The unknowns, sorted alphabetically and case-insensitive. optimizers : list A list of optimizers to be used on individual islands. optimizers_kwargs : list A list of corresponding kwargs. pg_problem : pygmo.problem An pygmo problem instance. archipelago_kwargs : dict Additional kwargs for archipelago creation. log_each_nth_gen : int Specifies at which each n-th generation the algorithm stores logs. report_level : int Prints information on the archipelago creation for values >= 1. Returns ------- archipelago : PyfoombArchipelago """ _cpus = joblib.cpu_count() # There is one optimizer with a set of kwargs if len(optimizers) == 1 and len(optimizers_kwargs) == 1: optimizers = optimizers * _cpus optimizers_kwargs = optimizers_kwargs * _cpus # Several optimizers with the same kwargs elif len(optimizers) > 1 and len(optimizers_kwargs) == 1: optimizers_kwargs = optimizers_kwargs * len(optimizers) # Several kwargs for the same optimizer elif len(optimizers) == 1 and len(optimizers_kwargs) > 1: optimizers = optimizers * len(optimizers_kwargs) elif len(optimizers) != len(optimizers_kwargs): raise ValueError( 'Number of optimizers does not match number of corresponding kwarg dicts' ) # Get the optimizer intances algos = [ PygmoOptimizers.get_optimizer_algo_instance( name=_optimizers, kwargs=_optimizers_kwargs) for _optimizers, _optimizers_kwargs in zip(optimizers, optimizers_kwargs) ] # Update number of islands n_islands = len(algos) if report_level >= 1: print( f'Creating archipelago with {n_islands} islands. May take some time...' ) pop_size = int(numpy.ceil(rel_pop_size * len(unknowns))) prop_create_args = ((pg_problem, pop_size, seed * numpy.random.randint(0, 1e4)) for seed, pop_size in enumerate([pop_size] * n_islands)) try: parallel_verbose = 0 if report_level == 0 else 1 with joblib.parallel_backend('loky', n_jobs=n_islands): pops = joblib.Parallel(verbose=parallel_verbose)(map( joblib.delayed( ArchipelagoHelpers.parallel_create_population), prop_create_args)) except Exception as ex: print( f'Parallelized archipelago creation failed, falling back to sequential\n{ex}' ) pops = ( ArchipelagoHelpers.parallel_create_population(prop_create_arg) for prop_create_arg in prop_create_args) # Now create the empyty archipelago if not 't' in archipelago_kwargs.keys(): archipelago_kwargs['t'] = pygmo.fully_connected() archi = PyfoombArchipelago(**archipelago_kwargs) archi.set_migrant_handling(pygmo.migrant_handling.preserve) # Add the populations to the archipelago and wait for its construction with contextlib.redirect_stdout(io.StringIO()): for _pop, _algo in zip(pops, algos): if log_each_nth_gen is not None: _algo.set_verbosity(int(log_each_nth_gen)) _island = pygmo.island(algo=_algo, pop=_pop, udi=pygmo.mp_island()) archi.push_back(_island) archi.wait_check() return archi
from numpy import array, save, savez, mean, std, nan_to_num, vstack from tempfile import gettempdir from os.path import join import json single_champions = [] N = 100 for i in range(N): with RBRun('luna', 11211) as run: print('Started run {} of {}'.format(i + 1, N)) from rbsetup import make_problem algorithm = pg.de(gen=10) problem = make_problem() i = pg.island(algo=algorithm, prob=problem, size=20) rounds = 10 c = [] for x in range(rounds): i.evolve() i.wait() c.append(float(i.get_population().champion_f[0])) print('round {}'.format(x)) print(i.get_population().champion_f) print(i.get_population().get_x()[0, :]) single_champions.append(array(c)) single_champions_stack = vstack(single_champions) single_champions_mean = mean(single_champions_stack, axis=0) single_champions_std = std(single_champions_stack, axis=0)
import pygmo as pg """ Okay, so the lesson learned here is: not sure :-D """ if __name__ == "__main__": prob = pg.problem(pg.rosenbrock(dim=5)) pool_creator = pg.mp_island() # pool_creator.resize_pool(1) pool_creator.init_pool(1) island = pg.island(udi=pool_creator, algo=pg.sga(gen=200), pop=pg.population(prob, 200)) island.evolve() island.wait() print("island: ***** \n", island)
import uda_basic import pygmo import pickle ub = pygmo.algorithm(uda_basic.uda_basic()) assert pickle.dumps(pickle.loads(pickle.dumps(ub))) == pickle.dumps(ub) isl = pygmo.island(algo=ub, prob=pygmo.rosenbrock(), size=20) risl = repr(isl) assert "Thread" in risl isl.evolve() isl.wait_check() assert risl == repr(isl) class py_udp(object): def get_bounds(self): return ([0, 0], [1, 1]) def fitness(self, a): return [42] if __name__ == '__main__': isl = pygmo.island(algo=ub, prob=py_udp(), size=20) isl.evolve() isl.wait_check() pygmo.mp_island.shutdown_pool() pygmo.mp_bfe.shutdown_pool() print("All good!")
def test_one_way_ring_migration(): ''' Tests the migration on a one way ring. ''' from sabaody.topology import TopologyFactory def make_problem(): import pygmo as pg return pg.problem(pg.rosenbrock(3)) def make_algorithm(): return pg.de(gen=10) domain_qual = partial(getQualifiedName, 'com.how2cell.sabaody.test_one_way_ring_migration') topology_factory = TopologyFactory(make_problem, domain_qual, 'localhost', 11211) topology = topology_factory.createOneWayRing(make_algorithm, 5) assert len(topology.island_ids) == 5 from sabaody.migration_central import CentralMigrator, start_migration_service from sabaody.migration import BestSPolicy, FairRPolicy, sort_by_fitness import pygmo as pg try: process = start_migration_service() sleep(2) migrator = CentralMigrator('http://localhost:10100', BestSPolicy(migration_rate=1), FairRPolicy()) from collections import OrderedDict islands = OrderedDict((i.id, pg.island(algo=i.algorithm_constructor(), prob=i.problem_constructor(), size=i.size)) for i in topology.islands) for island_id in islands.keys(): migrator.defineMigrantPool(island_id, 3) # seed solution in one island seed_first(islands, topology) assert count_hits(islands.values()) == 1 for n in range(1, 5 + 1): assert count_hits(islands.values()) == n # perform migration for island_id, i in islands.items(): migrator.sendMigrants(island_id, i, topology) for island_id, i in islands.items(): deltas, src_ids = migrator.receiveMigrants( island_id, i, topology) assert n == 5 # reset & differentiate from chain islands = OrderedDict((i.id, pg.island(algo=i.algorithm_constructor(), prob=i.problem_constructor(), size=i.size)) for i in topology.islands) seed_predecessor(islands, topology) # perform migration for island_id, i in islands.items(): migrator.sendMigrants(island_id, i, topology) for island_id, i in islands.items(): deltas, src_ids = migrator.receiveMigrants(island_id, i, topology) assert tuple(islands.values())[0].get_population().champion_f[0] == 0. assert count_hits(islands.values()) == 2 finally: process.terminate()
GA_based_optimize(pm, cm, prob_var.left_ind, prob_var.mid_ind, prob_var.right_ind, gen=max_gen, seed=rnd_seed, log_lvl=log_lvl)) # Generating isl_num initial populations pop_lst = [pg.population(prob) for _ in range(isl_num)] [[p.push_back(next(GenerateIndividual)) for p in pop_lst] for _ in range(pop_size)] # Generating pm_lst × cm_lst algorithms with different crossover and # mutation probability alg_lst = [ pg.algorithm( GA_based_optimize(p, c, prob_var.left_ind, prob_var.mid_ind, prob_var.right_ind, gen=max_gen, seed=rnd_seed, log_lvl=log_lvl)) for c in cm_lst for p in pm_lst ] # Creating islands islands = [ pg.island(algo=a, pop=p, udi=pg.mp_island()) for a, p in zip(alg_lst, pop_lst) ]
import uda_basic import udp_basic import pygmo import pickle ub = pygmo.problem(udp_basic.udp_basic()) assert pickle.dumps(pickle.loads(pickle.dumps(ub))) == pickle.dumps(ub) isl = pygmo.island(algo=uda_basic.uda_basic(), prob=ub, size=20) risl = repr(isl) assert "Thread" in risl isl.evolve() isl.wait_check() assert risl == repr(isl) class py_uda(object): def evolve(self, pop): return pop if __name__ == '__main__': isl = pygmo.island(algo=py_uda(), prob=ub, size=20) isl.evolve() isl.wait_check() pygmo.mp_island.shutdown_pool() pygmo.mp_bfe.shutdown_pool() print("All good!")