def pool(self): """ Returns: The computing pool to process run the queries """ if self.servers is None: return ProcessPool(nodes=self.workers) else: return ParallelPool(nodes=self.workers, servers=self.servers)
def add_mp_icsd(self, table: str, mp_data: Optional[List[Dict[str, Union[pymatgen.core.Structure, str]]]] = None, mp_api_key: Optional[str] = None) -> int: """Add a table populated with Materials Project-hosted ICSD structures. Note: This is very computationally expensive for large datasets and will not likely run on a laptop. If possible, download a pre-constructed database. Args: table (str): The name of the table to add. mp_data: The Materials Project data to parse. If this is None, data will be downloaded. Downloading data needs `mp_api_key` to be set. mp_api_key (str): A Materials Project API key. Only needed if `mp_data` is None. Returns: The number of structs added. """ if mp_data is None: #pragma: no cover with MPRester(mp_api_key) as m: data = m.query( criteria={'icsd_ids.0': { '$exists': True }}, properties=['structure', 'material_id'], ) else: data = mp_data self.add_table(table) if pathos_available: pool = ParallelPool() parse_iter = pool.uimap(parse_mprest, data) else: #pragma: no cover parse_iter = map(parse_mprest, data) return self.add_structs(parse_iter, table, commit_after_each=True)
def main(): npool = 4 ppool = ProcessPool(npool) tpool = ThreadPool(npool) parapool = ParallelPool(npool) spool = SerialPool() pool = Pool(npool) nloops = 8 print('For Loop') forloop(nloops) print('ThreadPool') test(nloops, tpool) print('ParallelPool') test(nloops, parapool) print('SerialPool') test(nloops, spool) print('Pool') test(nloops, pool) print('ProcessPool') test(nloops, ppool)
print("Running serial python ...") y = list(map(sin2, x)) print("Output: %s\n" % np.asarray(y)) if HAS_PYINA: # map sin2 to the workers, then print to screen print("Running mpi4py on %d cores..." % nodes) y = MpiPool(nodes).map(sin2, x) print("Output: %s\n" % np.asarray(y)) # map sin2 to the workers, then print to screen print("Running multiprocesing on %d processors..." % nodes) y = ProcessPool(nodes).map(sin2, x) print("Output: %s\n" % np.asarray(y)) # map sin2 to the workers, then print to screen print("Running multiprocesing on %d threads..." % nodes) y = ThreadPool(nodes).map(sin2, x) print("Output: %s\n" % np.asarray(y)) # map sin2 to the workers, then print to screen print("Running parallelpython on %d cpus..." % nodes) y = ParallelPool(nodes).map(sin2, x) print("Output: %s\n" % np.asarray(y)) # EOF
xp = np.arange(N * nodes, dtype=np.float64)[::-1] print("Input: %s\n" % x) # map sin_diff to the workers, then print to screen print("Running serial python ...") y = list(map(sin_diff, x, xp)) print("Output: %s\n" % np.asarray(y)) if HAS_PYINA: # map sin_diff to the workers, then print to screen print("Running mpi4py on %d cores..." % nodes) y = MpiPool(nodes).map(sin_diff, x, xp) print("Output: %s\n" % np.asarray(y)) # map sin_diff to the workers, then print to screen print("Running multiprocesing on %d processors..." % nodes) y = ProcessPool(nodes).map(sin_diff, x, xp) print("Output: %s\n" % np.asarray(y)) # map sin_diff to the workers, then print to screen print("Running multiprocesing on %d threads..." % nodes) y = ThreadPool(nodes).map(sin_diff, x, xp) print("Output: %s\n" % np.asarray(y)) # map sin_diff to the workers, then print to screen print("Running parallelpython on %d cpus..." % nodes) y = ParallelPool(nodes).map(sin_diff, x, xp) print("Output: %s\n" % np.asarray(y)) # EOF
#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 1997-2015 California Institute of Technology. # License: 3-clause BSD. The full license text is available at: # - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/pathos/LICENSE """ example of using the 'raw' distributed parallel mapper To run: python pp_map.py """ from pathos.pools import ParallelPool as Pool pool = Pool() if __name__ == "__main__": def add(x, y, z): """Add three values""" return x + y + z def busybeaver(x): """This can take a while""" for num in range(1000000): x = x + num return x # Immediate evaluation example import time
def test_ppmap(obj): from pathos.pools import ParallelPool p = ParallelPool(2) x = [1,2,3] assert map(obj, x) == p.map(obj, x)
#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 1997-2016 California Institute of Technology. # Copyright (c) 2016-2017 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/pathos/blob/master/LICENSE """ example of using the 'raw' distributed parallel mapper To run: python pp_map.py """ from pathos.pools import ParallelPool as Pool pool = Pool() if __name__ == '__main__': def add(x, y, z): """Add three values""" return x + y + z def busybeaver(x): """This can take a while""" for num in range(1000000): x = x + num return x # Immediate evaluation example import time start = time.time()
def __init__(self, ncpus='autodetect', ppservers=(), silent=False, **kwargs): if isinstance(ncpus, int) and 0 <= ncpus: self.ncpus = ncpus else: from pathos.helpers import cpu_count self.ncpus = cpu_count() self.__ppservers = () self.__locals = () self.__pool = () import socket local_host = socket.getfqdn().lower() from ostap.core.ostap_types import string_types if isinstance( ppservers, string_types) and ppservers.lower() in ('config', 'auto', '*'): from ostap.parallel.utils import get_ppservers ppservers = get_ppservers(local_host) if ppservers: ## remove duplicates (if any) pps = [] for p in ppservers: if p not in pps: pps.append(p) ppservers = tuple(pps) environment = kwargs.pop('environment', '') script = kwargs.pop('script', None) profile = kwargs.pop('profile', None) secret = kwargs.pop('secret', None) timeout = kwargs.pop('timeout', 7200) if script: assert os.path.exists ( script ) and os.path.isfile ( script ) ,\ 'WorkManager: no script %s is found' % script if secret is None: from ostap.utils.utils import gen_password secret = gen_password(16) ppsrvs = [ ppServer(remote, environment=environment, script=script, profile=profile, secret=secret, timeout=timeout, silent=silent) for remote in ppservers ] ppbad = [p for p in ppsrvs if not p.pid] ppgood = [p for p in ppsrvs if p.pid] self.__ppservers = tuple(ppgood) if ppbad: rs = [p.remote_host for p in ppbad] logger.warning('Failed to start remote ppservers at %s' % rs) ## if remote servers are available, reduce a bit the load for local server ## if ncpus == 'autodetect' or ncpus == 'auto' : ## self.ncpus = max ( 0 , self.ncpus - 2 ) ## some trick to setup the password. ## unfortunately ParallelPool interface does not allow it :-( if secret: import pathos.parallel as PP _ds = PP.pp.Server.default_secret PP.pp.Server.default_secret = secret from pathos.pools import ParallelPool self.__pool = ParallelPool(ncpus=self.ncpus, servers=self.locals) ## end of the trick PP.pp.Server.default_secret = _ds else: ## from pathos.multiprocessing import ProcessPool from pathos.pools import ProcessPool self.__pool = ProcessPool(self.ncpus) ps = '%s' % self.pool ps = ps.replace('<pool ', '').replace('>', '').replace('servers', 'remotes') for p in self.ppservers: ps = ps.replace(p.local, p.remote) if not silent: logger.info('WorkManager is %s' % ps) self.stats = {} self.silent = True if silent else False
def run_ppmap(obj): from pathos.pools import ParallelPool p = ParallelPool(2) x = [1,2,3] assert list(map(obj, x)) == p.map(obj, x)
def __init__(self, ncpus='autodetect', ppservers=(), silent=False, progress=True, **kwargs): if not (isinstance(ncpus, int) and 0 <= ncpus): from pathos.helpers import cpu_count ncpus = cpu_count() ## initialize the base class TaskManager.__init__(self, ncpus=ncpus, silent=silent, progress=progress) from ostap.utils.cidict import cidict kwa = cidict(**kwargs) self.__ppservers = () self.__locals = () self.__pool = () import socket local_host = socket.getfqdn().lower() from ostap.core.ostap_types import string_types if isinstance ( ppservers , string_types ) and \ ppservers.lower() in ( 'config' , 'auto' , '*' ) : from ostap.parallel.utils import get_ppservers ppservers = get_ppservers(local_host) ## use Paralell python if ppservers are specified or explicit flag if ppservers or kwa.pop('PP', False) or kwa.pop('Parallel', False): ## remove duplicates (if any) - do not sort! pps = [] for p in ppservers: if p not in pps: pps.append(p) ppservers = tuple(pps) ## check local ports local_ports = [] remotes = [] for p in ppservers: port = get_local_port(p) if port: local_ports.append("localhost:%d" % port) else: remotes.append(p) ## check alive remote hosts from ostap.parallel.utils import good_pings alive = good_pings(*remotes) if len(alive) != len(remotes): dead = list(set(remotes) - set(alive)) logger.warning("Dead remote hosts: %s" % dead) remotes = alive environment = kwa.pop('environment', '') script = kwa.pop('script', None) profile = kwa.pop('profile', None) secret = kwa.pop('secret', None) timeout = kwa.pop('timeout', 7200) if script: assert os.path.exists ( script ) and os.path.isfile ( script ) ,\ 'WorkManager: no script %s is found' % script if secret is None: from ostap.utils.utils import gen_password secret = gen_password(16) from ostap.parallel.pptunnel import ppServer, show_tunnels ppsrvs = [ ppServer(remote, environment=environment, script=script, profile=profile, secret=secret, timeout=timeout, silent=self.silent) for remote in remotes ] ppbad = [p for p in ppsrvs if not p.pid] ppgood = [p for p in ppsrvs if p.pid] self.__ppservers = tuple(ppgood) if ppbad: rs = [p.remote_host for p in ppbad] logger.warning('Failed to start remote ppservers at %s' % rs) ## if remote servers are available, reduce a bit the load for local server ## if ncpus == 'autodetect' or ncpus == 'auto' : ## self.ncpus = max ( 0 , self.ncpus - 2 ) ## some trick to setup the password. ## unfortunately ParallelPool interface does not allow it :-( if secret: import pathos.parallel as PP _ds = PP.pp.Server.default_secret PP.pp.Server.default_secret = secret ## add remote connections to the list of local ports for p in self.ppservers: local_ports.append(p.local) self.__locals = tuple(local_ports) from pathos.pools import ParallelPool self.__pool = ParallelPool(ncpus=self.ncpus, servers=self.locals) ## end of the trick PP.pp.Server.default_secret = _ds if not self.silent and self.ppservers: show_tunnels(self.ppservers) else: ## from pathos.multiprocessing import ProcessPool from pathos.pools import ProcessPool self.__pool = ProcessPool(self.ncpus) ps = '%s' % self.pool ps = ps.replace('<pool ', '').replace('>', '').replace('servers', 'remotes') for p in self.ppservers: ps = ps.replace(p.local, p.remote) if not self.silent: logger.info('WorkManager is %s' % ps)
def sp_ant_size( ): # this function runs the availability for a single point and shows a complete output with open('temp\\args.pkl', 'rb') as f: (site_lat, site_long, sat_long, freq, max_eirp, sat_height, max_bw, bw_util, modcod, pol, roll_off, ant_eff, lnb_gain, lnb_temp, aditional_losses, cable_loss, max_depoint, max_ant_size, min_ant_size, margin, cores) = pickle.load(f) f.close() ##################################### ##### ground station parameters ##### ##################################### # creating a ground station object station = GroundStation(site_lat, site_long) ############################## ### satellite parameters ### ############################## data = pd.read_csv('models\\Modulation_dB.csv', sep=';') line = data.loc[(data.Modcod) == modcod] # tech = line['Tech'].values[0] mod = line['Modulation'].values[0] fec = line['FEC'].values[0] # criando o objeto satélite satellite = Satellite(sat_long, freq, max_eirp, sat_height, max_bw, bw_util, 0, 0, mod, roll_off, fec) # atribuindo uma estação terrena à um satélite satellite.set_grstation(station) ############################## ### reception parametters #### ############################## polarization_loss = 3 # perda de polarização, em dB # criando o objeto receptor reception = Reception(None, ant_eff, aditional_losses, polarization_loss, lnb_gain, lnb_temp, cable_loss, max_depoint) # atribuindo uma recepção à um enlace de satélite satellite.set_reception( reception) # setando o receptor do link de satélite ################################### ######### OUTPUTS ######### ################################### ############ SNR target's calcullation ################ step = 0.2 interp_step = int(round((max_ant_size - min_ant_size) * 100)) ant_size_vector = np.arange(min_ant_size, max_ant_size, step) ant_size_vector_interp = np.linspace(min_ant_size, max_ant_size, interp_step) # parallel loop for each antenna size pool = ParallelPool(nodes=round(cores / 2)) #ARRUMAR AQUI availability_vector = list( pool.imap(loop_graph_ant_size, [(satellite, margin, 1, ant_size) for ant_size in ant_size_vector])) pool.clear() ant_size_vector = np.array(ant_size_vector) availability_vector = np.array(availability_vector) ant_size_vector = ant_size_vector[availability_vector > 60] availability_vector = availability_vector[availability_vector > 60] # a_BSpline = interpolate.make_interp_spline(ant_size_vector, availability_vector, k=2) # availability_vector_interp = a_BSpline(ant_size_vector_interp) availability_vector_interp = 0 with open('temp\\args.pkl', 'wb') as f: pickle.dump([ ant_size_vector, ant_size_vector_interp, availability_vector, availability_vector_interp ], f) f.close() return
def mp_ant_size(): with open('temp\\args.pkl', 'rb') as f: # opening the input variables in the temp file (gr_station_path, sat_long, freq, max_eirp, sat_height, max_bw, bw_util, modcod, pol, roll_off, ant_eff, lnb_gain, lnb_temp, aditional_losses, cable_loss, max_depoint, availability_target, snr_relaxation, margin, threads) = pickle.load(f) f.close() # reading the input table # dir = 'models\\' # file = 'CitiesBrazil' # cities = pd.read_csv(dir + file + '.csv', sep=';', encoding='latin1') # cities['availability'] = np.nan # creating an empty results column point_list = pd.read_csv( gr_station_path, sep=';', encoding='latin1') # creating a point dataframe from csv file data = pd.read_csv('models\\Modulation_dB.csv', sep=';') line = data.loc[data.Modcod == modcod] # tech = line['Tech'].values[0] mod = line['Modulation'].values[0] fec = line['FEC'].values[0] # creating the satellite object sat = Satellite(sat_long, freq, max_eirp, sat_height, max_bw, bw_util, 0, 0, mod, roll_off, fec) polarization_loss = 3 reception = Reception(None, ant_eff, aditional_losses, polarization_loss, lnb_gain, lnb_temp, cable_loss, max_depoint) # creating the receptor object # ======================== PARALLEL POOL ============================= pool = ParallelPool(nodes=threads) # creating the parallelPoll sys.stderr = open('temp\\out.txt', 'w') # to print the output dynamically print('initializing . . .', file=sys.stderr) # running the parallel pool data = list( tqdm.tqdm(pool.imap(point_ant_size, [(point, sat, reception, margin, snr_relaxation, availability_target) for index, point in point_list.iterrows()]), total=len(point_list))) pool.clear() point_list.drop(point_list.index, inplace=True) point_list = point_list.append(data, ignore_index=True) # saving the results into a csv file dir = 'results' if not os.path.exists(dir): os.makedirs(dir) point_list.to_csv(dir + '\\' + 'results_ant ' + datetime.datetime.now().strftime('%y-%m-%d_%H-%M-%S') + '.csv', sep=';', encoding='latin1') print('Complete!!!', file=sys.stderr) sys.stderr.close() return
t1 = time() T1 = t1 - t0 print(result) print("in time = {0:.3f}".format(T1)) print('') # parallel calculation numbers = [ 3093215881333057, 3093215881333057, 3093215881333057, 3093215881333057 ] print("{} parallel calculations with {} out of {} CPUs".format( len(numbers), WORKERS, cpu_count())) t0 = time() # create the pool of workers pool = ParallelPool(WORKERS) # open the functions in their own threads and return the results results = pool.map(function, numbers) pool.close() pool.join() t1 = time() T2 = t1 - t0 print(results) print("in time = {0:.3f}".format(T2)) print('') print("ratio = {0:.2f}%".format(100. * T2 / T1))
def simple_search(dataset, params, val_accuracy, N_workers, restart_num=3, output_dim=10, lr=0.001, epoch_limit=20, random_seed=29): print("--------------------------------------------------------------") print("SAS starts") print("--------------------------------------------------------------") print("run for at most " + str(epoch_limit) + " epochs each") print("running in parallel using " + str(N_workers) + " workers ") print(datetime.datetime.now()) val_param = copy.deepcopy(params) architecture = params['architecture'] layer = len(architecture) instructions = [] ''' instructions are guideline for architecture transformations ''' instructions.append({'Wider': [layer], 'Deeper': []}) instructions.append({'Wider': [layer, layer], 'Deeper': []}) instructions.append({'Wider': [], 'Deeper': [layer, layer + 1]}) instructions.append({'Wider': [layer], 'Deeper': [layer]}) instructions.append({'Wider': [], 'Deeper': [layer]}) instructions.append({'Wider': [layer, layer], 'Deeper': [layer]}) EveryLayer = np.arange(layer) + 1 #Widen all layers at once instructions.append({'Wider': EveryLayer, 'Deeper': []}) print("instruction generation complete") pool = ParallelPool(N_workers) print(" creating Pool and setting up workers") num_instructions = len(instructions) l_dataset = [dataset] * num_instructions l_output_dim = [output_dim] * num_instructions l_params = [params] * num_instructions l_lr = [lr] * num_instructions l_epoch_limit = [epoch_limit] * num_instructions l_restart_num = [restart_num] * num_instructions l_seed = [random_seed] * num_instructions print("function call preparation complete ") ''' train the candidates in parallel ''' candidates = pool.map(Just_Train, l_dataset, l_output_dim, l_params, instructions, l_lr, l_epoch_limit, l_restart_num, l_seed) print("all candidates received") print(datetime.datetime.now()) best_accu = 0 best_param = 0 ''' identify the best candidate ''' for candidate in candidates: if candidate is None: print("find a none type") continue accuracy = candidate['accuracy'] architecture = candidate['params']['architecture'] print("for architecture : " + str(architecture)) print(" achieved validation accuracy of " + str(accuracy)) if (accuracy > best_accu): best_accu = accuracy best_param = candidate['params'] if (val_accuracy > best_accu): best_accu = val_accuracy best_param = val_param print("best candidate has architecture") print(best_param['architecture']) return {'accuracy': best_accu, 'params': best_param}
break print("") y = m.get() print("I'm awake") print("y = %s" % str(y[:10])) if __name__ == '__main__': x = list(range(500)) delay = 0.01 maxtries = 200 f = busy_add #f = busy_squared #f = squared #from pathos.pools import ProcessPool as Pool #from pathos.pools import ThreadPool as Pool from pathos.pools import ParallelPool as Pool #from pathos.helpers import freeze_support, shutdown #freeze_support() pool = Pool(nodes=4) test_ready(pool, f, maxtries, delay) # shutdown pool.close() pool.join() pool.clear() # EOF
def test_ppmap(obj): from pathos.pools import ParallelPool p = ParallelPool(2) x = [1, 2, 3] assert map(obj, x) == p.map(obj, x)