def pool(self):
"""
Returns: The computing pool to process run the queries
"""
if self.servers is None:
return ProcessPool(nodes=self.workers)
else:
return ParallelPool(nodes=self.workers, servers=self.servers)
def add_mp_icsd(self,
table: str,
mp_data: Optional[List[Dict[str,
Union[pymatgen.core.Structure,
str]]]] = None,
mp_api_key: Optional[str] = None) -> int:
"""Add a table populated with Materials Project-hosted ICSD structures.
Note:
This is very computationally expensive for large datasets
and will not likely run on a laptop.
If possible, download a pre-constructed database.
Args:
table (str): The name of the table to add.
mp_data: The Materials Project data to parse. If this is None, data
will be downloaded. Downloading data needs `mp_api_key` to be set.
mp_api_key (str): A Materials Project API key. Only needed if `mp_data`
is None.
Returns:
The number of structs added.
"""
if mp_data is None: #pragma: no cover
with MPRester(mp_api_key) as m:
data = m.query(
criteria={'icsd_ids.0': {
'$exists': True
}},
properties=['structure', 'material_id'],
)
else:
data = mp_data
self.add_table(table)
if pathos_available:
pool = ParallelPool()
parse_iter = pool.uimap(parse_mprest, data)
else: #pragma: no cover
parse_iter = map(parse_mprest, data)
return self.add_structs(parse_iter, table, commit_after_each=True)
def main():
npool = 4
ppool = ProcessPool(npool)
tpool = ThreadPool(npool)
parapool = ParallelPool(npool)
spool = SerialPool()
pool = Pool(npool)
nloops = 8
print('For Loop')
forloop(nloops)
print('ThreadPool')
test(nloops, tpool)
print('ParallelPool')
test(nloops, parapool)
print('SerialPool')
test(nloops, spool)
print('Pool')
test(nloops, pool)
print('ProcessPool')
test(nloops, ppool)
print("Running serial python ...")
y = list(map(sin2, x))
print("Output: %s\n" % np.asarray(y))
if HAS_PYINA:
# map sin2 to the workers, then print to screen
print("Running mpi4py on %d cores..." % nodes)
y = MpiPool(nodes).map(sin2, x)
print("Output: %s\n" % np.asarray(y))
# map sin2 to the workers, then print to screen
print("Running multiprocesing on %d processors..." % nodes)
y = ProcessPool(nodes).map(sin2, x)
print("Output: %s\n" % np.asarray(y))
# map sin2 to the workers, then print to screen
print("Running multiprocesing on %d threads..." % nodes)
y = ThreadPool(nodes).map(sin2, x)
print("Output: %s\n" % np.asarray(y))
# map sin2 to the workers, then print to screen
print("Running parallelpython on %d cpus..." % nodes)
y = ParallelPool(nodes).map(sin2, x)
print("Output: %s\n" % np.asarray(y))
# EOF
xp = np.arange(N * nodes, dtype=np.float64)[::-1]
print("Input: %s\n" % x)
# map sin_diff to the workers, then print to screen
print("Running serial python ...")
y = list(map(sin_diff, x, xp))
print("Output: %s\n" % np.asarray(y))
if HAS_PYINA:
# map sin_diff to the workers, then print to screen
print("Running mpi4py on %d cores..." % nodes)
y = MpiPool(nodes).map(sin_diff, x, xp)
print("Output: %s\n" % np.asarray(y))
# map sin_diff to the workers, then print to screen
print("Running multiprocesing on %d processors..." % nodes)
y = ProcessPool(nodes).map(sin_diff, x, xp)
print("Output: %s\n" % np.asarray(y))
# map sin_diff to the workers, then print to screen
print("Running multiprocesing on %d threads..." % nodes)
y = ThreadPool(nodes).map(sin_diff, x, xp)
print("Output: %s\n" % np.asarray(y))
# map sin_diff to the workers, then print to screen
print("Running parallelpython on %d cpus..." % nodes)
y = ParallelPool(nodes).map(sin_diff, x, xp)
print("Output: %s\n" % np.asarray(y))
# EOF
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2015 California Institute of Technology.
# License: 3-clause BSD. The full license text is available at:
# - http://trac.mystic.cacr.caltech.edu/project/pathos/browser/pathos/LICENSE
"""
example of using the 'raw' distributed parallel mapper
To run: python pp_map.py
"""
from pathos.pools import ParallelPool as Pool
pool = Pool()
if __name__ == "__main__":
def add(x, y, z):
"""Add three values"""
return x + y + z
def busybeaver(x):
"""This can take a while"""
for num in range(1000000):
x = x + num
return x
# Immediate evaluation example
import time
def test_ppmap(obj):
from pathos.pools import ParallelPool
p = ParallelPool(2)
x = [1,2,3]
assert map(obj, x) == p.map(obj, x)
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2017 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - https://github.com/uqfoundation/pathos/blob/master/LICENSE
"""
example of using the 'raw' distributed parallel mapper
To run: python pp_map.py
"""
from pathos.pools import ParallelPool as Pool
pool = Pool()
if __name__ == '__main__':
def add(x, y, z):
"""Add three values"""
return x + y + z
def busybeaver(x):
"""This can take a while"""
for num in range(1000000):
x = x + num
return x
# Immediate evaluation example
import time
start = time.time()
def __init__(self,
ncpus='autodetect',
ppservers=(),
silent=False,
**kwargs):
if isinstance(ncpus, int) and 0 <= ncpus: self.ncpus = ncpus
else:
from pathos.helpers import cpu_count
self.ncpus = cpu_count()
self.__ppservers = ()
self.__locals = ()
self.__pool = ()
import socket
local_host = socket.getfqdn().lower()
from ostap.core.ostap_types import string_types
if isinstance(
ppservers,
string_types) and ppservers.lower() in ('config', 'auto', '*'):
from ostap.parallel.utils import get_ppservers
ppservers = get_ppservers(local_host)
if ppservers:
## remove duplicates (if any)
pps = []
for p in ppservers:
if p not in pps:
pps.append(p)
ppservers = tuple(pps)
environment = kwargs.pop('environment', '')
script = kwargs.pop('script', None)
profile = kwargs.pop('profile', None)
secret = kwargs.pop('secret', None)
timeout = kwargs.pop('timeout', 7200)
if script:
assert os.path.exists ( script ) and os.path.isfile ( script ) ,\
'WorkManager: no script %s is found' % script
if secret is None:
from ostap.utils.utils import gen_password
secret = gen_password(16)
ppsrvs = [
ppServer(remote,
environment=environment,
script=script,
profile=profile,
secret=secret,
timeout=timeout,
silent=silent) for remote in ppservers
]
ppbad = [p for p in ppsrvs if not p.pid]
ppgood = [p for p in ppsrvs if p.pid]
self.__ppservers = tuple(ppgood)
if ppbad:
rs = [p.remote_host for p in ppbad]
logger.warning('Failed to start remote ppservers at %s' % rs)
## if remote servers are available, reduce a bit the load for local server
## if ncpus == 'autodetect' or ncpus == 'auto' :
## self.ncpus = max ( 0 , self.ncpus - 2 )
## some trick to setup the password.
## unfortunately ParallelPool interface does not allow it :-(
if secret:
import pathos.parallel as PP
_ds = PP.pp.Server.default_secret
PP.pp.Server.default_secret = secret
from pathos.pools import ParallelPool
self.__pool = ParallelPool(ncpus=self.ncpus, servers=self.locals)
## end of the trick
PP.pp.Server.default_secret = _ds
else:
## from pathos.multiprocessing import ProcessPool
from pathos.pools import ProcessPool
self.__pool = ProcessPool(self.ncpus)
ps = '%s' % self.pool
ps = ps.replace('<pool ',
'').replace('>', '').replace('servers', 'remotes')
for p in self.ppservers:
ps = ps.replace(p.local, p.remote)
if not silent: logger.info('WorkManager is %s' % ps)
self.stats = {}
self.silent = True if silent else False
def run_ppmap(obj):
from pathos.pools import ParallelPool
p = ParallelPool(2)
x = [1,2,3]
assert list(map(obj, x)) == p.map(obj, x)
def __init__(self,
ncpus='autodetect',
ppservers=(),
silent=False,
progress=True,
**kwargs):
if not (isinstance(ncpus, int) and 0 <= ncpus):
from pathos.helpers import cpu_count
ncpus = cpu_count()
## initialize the base class
TaskManager.__init__(self,
ncpus=ncpus,
silent=silent,
progress=progress)
from ostap.utils.cidict import cidict
kwa = cidict(**kwargs)
self.__ppservers = ()
self.__locals = ()
self.__pool = ()
import socket
local_host = socket.getfqdn().lower()
from ostap.core.ostap_types import string_types
if isinstance ( ppservers , string_types ) and \
ppservers.lower() in ( 'config' , 'auto' , '*' ) :
from ostap.parallel.utils import get_ppservers
ppservers = get_ppservers(local_host)
## use Paralell python if ppservers are specified or explicit flag
if ppservers or kwa.pop('PP', False) or kwa.pop('Parallel', False):
## remove duplicates (if any) - do not sort!
pps = []
for p in ppservers:
if p not in pps:
pps.append(p)
ppservers = tuple(pps)
## check local ports
local_ports = []
remotes = []
for p in ppservers:
port = get_local_port(p)
if port: local_ports.append("localhost:%d" % port)
else: remotes.append(p)
## check alive remote hosts
from ostap.parallel.utils import good_pings
alive = good_pings(*remotes)
if len(alive) != len(remotes):
dead = list(set(remotes) - set(alive))
logger.warning("Dead remote hosts: %s" % dead)
remotes = alive
environment = kwa.pop('environment', '')
script = kwa.pop('script', None)
profile = kwa.pop('profile', None)
secret = kwa.pop('secret', None)
timeout = kwa.pop('timeout', 7200)
if script:
assert os.path.exists ( script ) and os.path.isfile ( script ) ,\
'WorkManager: no script %s is found' % script
if secret is None:
from ostap.utils.utils import gen_password
secret = gen_password(16)
from ostap.parallel.pptunnel import ppServer, show_tunnels
ppsrvs = [
ppServer(remote,
environment=environment,
script=script,
profile=profile,
secret=secret,
timeout=timeout,
silent=self.silent) for remote in remotes
]
ppbad = [p for p in ppsrvs if not p.pid]
ppgood = [p for p in ppsrvs if p.pid]
self.__ppservers = tuple(ppgood)
if ppbad:
rs = [p.remote_host for p in ppbad]
logger.warning('Failed to start remote ppservers at %s' % rs)
## if remote servers are available, reduce a bit the load for local server
## if ncpus == 'autodetect' or ncpus == 'auto' :
## self.ncpus = max ( 0 , self.ncpus - 2 )
## some trick to setup the password.
## unfortunately ParallelPool interface does not allow it :-(
if secret:
import pathos.parallel as PP
_ds = PP.pp.Server.default_secret
PP.pp.Server.default_secret = secret
## add remote connections to the list of local ports
for p in self.ppservers:
local_ports.append(p.local)
self.__locals = tuple(local_ports)
from pathos.pools import ParallelPool
self.__pool = ParallelPool(ncpus=self.ncpus, servers=self.locals)
## end of the trick
PP.pp.Server.default_secret = _ds
if not self.silent and self.ppservers: show_tunnels(self.ppservers)
else:
## from pathos.multiprocessing import ProcessPool
from pathos.pools import ProcessPool
self.__pool = ProcessPool(self.ncpus)
ps = '%s' % self.pool
ps = ps.replace('<pool ',
'').replace('>', '').replace('servers', 'remotes')
for p in self.ppservers:
ps = ps.replace(p.local, p.remote)
if not self.silent: logger.info('WorkManager is %s' % ps)
def sp_ant_size(
): # this function runs the availability for a single point and shows a complete output
with open('temp\\args.pkl', 'rb') as f:
(site_lat, site_long, sat_long, freq, max_eirp, sat_height, max_bw,
bw_util, modcod, pol, roll_off, ant_eff, lnb_gain, lnb_temp,
aditional_losses, cable_loss, max_depoint, max_ant_size, min_ant_size,
margin, cores) = pickle.load(f)
f.close()
#####################################
##### ground station parameters #####
#####################################
# creating a ground station object
station = GroundStation(site_lat, site_long)
##############################
### satellite parameters ###
##############################
data = pd.read_csv('models\\Modulation_dB.csv', sep=';')
line = data.loc[(data.Modcod) == modcod]
# tech = line['Tech'].values[0]
mod = line['Modulation'].values[0]
fec = line['FEC'].values[0]
# criando o objeto satélite
satellite = Satellite(sat_long, freq, max_eirp, sat_height, max_bw,
bw_util, 0, 0, mod, roll_off, fec)
# atribuindo uma estação terrena à um satélite
satellite.set_grstation(station)
##############################
### reception parametters ####
##############################
polarization_loss = 3 # perda de polarização, em dB
# criando o objeto receptor
reception = Reception(None, ant_eff, aditional_losses, polarization_loss,
lnb_gain, lnb_temp, cable_loss, max_depoint)
# atribuindo uma recepção à um enlace de satélite
satellite.set_reception(
reception) # setando o receptor do link de satélite
###################################
######### OUTPUTS #########
###################################
############ SNR target's calcullation ################
step = 0.2
interp_step = int(round((max_ant_size - min_ant_size) * 100))
ant_size_vector = np.arange(min_ant_size, max_ant_size, step)
ant_size_vector_interp = np.linspace(min_ant_size, max_ant_size,
interp_step)
# parallel loop for each antenna size
pool = ParallelPool(nodes=round(cores / 2)) #ARRUMAR AQUI
availability_vector = list(
pool.imap(loop_graph_ant_size, [(satellite, margin, 1, ant_size)
for ant_size in ant_size_vector]))
pool.clear()
ant_size_vector = np.array(ant_size_vector)
availability_vector = np.array(availability_vector)
ant_size_vector = ant_size_vector[availability_vector > 60]
availability_vector = availability_vector[availability_vector > 60]
# a_BSpline = interpolate.make_interp_spline(ant_size_vector, availability_vector, k=2)
# availability_vector_interp = a_BSpline(ant_size_vector_interp)
availability_vector_interp = 0
with open('temp\\args.pkl', 'wb') as f:
pickle.dump([
ant_size_vector, ant_size_vector_interp, availability_vector,
availability_vector_interp
], f)
f.close()
return
def mp_ant_size():
with open('temp\\args.pkl',
'rb') as f: # opening the input variables in the temp file
(gr_station_path, sat_long, freq, max_eirp, sat_height, max_bw,
bw_util, modcod, pol, roll_off, ant_eff, lnb_gain, lnb_temp,
aditional_losses, cable_loss, max_depoint, availability_target,
snr_relaxation, margin, threads) = pickle.load(f)
f.close()
# reading the input table
# dir = 'models\\'
# file = 'CitiesBrazil'
# cities = pd.read_csv(dir + file + '.csv', sep=';', encoding='latin1')
# cities['availability'] = np.nan # creating an empty results column
point_list = pd.read_csv(
gr_station_path, sep=';',
encoding='latin1') # creating a point dataframe from csv file
data = pd.read_csv('models\\Modulation_dB.csv', sep=';')
line = data.loc[data.Modcod == modcod]
# tech = line['Tech'].values[0]
mod = line['Modulation'].values[0]
fec = line['FEC'].values[0]
# creating the satellite object
sat = Satellite(sat_long, freq, max_eirp, sat_height, max_bw, bw_util, 0,
0, mod, roll_off, fec)
polarization_loss = 3
reception = Reception(None, ant_eff, aditional_losses, polarization_loss,
lnb_gain, lnb_temp, cable_loss,
max_depoint) # creating the receptor object
# ======================== PARALLEL POOL =============================
pool = ParallelPool(nodes=threads) # creating the parallelPoll
sys.stderr = open('temp\\out.txt', 'w') # to print the output dynamically
print('initializing . . .', file=sys.stderr)
# running the parallel pool
data = list(
tqdm.tqdm(pool.imap(point_ant_size,
[(point, sat, reception, margin, snr_relaxation,
availability_target)
for index, point in point_list.iterrows()]),
total=len(point_list)))
pool.clear()
point_list.drop(point_list.index, inplace=True)
point_list = point_list.append(data, ignore_index=True)
# saving the results into a csv file
dir = 'results'
if not os.path.exists(dir):
os.makedirs(dir)
point_list.to_csv(dir + '\\' + 'results_ant ' +
datetime.datetime.now().strftime('%y-%m-%d_%H-%M-%S') +
'.csv',
sep=';',
encoding='latin1')
print('Complete!!!', file=sys.stderr)
sys.stderr.close()
return
t1 = time()
T1 = t1 - t0
print(result)
print("in time = {0:.3f}".format(T1))
print('')
# parallel calculation
numbers = [
3093215881333057, 3093215881333057, 3093215881333057, 3093215881333057
]
print("{} parallel calculations with {} out of {} CPUs".format(
len(numbers), WORKERS, cpu_count()))
t0 = time()
# create the pool of workers
pool = ParallelPool(WORKERS)
# open the functions in their own threads and return the results
results = pool.map(function, numbers)
pool.close()
pool.join()
t1 = time()
T2 = t1 - t0
print(results)
print("in time = {0:.3f}".format(T2))
print('')
print("ratio = {0:.2f}%".format(100. * T2 / T1))
def simple_search(dataset,
params,
val_accuracy,
N_workers,
restart_num=3,
output_dim=10,
lr=0.001,
epoch_limit=20,
random_seed=29):
print("--------------------------------------------------------------")
print("SAS starts")
print("--------------------------------------------------------------")
print("run for at most " + str(epoch_limit) + " epochs each")
print("running in parallel using " + str(N_workers) + " workers ")
print(datetime.datetime.now())
val_param = copy.deepcopy(params)
architecture = params['architecture']
layer = len(architecture)
instructions = []
'''
instructions are guideline for architecture transformations
'''
instructions.append({'Wider': [layer], 'Deeper': []})
instructions.append({'Wider': [layer, layer], 'Deeper': []})
instructions.append({'Wider': [], 'Deeper': [layer, layer + 1]})
instructions.append({'Wider': [layer], 'Deeper': [layer]})
instructions.append({'Wider': [], 'Deeper': [layer]})
instructions.append({'Wider': [layer, layer], 'Deeper': [layer]})
EveryLayer = np.arange(layer) + 1
#Widen all layers at once
instructions.append({'Wider': EveryLayer, 'Deeper': []})
print("instruction generation complete")
pool = ParallelPool(N_workers)
print(" creating Pool and setting up workers")
num_instructions = len(instructions)
l_dataset = [dataset] * num_instructions
l_output_dim = [output_dim] * num_instructions
l_params = [params] * num_instructions
l_lr = [lr] * num_instructions
l_epoch_limit = [epoch_limit] * num_instructions
l_restart_num = [restart_num] * num_instructions
l_seed = [random_seed] * num_instructions
print("function call preparation complete ")
'''
train the candidates in parallel
'''
candidates = pool.map(Just_Train, l_dataset, l_output_dim, l_params,
instructions, l_lr, l_epoch_limit, l_restart_num,
l_seed)
print("all candidates received")
print(datetime.datetime.now())
best_accu = 0
best_param = 0
'''
identify the best candidate
'''
for candidate in candidates:
if candidate is None:
print("find a none type")
continue
accuracy = candidate['accuracy']
architecture = candidate['params']['architecture']
print("for architecture : " + str(architecture))
print(" achieved validation accuracy of " + str(accuracy))
if (accuracy > best_accu):
best_accu = accuracy
best_param = candidate['params']
if (val_accuracy > best_accu):
best_accu = val_accuracy
best_param = val_param
print("best candidate has architecture")
print(best_param['architecture'])
return {'accuracy': best_accu, 'params': best_param}
break
print("")
y = m.get()
print("I'm awake")
print("y = %s" % str(y[:10]))
if __name__ == '__main__':
x = list(range(500))
delay = 0.01
maxtries = 200
f = busy_add
#f = busy_squared
#f = squared
#from pathos.pools import ProcessPool as Pool
#from pathos.pools import ThreadPool as Pool
from pathos.pools import ParallelPool as Pool
#from pathos.helpers import freeze_support, shutdown
#freeze_support()
pool = Pool(nodes=4)
test_ready(pool, f, maxtries, delay)
# shutdown
pool.close()
pool.join()
pool.clear()
# EOF
def run_ppmap(obj):
from pathos.pools import ParallelPool
p = ParallelPool(2)
x = [1,2,3]
assert list(map(obj, x)) == p.map(obj, x)
def test_ppmap(obj):
from pathos.pools import ParallelPool
p = ParallelPool(2)
x = [1, 2, 3]
assert map(obj, x) == p.map(obj, x)
