def energyMF_pool(self):
start_time=time.time()
k_req_max=np.maximum(self.estimate[:self.N_Al[2]//2],20)
# self.energyall=[[] for _ in self.k_req_max]
# self.wfall=self.energyall
H_bdg_all=[]
self.make_system()
self.k_req_max=k_req_max
for kz_index,k_req in enumerate(k_req_max):
k_req=k_req_max[kz_index]
k_req=np.max([40,2*(1.1*k_req//2).astype(int)])
H_bdg=self.system.hamiltonian_submatrix(params=dict(kz_index=kz_index),sparse=True)
H_bdg=csc_matrix(np.real((H_bdg+H_bdg.T.conj())/2))
H_bdg_all.append(H_bdg)
self.H_bdg_all=H_bdg_all
print('Elapsed time on constructing Hamiltonian is: {:.1f}s'.format(time.time()-start_time))
executor=MPIPoolExecutor()
pool_results=executor.map(diagonalization,[(i,self.E_D,j,k) for i,j,k in zip(H_bdg_all,k_req_max,range(len(k_req_max)))])
executor.shutdown()
self.energyall=[]
self.wfall=[]
for pool_result in pool_results:
pool_val,pool_vec=pool_result
self.energyall.append(pool_val)
self.wfall.append(pool_vec)
if self.store_history:
self.energyall_history.append(self.energyall)
self.wfall_history.append(self.wfall)
print('Elapsed time is: {:.1f}s'.format(time.time()-start_time))
def __init__(self, procs_per_worker=1):
"""
:param procs_per_worker: int
"""
try:
from mpi4py import MPI
from mpi4py.futures import MPIPoolExecutor
except ImportError:
raise ImportError(
'nested: MPIFuturesInterface: problem with importing from mpi4py.futures'
)
self.global_comm = MPI.COMM_WORLD
if procs_per_worker > 1:
print 'nested: MPIFuturesInterface: procs_per_worker reduced to 1; collective operations not yet ' \
'implemented'
self.procs_per_worker = 1
self.executor = MPIPoolExecutor()
self.rank = self.global_comm.rank
self.global_size = self.global_comm.size
self.num_workers = self.global_size - 1
self.apply_counter = 0
self.map = self.map_sync
self.apply = self.apply_sync
self.init_workers(disp=True)
def run_function(function, folder_dataset, list_subj, list_args=[], nb_cpu=None, verbose=1, test_integrity=0):
"""
Run a test function on the dataset using multiprocessing and save the results
:return: results
# results are organized as the following: tuple of (status, output, DataFrame with results)
"""
# add full path to each subject
list_subj_path = [os.path.join(folder_dataset, subject) for subject in list_subj]
# All scripts that are using multithreading with ITK must not use it when using multiprocessing
os.environ["ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS"] = "1"
# create list that finds all the combinations for function + subject path + arguments. Example of one list element:
# ('sct_propseg', os.path.join(path_sct, 'data', 'sct_test_function', '200_005_s2''), '-i ' + os.path.join("t2", "t2.nii.gz") + ' -c t2', 1)
list_func_subj_args = list(itertools.product(*[[function], list_subj_path, list_args, [test_integrity]]))
# data_and_params = itertools.izip(itertools.repeat(function), data_subjects, itertools.repeat(parameters))
logger.debug("stating pool with {} thread(s)".format(nb_cpu))
pool = PoolExecutor(nb_cpu)
compute_time = None
try:
compute_time = time.time()
count = 0
all_results = []
# logger.info('Waiting for results, be patient')
future_dirs = {pool.submit(function_launcher, subject_arg): subject_arg
for subject_arg in list_func_subj_args}
for future in concurrent.futures.as_completed(future_dirs):
count += 1
subject = os.path.basename(future_dirs[future][1])
arguments = future_dirs[future][2]
try:
result = future.result()
sct.no_new_line_log('Processing subjects... {}/{}'.format(count, len(list_func_subj_args)))
all_results.append(result)
except Exception as exc:
logger.error('{} {} generated an exception: {}'.format(subject, arguments, exc))
compute_time = time.time() - compute_time
# concatenate all_results into single Panda structure
results_dataframe = pd.concat(all_results)
except KeyboardInterrupt:
logger.warning("\nCaught KeyboardInterrupt, terminating workers")
for job in future_dirs:
job.cancel()
except Exception as e:
logger.error('Error on line {}'.format(sys.exc_info()[-1].tb_lineno))
logger.exception(e)
for job in future_dirs:
job.cancel()
raise
finally:
pool.shutdown()
return {'results': results_dataframe, "compute_time": compute_time}
class MPIExecutor(BaseExecutor):
'''Executor for parallel execution using MPI
Parameters
----------
kwargs : all kwargs will be passed on to
mpi4py.futures.MPIPoolExecutor
Attributes
----------
pool : concurrent.futures.ProcessPoolExecutor instance
'''
def __init__(self, **kwargs):
super(MPIExecutor, self).__init__()
self.pool = MPIPoolExecutor(**kwargs)
def __exit__(self, exc_type, exc_val, exc_tb):
self.pool.shutdown(wait=True)
return False
def map(self, function, population):
results = self.pool.map(function, population)
population, objectives = list(zip(*results))
objectives = np.asarray(objectives)
return population, objectives
def mutual_info_run_MPI(L=512, es=100, Bp=False):
eta_pos_list = []
MI_pos_list = []
executor = MPIPoolExecutor()
inputs = [(L, Bp) for _ in range(es)]
executor_pool = executor.starmap(MI_pool, inputs)
executor.shutdown()
for result in executor_pool:
eta, MI = result
eta_pos_list.append(eta)
MI_pos_list.append(MI)
return eta_pos_list, MI_pos_list
def run_extrinsic(args) -> None:
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
# The pool executor handles all other workers.
if rank != 0:
return
if args.config_format == "yaml":
cfg = yaml.safe_load(args.config)
elif args.config_format == "json":
cfg = json.load(args.config)
else:
raise ValueError("Currently only json and yaml formats are supported.")
cfg_factory = extrinsic_metaparameters.parser(cfg)
if args.continue_ is None:
results: List[Result] = []
else:
results = parse_existing_trials(args.continue_)
seen = {str(result.config) for result in results}
configs = generate_configs(args.ntrials, seen, cfg_factory)
run = partial(
evaluate,
genome=args.genome,
gff=args.gff,
hints=args.hints,
species=args.species,
tmpdir=args.tmpdir,
softmasking=args.softmasking,
singlestrand=args.singlestrand,
utr=args.utr,
alt_from_evidence=args.alt_from_evidence,
min_intron_len=args.min_intron_len,
allow_hinted_splicesites=args.allow_hinted_splicesites,
augustus_config_path=args.augustus_config_path,
)
# max_workers=None means that the universe size is defined by the master
# mpi process.
executor = MPIPoolExecutor(max_workers=None)
for result in executor.map(run, configs, unordered=True):
results.append(result)
seen.add(str(result.config))
print(json.dumps(result.to_obj()), file=args.outfile)
return
def main():
num_workers = size - 1
executor = MPIPoolExecutor()
for i in xrange(3):
start_time = time.time()
tasks = range(num_workers)
future_list = executor.map(do_work, tasks)
returned_ranks = []
for result in future_list:
returned_ranks.append(result)
used_workers = len(set(returned_ranks))
print 'Map: %i used %i/%i unique workers and took %.4f s' % \
(i, used_workers, num_workers, time.time() - start_time)
if used_workers != num_workers:
pprint.pprint(returned_ranks)
def main():
# Handle CLI
parser = argparse.ArgumentParser()
parser.add_argument("--angle-start", type=float, default=0.0, help="the "
"inclusive lower bound of angles to optimize the "
"slice for (units in degrees, behaves like np.arange)")
parser.add_argument("--angle-stop", type=float, default=1.0, help="the "
"exclusive upper bound of angles to optimize the "
"slice for (units in degrees, behaves like np.arange)")
parser.add_argument("--angle-step", type=float, default=5.0, help="the step size "
"between angle values (units in degrees, behaves "
"like np.arange)")
parser.add_argument("--slice-index", type=int, default=0, help="the "
"slice to perform QOC on (0-7)")
args = vars(parser.parse_args())
angle_start = args["angle_start"]
angle_stop = args["angle_stop"]
angle_step = args["angle_step"]
slice_index = args["slice_index"]
# Trim slices to only include start thru stop.
uccsdslice = UCCSD_LIH_SLICES[slice_index]
# Get the angles to optimize for.
angle_deg_list = list(np.arange(angle_start, angle_stop, angle_step))
# Construct a state for each job. I.e. each angle from the angle
# list on the slice specified.
state_iter = [ProcessState(uccsdslice, slice_index, angle_deg)
for angle_deg in angle_deg_list]
# Run QOC for each process state.
with MPIPoolExecutor(BROADWELL_CORE_COUNT) as executor:
executor.map(process_init, state_iter)
def main(basedir, glob, mask_path, pixel_size, no_mpi, processes, full):
all_paths = list(basedir.glob(glob))
pixel_size *= np.pi / 10800.
with h5py.File(mask_path, 'r') as f2:
tmask = f2['t'][:]
pmask = f2['p'][:]
_get_weight_fsky = partial(get_weight_fsky, tmask, pmask, pixel_size)
if not no_mpi:
from mpi4py.futures import MPIPoolExecutor
with MPIPoolExecutor() as executor:
data = executor.map(_get_weight_fsky, all_paths)
else:
from dautil.util import map_parallel
data = map_parallel(_get_weight_fsky, all_paths, processes=processes)
index = pd.Index(list(map(get_greg, all_paths)),
name='date') if full else pd.MultiIndex.from_tuples(
list(map(get_idx, all_paths)),
names=('nullsplit', 'subsplit', 'date'))
df = pd.DataFrame(data=data,
index=index,
columns=('tweight', 'pweight', 't_fsky', 'p_fsky'))
df.sort_index(inplace=True)
return df
def test_crawl():
with MPIPoolExecutor(10) as executor:
for url, content in executor.map(load_url,
URLS,
timeout=10,
unordered=True):
print('%-25s: %6.2f KiB' % (url, len(content) / (1 << 10)))
def mpirun(n_workers, f, cfg):
mpi_args = dict(max_workers=n_workers) #, path = [])
with MPIPoolExecutor(**mpi_args) as p:
out = p.starmap(
mpistart, zip([cloudpickle.dumps(f)] * n_workers,
range(n_workers)))
return next(out)
def launch_with_MPI_futures(population, config, tries=0):
for individ in population:
individ.failed = False
args = pack_args(population, config)
try:
print(
f"--> Starting MPI Pool executor. {len(args)} jobs running on {len(config.servers)} servers"
)
with MPIPoolExecutor() as executor:
results = [result for result in executor.map(trainer.run, args)]
executor.shutdown(wait=True)
except TypeError as e:
print("Caught the infamous _thread.RLock exception")
print(e)
if tries > 0:
exit(0)
return launch_with_MPI_futures(population, config, tries=tries + 1)
# Exceptions may occur inside the async training loop.
# The failed solutions will be discarded:
original = len(results)
results = [individ for individ in results if not individ.failed]
filtered = len(results)
print(
f"--> Entire population trained. {original-filtered}/{original} failed."
)
for individ in results:
del individ.failed
return results
def main(args):
cap, out = get_video(args)
# Grab the shape of the input
width = int(cap.get(3))
height = int(cap.get(4))
color_state_image = np.zeros((width, height, 3))
alpr = Alpr("eu", "./ALPR/alpr_config/runtime_data/gb.conf",
"./ALPR/alpr_config/runtime_data")
if not alpr.is_loaded():
print("Error loading OpenALPR")
sys.exit(1)
lib_merges = []
while True:
ret, frame = cap.read()
if ret == True:
with MPIPoolExecutor(max_workers=3) as executor:
results = []
for result in executor.map(mpi_function,
range(3), [args]*3, [frame]*3, [cap]*3,
[color_state_image]*3):
results.append(result)
if args.save_video:
out.write(output)
if args.real_time:
cv2.imshow(identifier, output)
else:
break
if out is not None:
out.release()
if cap is not None:
cap.release()
def distribute_tasks(func_task, tasks, num_proc=1, is_distributed=False):
"""Distribute workload.
This function distributes the workload using the ``multiprocessing`` or ``mpi4py`` library.
It simply creates a pool of processes that allow to work on the tasks using shared or
distributed memory.
Notes
-----
We need to ensure that the number of processes is never larger as the number of tasks as
otherwise the MPI implementation does not terminate properly.
* MP Pool, see `here <https://docs.python.org/3/library/multiprocessing.html#multiprocessing.pool.Pool>`_ for details
* MPI Pool, see `here <https://mpi4py.readthedocs.io/en/stable/mpi4py.futures.html#mpipoolexecutor>`__ for details
"""
num_proc_intern = min(len(tasks), num_proc)
if is_distributed:
assert "PMI_SIZE" in os.environ.keys(
), "MPI environment not available."
from mpi4py.futures import MPIPoolExecutor
executor = MPIPoolExecutor(num_proc_intern)
else:
executor = mp.Pool(num_proc_intern)
with executor as e:
rslt = list(e.map(func_task, tasks))
return rslt
def __init__(self, max_workers=0, use_threads=True, mpi=False):
self.workers = max_workers
self.use_threads = use_threads
try:
assert self.workers or self.use_threads
except AssertionError:
print(
'WARNING: Specifying no workers and no threads is not allowed, forcing use_threads=True.'
)
self.use_threads = True
#self.tex = ThreadPoolExecutor(max_workers=1) # MIXING threads and processes DOESN'T WORK
self.processes = []
self.processes_count = 0
self.threads = []
self.threads_count = 0
self.results = {}
self.pex = None
if self.workers:
if mpi:
from mpi4py.futures import MPIPoolExecutor
self.pex = MPIPoolExecutor(max_workers=self.workers)
else:
from concurrent.futures import ProcessPoolExecutor
self.pex = ProcessPoolExecutor(max_workers=self.workers)
def _run_master(self, amount_input_nodes: int, amount_output_nodes,
activation_function, challenge: Challenge,
config: NeatConfig, seed: int):
logger.info("Maser - Name: {}, Size: {}, Rank {}/{}", self.name,
self.size, self.rank, self.size - 1)
logger.info("Waiting for workers to start...")
with MPIPoolExecutor() as self.executor:
self.executor.bootup(wait=True)
logger.info("All Workers should have started")
# Initialize Parameters
innovation_number_generator = InnovationNumberGeneratorSingleCore()
species_id_generator = SpeciesIDGeneratorSingleCore()
agent_id_generator = AgentIDGeneratorSingleCore()
# Notify callback about starting evaluation
self._notify_reporters_callback(lambda r: r.on_initialization())
initial_generation = gs.create_initial_generation(
amount_input_nodes, amount_output_nodes, activation_function,
innovation_number_generator, species_id_generator,
agent_id_generator, config, seed)
finished_generation = self._evaluation_loop(
initial_generation, challenge, innovation_number_generator,
species_id_generator, agent_id_generator, config)
self._notify_reporters_callback(
lambda r: r.on_finish(finished_generation, self.reporters))
def test_julia_pool():
with MPIPoolExecutor() as executor:
print("Loaded Executor")
tic = time.time()
image = list(executor.map(julia_line, range(h), chunksize=10))
toc = time.time()
print("%s - %s Set %dx%d in %.2f seconds." %
(gethostname(), 'Julia', w, h, toc - tic))
def run(self, data):
from mpi4py import MPI
from mpi4py.futures import MPIPoolExecutor
MPI.pickle.__init__(dill.dumps, dill.loads)
with MPIPoolExecutor(max_workers=self.max_workers) as pool:
result = pool.map(self.operations, data)
return result
def get_executor(workers=4, backend='thread'):
if backend == 'mpi':
executor = MPIPoolExecutor(max_workers=workers)
elif backend == 'process':
executor = ProcessPoolExecutor(max_workers=workers)
else:
executor = ThreadPoolExecutor(max_workers=workers)
return executor
def main(args):
with MPIPoolExecutor(max_workers=3) as executor:
executor.map(mpi_function, range(3), [args]*3)
if out is not None:
out.release()
if cap is not None:
cap.release()
def mutual_info_run_MPI(T, es, L):
delta_list = np.linspace(-1, 1, 50)
log_neg_dis_list = []
ensemblesize = es
for delta in delta_list:
log_neg_ensemble_list = []
mutual_info_ensemble_list_pool = []
executor = MPIPoolExecutor()
inputs = [(delta, T, L) for _ in range(ensemblesize)]
mutual_info_ensemble_list_pool = executor.starmap(MI_pool, inputs)
executor.shutdown()
for result in mutual_info_ensemble_list_pool:
LN = result
log_neg_ensemble_list.append(LN)
log_neg_dis_list.append(log_neg_ensemble_list)
return delta_list, log_neg_dis_list
def run_MPIPool(self, fileformat, jobs):
def runGaussian(job):
return self.runGaussianInParallel(fileformat, job)
with MPIPoolExecutor() as executor:
results = executor.map(runGaussian, jobs)
for _ in results:
pass
self._logging("finish jobs:", *jobs)
def run_MPIPool(self, fileformat, jobs):
def runGaussian(job):
return self.runGaussianInParallel(fileformat, job)
with MPIPoolExecutor() as executor:
results = executor.map(runGaussian, jobs)
for _ in results:
pass
logging.info(f"finish {len(jobs)} jobs.")
def mutual_info_run_MPI(s_prob, es=100):
delta_list = np.linspace(-1, 1, 100)**3
mutual_info_dis_list = []
if s_prob == 0 or s_prob == 1:
ensemblesize = 1
else:
ensemblesize = es
for delta in delta_list:
mutual_info_ensemble_list = []
mutual_info_ensemble_list_pool = []
executor = MPIPoolExecutor()
inputs = [(delta, s_prob) for _ in range(ensemblesize)]
mutual_info_ensemble_list_pool = executor.starmap(MI_pool, inputs)
executor.shutdown()
for result in mutual_info_ensemble_list_pool:
mutual_info_ensemble_list.append(result)
mutual_info_dis_list.append(mutual_info_ensemble_list)
return delta_list, mutual_info_dis_list
def mutual_info_run_MPI(s_prob,L):
delta_list=np.linspace(-1,1,100)**3
mutual_info_dis_list=[]
params=Params(delta=0,L=L,bc=-1,basis='m')
proj_range=np.arange(int(params.L/2),params.L,2)
s_list_list=params.generate_position_list(np.arange(int(params.L/2),params.L,2),s_prob)
for delta in delta_list:
mutual_info_ensemble_list=[]
mutual_info_ensemble_list_pool=[]
executor=MPIPoolExecutor()
inputs=[(delta,proj_range,s_list,L) for s_list in (s_list_list)]
mutual_info_ensemble_list_pool=executor.starmap(MI_pool,inputs)
executor.shutdown()
for result in mutual_info_ensemble_list_pool:
mutual_info_ensemble_list.append(result)
mutual_info_dis_list.append(mutual_info_ensemble_list)
return delta_list,mutual_info_dis_list
def _starmap_parallel_mpi(f: 'Callable',
args: 'Iterable[Iterable]',
return_results: 'bool' = True,
**kwargs) -> 'list':
from mpi4py.futures import MPIPoolExecutor
with MPIPoolExecutor() as mpi_pool_executor:
res = mpi_pool_executor.starmap(f, args)
if return_results:
return list(res)
else:
return []
def make_surrogate_L10(self, inflow, cases, seeds, turbine, x_cross,
y_down, outfile):
grid = []
with MPIPoolExecutor(max_workers=num_cores) as executor:
for L10 in executor.map(self.calc_L10, cases, repeat(inflow),
repeat(seeds), repeat(turbine),
repeat(outfile)):
grid.append(L10)
grid = np.reshape(grid, (len(x_cross), len(y_down)))
f_interp = interp2d(x_cross, y_down, grid[:, :])
return f_interp
def _handler(self, request, response):
if 'delay' in request.inputs:
seconds = request.inputs['delay'][0].data
else:
seconds = 1
response.update_status('PyWPS Process started. Waiting...', 50)
with MPIPoolExecutor(max_workers=None, path=[MODULE_PATH]) as executor:
data = [seconds for i in range(4)]
result = executor.map(sleep, data)
print(result)
response.outputs['sleep_output'].data = 'done sleeping'
return response
def mutual_info_run_MPI(T, es, L, mtype):
delta_list = np.linspace(0, 1, 26)
mutual_info_dis_list = []
log_neg_dis_list = []
ensemblesize = es
for delta in delta_list:
log_neg_ensemble_list = []
mutual_info_ensemble_list = []
mutual_info_ensemble_list_pool = []
executor = MPIPoolExecutor()
inputs = [(delta, T, L, mtype) for _ in range(ensemblesize)]
mutual_info_ensemble_list_pool = executor.map(MI_pool, inputs)
executor.shutdown()
for result in mutual_info_ensemble_list_pool:
MI, LN = result
mutual_info_ensemble_list.append(MI)
log_neg_ensemble_list.append(LN)
mutual_info_dis_list.append(mutual_info_ensemble_list)
log_neg_dis_list.append(log_neg_ensemble_list)
return delta_list, log_neg_dis_list
def main():
# Define a data queue. This serves as the connection between the receiver (datastream from KSTAR)
# to the executor that handles the analysis routines
dq = queue.Queue()
msg = AdiosMessage(0, None)
# Dummy data of the same shape as the DFT of a time-chunk.
data = np.zeros([192, 512, 38], dtype=np.complex128)
# Define an executor that handles the analysis tasks
executor = MPIPoolExecutor(max_workers=2)
#executor = concurrent.futures.ThreadPoolExecutor(max_workers=4)
# Start a worker thread that pops element from the queue and dispatches it to
# the executor
worker = threading.Thread(target=consume, args=(dq, executor))
worker.start()
# Start the receiver loop. Here we receive data chunks from remote
for i in range(5):
# Receive time chunk data
logging.info(f"Received time chunk {i}")
# Compile a message with the current data
msg = AdiosMessage(tstep_idx=i, data=data)
# Put the data in the queue
dq.put(msg)
logging.info("Finished the receiver loop")
# Put the hang-up message in the queue
dq.put(AdiosMessage(None, None))
# Close the queue
dq.join()
# Stop the worker process
worker.join()
# Shutdown the MPIPoolExecutor
# This has to be done after the queue has joined!
executor.shutdown()
def main(args):
in_paths = chain(*(glob(glob_i, recursive=True) for glob_i in args.input))
__h5assert_isfinite = partial(_h5assert_isfinite, verbose=args.verbose)
if args.use_mpi:
from mpi4py.futures import MPIPoolExecutor
with MPIPoolExecutor() as executor:
executor.map(__h5assert_isfinite, in_paths)
elif args.p > 1:
from dautil.util import get_map_parallel
map_parallel = get_map_parallel(args.p)
map_parallel(__h5assert_isfinite, in_paths)
else:
list(map(__h5assert_isfinite, in_paths))