def energyMF_pool(self):
start_time=time.time()
k_req_max=np.maximum(self.estimate[:self.N_Al[2]//2],20)
# self.energyall=[[] for _ in self.k_req_max]
# self.wfall=self.energyall
H_bdg_all=[]
self.make_system()
self.k_req_max=k_req_max
for kz_index,k_req in enumerate(k_req_max):
k_req=k_req_max[kz_index]
k_req=np.max([40,2*(1.1*k_req//2).astype(int)])
H_bdg=self.system.hamiltonian_submatrix(params=dict(kz_index=kz_index),sparse=True)
H_bdg=csc_matrix(np.real((H_bdg+H_bdg.T.conj())/2))
H_bdg_all.append(H_bdg)
self.H_bdg_all=H_bdg_all
print('Elapsed time on constructing Hamiltonian is: {:.1f}s'.format(time.time()-start_time))
executor=MPIPoolExecutor()
pool_results=executor.map(diagonalization,[(i,self.E_D,j,k) for i,j,k in zip(H_bdg_all,k_req_max,range(len(k_req_max)))])
executor.shutdown()
self.energyall=[]
self.wfall=[]
for pool_result in pool_results:
pool_val,pool_vec=pool_result
self.energyall.append(pool_val)
self.wfall.append(pool_vec)
if self.store_history:
self.energyall_history.append(self.energyall)
self.wfall_history.append(self.wfall)
print('Elapsed time is: {:.1f}s'.format(time.time()-start_time))
def run_function(function, folder_dataset, list_subj, list_args=[], nb_cpu=None, verbose=1, test_integrity=0):
"""
Run a test function on the dataset using multiprocessing and save the results
:return: results
# results are organized as the following: tuple of (status, output, DataFrame with results)
"""
# add full path to each subject
list_subj_path = [os.path.join(folder_dataset, subject) for subject in list_subj]
# All scripts that are using multithreading with ITK must not use it when using multiprocessing
os.environ["ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS"] = "1"
# create list that finds all the combinations for function + subject path + arguments. Example of one list element:
# ('sct_propseg', os.path.join(path_sct, 'data', 'sct_test_function', '200_005_s2''), '-i ' + os.path.join("t2", "t2.nii.gz") + ' -c t2', 1)
list_func_subj_args = list(itertools.product(*[[function], list_subj_path, list_args, [test_integrity]]))
# data_and_params = itertools.izip(itertools.repeat(function), data_subjects, itertools.repeat(parameters))
logger.debug("stating pool with {} thread(s)".format(nb_cpu))
pool = PoolExecutor(nb_cpu)
compute_time = None
try:
compute_time = time.time()
count = 0
all_results = []
# logger.info('Waiting for results, be patient')
future_dirs = {pool.submit(function_launcher, subject_arg): subject_arg
for subject_arg in list_func_subj_args}
for future in concurrent.futures.as_completed(future_dirs):
count += 1
subject = os.path.basename(future_dirs[future][1])
arguments = future_dirs[future][2]
try:
result = future.result()
sct.no_new_line_log('Processing subjects... {}/{}'.format(count, len(list_func_subj_args)))
all_results.append(result)
except Exception as exc:
logger.error('{} {} generated an exception: {}'.format(subject, arguments, exc))
compute_time = time.time() - compute_time
# concatenate all_results into single Panda structure
results_dataframe = pd.concat(all_results)
except KeyboardInterrupt:
logger.warning("\nCaught KeyboardInterrupt, terminating workers")
for job in future_dirs:
job.cancel()
except Exception as e:
logger.error('Error on line {}'.format(sys.exc_info()[-1].tb_lineno))
logger.exception(e)
for job in future_dirs:
job.cancel()
raise
finally:
pool.shutdown()
return {'results': results_dataframe, "compute_time": compute_time}
class MPIExecutor(BaseExecutor):
'''Executor for parallel execution using MPI
Parameters
----------
kwargs : all kwargs will be passed on to
mpi4py.futures.MPIPoolExecutor
Attributes
----------
pool : concurrent.futures.ProcessPoolExecutor instance
'''
def __init__(self, **kwargs):
super(MPIExecutor, self).__init__()
self.pool = MPIPoolExecutor(**kwargs)
def __exit__(self, exc_type, exc_val, exc_tb):
self.pool.shutdown(wait=True)
return False
def map(self, function, population):
results = self.pool.map(function, population)
population, objectives = list(zip(*results))
objectives = np.asarray(objectives)
return population, objectives
def mutual_info_run_MPI(L=512, es=100, Bp=False):
eta_pos_list = []
MI_pos_list = []
executor = MPIPoolExecutor()
inputs = [(L, Bp) for _ in range(es)]
executor_pool = executor.starmap(MI_pool, inputs)
executor.shutdown()
for result in executor_pool:
eta, MI = result
eta_pos_list.append(eta)
MI_pos_list.append(MI)
return eta_pos_list, MI_pos_list
def mutual_info_run_MPI(T, es, L):
delta_list = np.linspace(-1, 1, 50)
log_neg_dis_list = []
ensemblesize = es
for delta in delta_list:
log_neg_ensemble_list = []
mutual_info_ensemble_list_pool = []
executor = MPIPoolExecutor()
inputs = [(delta, T, L) for _ in range(ensemblesize)]
mutual_info_ensemble_list_pool = executor.starmap(MI_pool, inputs)
executor.shutdown()
for result in mutual_info_ensemble_list_pool:
LN = result
log_neg_ensemble_list.append(LN)
log_neg_dis_list.append(log_neg_ensemble_list)
return delta_list, log_neg_dis_list
def mutual_info_run_MPI(s_prob, es=100):
delta_list = np.linspace(-1, 1, 100)**3
mutual_info_dis_list = []
if s_prob == 0 or s_prob == 1:
ensemblesize = 1
else:
ensemblesize = es
for delta in delta_list:
mutual_info_ensemble_list = []
mutual_info_ensemble_list_pool = []
executor = MPIPoolExecutor()
inputs = [(delta, s_prob) for _ in range(ensemblesize)]
mutual_info_ensemble_list_pool = executor.starmap(MI_pool, inputs)
executor.shutdown()
for result in mutual_info_ensemble_list_pool:
mutual_info_ensemble_list.append(result)
mutual_info_dis_list.append(mutual_info_ensemble_list)
return delta_list, mutual_info_dis_list
def mutual_info_run_MPI(s_prob,L):
delta_list=np.linspace(-1,1,100)**3
mutual_info_dis_list=[]
params=Params(delta=0,L=L,bc=-1,basis='m')
proj_range=np.arange(int(params.L/2),params.L,2)
s_list_list=params.generate_position_list(np.arange(int(params.L/2),params.L,2),s_prob)
for delta in delta_list:
mutual_info_ensemble_list=[]
mutual_info_ensemble_list_pool=[]
executor=MPIPoolExecutor()
inputs=[(delta,proj_range,s_list,L) for s_list in (s_list_list)]
mutual_info_ensemble_list_pool=executor.starmap(MI_pool,inputs)
executor.shutdown()
for result in mutual_info_ensemble_list_pool:
mutual_info_ensemble_list.append(result)
mutual_info_dis_list.append(mutual_info_ensemble_list)
return delta_list,mutual_info_dis_list
def main():
executor = MPIPoolExecutor()
futures = []
for id in range(1, size):
futures.append(executor.submit(report, id))
report(0)
try:
results = [future.result() for future in futures]
num_returned = len(set(results))
print('%i workers completed %i tasks' % (num_workers, num_returned))
sys.stdout.flush()
time.sleep(0.1)
except Exception:
traceback.print_exc(file=sys.stdout)
sys.stdout.flush()
time.sleep(0.1)
executor.shutdown(wait=False)
os._exit(1)
executor.shutdown()
def _mpi_starmap(func_or_classmethod, param_list, *args, path=None):
""""""
if path is not None:
executor = MPIPoolExecutor(path=path)
else:
executor = MPIPoolExecutor()
# print('param_list:', param_list)
# print('args:', args)
# print('actual:', [tuple([param] + list(args)) for param in param_list])
futures = executor.starmap(
func_or_classmethod, [tuple([param] + list(args)) for param in param_list]
)
results = list(futures)
executor.shutdown(wait=True)
return results
def mutual_info_run_MPI(T, es, L, mtype):
delta_list = np.linspace(0, 1, 26)
mutual_info_dis_list = []
log_neg_dis_list = []
ensemblesize = es
for delta in delta_list:
log_neg_ensemble_list = []
mutual_info_ensemble_list = []
mutual_info_ensemble_list_pool = []
executor = MPIPoolExecutor()
inputs = [(delta, T, L, mtype) for _ in range(ensemblesize)]
mutual_info_ensemble_list_pool = executor.map(MI_pool, inputs)
executor.shutdown()
for result in mutual_info_ensemble_list_pool:
MI, LN = result
mutual_info_ensemble_list.append(MI)
log_neg_ensemble_list.append(LN)
mutual_info_dis_list.append(mutual_info_ensemble_list)
log_neg_dis_list.append(log_neg_ensemble_list)
return delta_list, log_neg_dis_list
def main():
# Define a data queue. This serves as the connection between the receiver (datastream from KSTAR)
# to the executor that handles the analysis routines
dq = queue.Queue()
msg = AdiosMessage(0, None)
# Dummy data of the same shape as the DFT of a time-chunk.
data = np.zeros([192, 512, 38], dtype=np.complex128)
# Define an executor that handles the analysis tasks
executor = MPIPoolExecutor(max_workers=2)
#executor = concurrent.futures.ThreadPoolExecutor(max_workers=4)
# Start a worker thread that pops element from the queue and dispatches it to
# the executor
worker = threading.Thread(target=consume, args=(dq, executor))
worker.start()
# Start the receiver loop. Here we receive data chunks from remote
for i in range(5):
# Receive time chunk data
logging.info(f"Received time chunk {i}")
# Compile a message with the current data
msg = AdiosMessage(tstep_idx=i, data=data)
# Put the data in the queue
dq.put(msg)
logging.info("Finished the receiver loop")
# Put the hang-up message in the queue
dq.put(AdiosMessage(None, None))
# Close the queue
dq.join()
# Stop the worker process
worker.join()
# Shutdown the MPIPoolExecutor
# This has to be done after the queue has joined!
executor.shutdown()
def iqtree_trees(fas_dir, output_dir, cores):
output_subdir = os.path.join(output_dir, os.path.basename(fas_dir))
if not os.path.exists(output_subdir):
os.mkdir(output_subdir)
fas_files = []
g = '*.fas'
globs = [g, g.title(), g.upper()]
for g in globs:
fas_files.extend(glob(os.path.join(fas_dir, g)))
tree_prefixes = [
os.path.join(output_subdir,
os.path.splitext(os.path.basename(f))[0])
for f in fas_files
]
arg_list = []
while len(fas_files) > 0:
f = fas_files.pop()
prefix = os.path.splitext(os.path.basename(f))[0]
out_file_dir = os.path.join(output_subdir, prefix)
if not os.path.exists(out_file_dir):
os.mkdir(out_file_dir)
out_file_prefix = os.path.join(out_file_dir, prefix)
arg_list.append((f, out_file_prefix))
ps = Pool(min(int(cores) - 1, len(arg_list)))
ps.map(generate_tree, arg_list)
if getattr(ps, 'shutdown', None):
ps.shutdown()
tree_files = []
for tree_prefix in tree_prefixes:
g = '*.treefile'
globs = [g, g.title(), g.upper()]
for g in globs:
tree_files.extend(glob(os.path.join(tree_prefix, g)))
return tree_files
class MyMPIPool(object):
def __init__(self, **kwargs):
from mpi4py.futures import MPIPoolExecutor
self.real = MPIPoolExecutor(**kwargs)
def map(self, func, args, chunksize=1):
return list(self.real.map(func, args, chunksize=chunksize))
def imap_unordered(self, func, args, chunksize=1):
return result_iter([self.real.submit(func, a) for a in args])
def bootup(self, **kwargs):
return self.real.bootup(**kwargs)
def shutdown(self, **kwargs):
return self.real.shutdown(**kwargs)
def close(self):
self.shutdown()
def join(self):
pass
def apply_async(self, *args, **kwargs):
raise RuntimeError('APPLY_ASYNC NOT IMPLEMENTED IN MyMPIPool')
def get_worker_cpu(self):
return 0.
def get_worker_wall(self):
return 0.
def get_pickle_traffic(self):
return None
def get_pickle_traffic_string(self):
return 'nope'
def main():
comm = MPI.COMM_WORLD
rank = comm.Get_rank()
size = comm.Get_size()
# Parse command line arguments and read configuration file
parser = argparse.ArgumentParser(
description="Receive data and dispatch analysis tasks to a mpi queue")
parser.add_argument('--config',
type=str,
help='Lists the configuration file',
default='configs/config_null.json')
parser.add_argument('--benchmark', action="store_true")
args = parser.parse_args()
global cfg
with open(args.config, "r") as df:
cfg = json.load(df)
df.close()
# Load logger configuration from file:
# http://zetcode.com/python/logging/
with open("configs/logger.yaml", "r") as f:
log_cfg = yaml.safe_load(f.read())
logging.config.dictConfig(log_cfg)
logger = logging.getLogger('simple')
# Create a global executor
#executor = concurrent.futures.ThreadPoolExecutor(max_workers=60)
executor_fft = MPIPoolExecutor(max_workers=16)
executor_anl = MPIPoolExecutor(max_workers=16)
#executor = MPIPoolExecutor(max_workers=120)
adios2_varname = channel_range.from_str(
cfg["transport_nersc"]["channel_range"][0])
cfg["run_id"] = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(6))
cfg["run_id"] = "ABC128"
cfg["storage"]["run_id"] = cfg["run_id"]
logger.info(f"Starting run {cfg['run_id']}")
# Instantiate a storage backend and store the run configuration and task configuration
if cfg['storage']['backend'] == "numpy":
store_backend = backends.backend_numpy(cfg['storage'])
elif cfg['storage']['backend'] == "mongo":
store_backend = backends.backend_mongodb(cfg["storage"])
elif cfg['storage']['backend'] == "null":
store_backend = backends.backend_null(cfg['storage'])
else:
raise NameError(
f"Unknown storage backend requested: {cfg['storage']['backend']}")
store_backend.store_one({"run_id": cfg['run_id'], "run_config": cfg})
logger.info(f"Stored one")
# Create ADIOS reader object
reader = reader_gen(cfg["transport_nersc"])
task_list = task_list_spectral(executor_anl, executor_fft,
cfg["task_list"], cfg["fft_params"],
cfg["ECEI_cfg"], cfg["storage"])
#task_list = task_list_spectral(executor, cfg["task_list"], cfg["fft_params"], cfg["ECEI_cfg"], cfg["storage"])
dq = queue.Queue()
msg = None #
tic_main = timeit.default_timer()
workers = []
for _ in range(4):
worker = threading.Thread(target=consume, args=(dq, task_list))
worker.start()
workers.append(worker)
# reader.Open() is blocking until it opens the data file or receives the
# data stream. Put this right before entering the main loop
logger.info(f"{rank} Waiting for generator")
reader.Open()
logger.info(f"Starting main loop")
rx_list = []
while True:
stepStatus = reader.BeginStep()
if stepStatus:
# Read data
stream_data = reader.Get(adios2_varname, save=False)
rx_list.append(reader.CurrentStep())
# Generate message id and publish
msg = AdiosMessage(tstep_idx=reader.CurrentStep(),
data=stream_data)
dq.put_nowait(msg)
logger.info(f"Published tidx {msg.tstep_idx}")
reader.EndStep()
else:
logger.info(f"Exiting: StepStatus={stepStatus}")
break
if reader.CurrentStep() > 100:
break
dq.join()
logger.info("Queue joined")
logger.info("Exiting main loop")
for thr in workers:
thr.join()
logger.info("Workers have joined")
# Shotdown the executioner
executor_anl.shutdown(wait=True)
executor_fft.shutdown(wait=True)
#executor.shutdown(wait=True)
toc_main = timeit.default_timer()
logger.info(
f"Run {cfg['run_id']} finished in {(toc_main - tic_main):6.4f}s")
logger.info(f"Processed {len(rx_list)} time_chunks: {rx_list}")
def run_function(function,
folder_dataset,
list_subj,
list_args=[],
nb_cpu=None,
verbose=1,
test_integrity=0):
"""
Run a test function on the dataset using multiprocessing and save the results
:return: results
# results are organized as the following: tuple of (status, output, DataFrame with results)
"""
# add full path to each subject
list_subj_path = [
os.path.join(folder_dataset, subject) for subject in list_subj
]
# All scripts that are using multithreading with ITK must not use it when using multiprocessing
os.environ["ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS"] = "1"
# create list that finds all the combinations for function + subject path + arguments. Example of one list element:
# ('sct_propseg', os.path.join(path_sct, 'data', 'sct_test_function', '200_005_s2''), '-i ' + os.path.join("t2", "t2.nii.gz") + ' -c t2', 1)
list_func_subj_args = list(
itertools.product(
*[[function], list_subj_path, list_args, [test_integrity]]))
# data_and_params = itertools.izip(itertools.repeat(function), data_subjects, itertools.repeat(parameters))
sct.log.debug("stating pool with {} thread(s)".format(nb_cpu))
pool = PoolExecutor(nb_cpu)
compute_time = None
try:
compute_time = time.time()
count = 0
all_results = []
# sct.log.info('Waiting for results, be patient')
future_dirs = {
pool.submit(function_launcher, subject_arg): subject_arg
for subject_arg in list_func_subj_args
}
for future in concurrent.futures.as_completed(future_dirs):
count += 1
subject = os.path.basename(future_dirs[future][1])
arguments = future_dirs[future][2]
try:
result = future.result()
sct.no_new_line_log('Processing subjects... {}/{}'.format(
count, len(list_func_subj_args)))
all_results.append(result)
except Exception as exc:
sct.log.error('{} {} generated an exception: {}'.format(
subject, arguments, exc))
compute_time = time.time() - compute_time
# concatenate all_results into single Panda structure
results_dataframe = pd.concat(all_results)
except KeyboardInterrupt:
sct.log.warning("\nCaught KeyboardInterrupt, terminating workers")
for job in future_dirs:
job.cancel()
except Exception as e:
sct.log.error('Error on line {}'.format(sys.exc_info()[-1].tb_lineno))
sct.log.exception(e)
for job in future_dirs:
job.cancel()
raise
finally:
pool.shutdown()
return {'results': results_dataframe, "compute_time": compute_time}
eta_list.append(eta)
delta = 0
for dis_i, (disorder,
disorder_J) in enumerate(zip(disorder_map,
disorder_J_map)):
for es_i in range(es):
inputs.append(
(subregion[0], subregion[2], subregion[1][::2], L,
disorder, disorder_J, delta, size_A_i, dis_i, es_i))
eta_list = np.array(eta_list)
pool = executor.map(run, inputs)
# LN0_list=np.zeros((L//2-1,len(disorder_map),es))
# LN1_list=np.zeros((L//2-1,len(disorder_map),es))
LN0_list = np.zeros((ps, len(disorder_map), es))
LN1_list = np.zeros((ps, len(disorder_map), es))
for _, result in enumerate(pool):
LN0, LN1, size_A_i, dis_i, es_i = result
LN0_list[size_A_i, dis_i, es_i] = (LN0)
LN1_list[size_A_i, dis_i, es_i] = (LN1)
executor.shutdown()
with open(
'SSH_disorder_delta0_L{:d}_var{:.1f}_es{:d}_ds{:d}_ps{:d}_{:s}.pickle'
.format(L, var, es, ds, ps, disorder_type), 'wb') as f:
pickle.dump([
eta_list, LN0_list, LN1_list, disorder_map, disorder_J_map, delta
], f)
print('{:f}'.format(time.time() - st))
output_file = os.path.join(
current_dir, 'tmp_%dto%d.treefile' % (i, len(tree_files)))
try:
os.remove(output_file)
except FileNotFoundError:
pass
with NamedTemporaryFile('w', dir=output_dir, delete=False) as t:
for treefile in tree_files[i:]:
t.write(open(treefile, 'r').read())
input_files.append(t.name)
output_files.append(output_file)
ps = Pool(min(int(args.cores) - 1, len(input_files)))
ps.map(astral_tree,
zip([args.astral] * len(input_files), input_files, output_files))
if getattr(ps, 'shutdown', None):
ps.shutdown()
for output_file in output_files:
validate(output_file)
if len(best_output) == 1:
os.rename(best_output[0], final_output_file)
print('Best rank was %f with output in %s' %
(best_rank, final_output_file))
else:
print('Best rank was %f with output in %s' %
(best_rank, str(best_output)))
for cleanup_file in input_files + output_files:
if cleanup_file in best_output:
continue
try:
os.remove(cleanup_file)
except FileNotFoundError:
def main():
"""Procesess a stream of data chunks on an executor."""
comm = MPI.COMM_WORLD
# Parse command line arguments and read configuration file
parser = argparse.ArgumentParser(
description="Receive data and dispatch analysis" +
"tasks to a mpi queue")
parser.add_argument('--config',
type=str,
help='Lists the configuration file',
default='configs/config_null.json')
parser.add_argument(
"--num_ranks_preprocess",
type=int,
help="Number of processes used in preprocessing executor",
default=4)
parser.add_argument("--num_ranks_analysis",
type=int,
help="Number of processes used in analysis executor",
default=4)
parser.add_argument(
"--num_queue_threads",
type=int,
help="Number of worker threads that consume item from the queue",
default=4)
parser.add_argument(
"--transport",
type=str,
help="Specifies the transport section used to configure the reader",
default="transport_rx")
parser.add_argument(
"--run_id",
type=str,
help="Name of database collection to store analysis results in",
required=True)
args = parser.parse_args()
with open(args.config, "r") as df:
cfg = json.load(df)
df.close()
# Load logger configuration from file:
# http://zetcode.com/python/logging/
with open("configs/logger.yaml", "r") as f:
log_cfg = yaml.safe_load(f.read())
logging.config.dictConfig(log_cfg)
logger = logging.getLogger('simple')
# PoolExecutor for pre-processing, on-node.
executor_pre = ThreadPoolExecutor(max_workers=args.num_ranks_preprocess)
# PoolExecutor for data analysis. off-node
executor_anl = MPIPoolExecutor(max_workers=args.num_ranks_analysis)
stream_varname = gen_var_name(cfg)[0]
cfg["run_id"] = args.run_id
cfg["storage"]["run_id"] = cfg["run_id"]
logger.info(f"Starting run {cfg['run_id']}")
# Instantiate a storage backend and store the run configuration and task configuration
store_type = get_storage_object(cfg["storage"])
store_backend = store_type(cfg["storage"])
store_backend.store_one({"run_id": cfg['run_id'], "run_config": cfg})
# TODO: (RMC) Should this be moved to where cfg updated?
# (would allow updating channels to process remotely)
reader = reader_gen(cfg[args.transport],
gen_channel_name(cfg["diagnostic"]))
reader.Open()
dq = queue.Queue()
# In a streaming setting, (SST, dataman) attributes can only be accessed after
# reading the first time step of a variable.
# Initialize stream_attrs with None and load it in the main loop below.
stream_attrs = None
data_model_gen = data_model_generator(cfg["diagnostic"])
my_preprocessor = preprocessor(executor_pre, cfg)
my_task_list = tasklist(executor_anl, cfg)
worker_thread_list = []
for _ in range(args.num_queue_threads):
new_worker = threading.Thread(target=consume,
args=(dq, my_task_list, my_preprocessor))
new_worker.start()
worker_thread_list.append(new_worker)
logger.info("Starting main loop")
tic_main = time.perf_counter()
rx_list = []
while True:
stepStatus = reader.BeginStep(timeoutSeconds=5.0)
if stepStatus:
# Load attributes
if stream_attrs is None:
logger.info("Waiting for attributes")
stream_attrs = reader.get_attrs("stream_attrs")
logger.info(f"Got attributes: {stream_attrs}")
# Read data
stream_data = reader.Get(stream_varname, save=False)
# if reader.CurrentStep() in [0, 140]:
rx_list.append(reader.CurrentStep())
# Create a datamodel instance from the raw data and push into the queue
msg = data_model_gen.new_chunk(stream_data, stream_attrs,
reader.CurrentStep())
dq.put_nowait(msg)
logger.info(f"Published tidx {reader.CurrentStep()}")
reader.EndStep()
else:
logger.info(f"Exiting: StepStatus={stepStatus}")
break
if reader.CurrentStep() > 100:
break
dq.join()
logger.info("Queue joined")
logger.info("Exiting main loop")
for thr in worker_thread_list:
thr.join()
logger.info("Workers have joined")
# Shutdown the executioner
executor_anl.shutdown(wait=True)
executor_pre.shutdown(wait=True)
# executor.shutdown(wait=True)
toc_main = time.perf_counter()
logger.info(
f"Run {cfg['run_id']} finished in {(toc_main - tic_main):6.4f}s")
logger.info(f"Processed {len(rx_list)} time_chunks: {rx_list}")
class MPIFuturesInterface(object):
"""
Class provides an interface to extend the mpi4py.futures concurrency tools for flexible nested parallel
computations.
"""
class AsyncResultWrapper(object):
"""
When ready(), get() returns results as a list in the same order as submission.
"""
def __init__(self, futures):
"""
:param futures: list of :class:'mpi4py.futures.Future'
"""
self.futures = futures
self._ready = False
def ready(self, wait=None):
"""
:param wait: int or float
:return: bool
"""
time_stamp = time.time()
if wait is None:
wait = 0
while not np.all([future.done() for future in self.futures]):
if time.time() - time_stamp > wait:
return False
self._ready = True
return True
def get(self):
"""
Returns None until all results have completed, then returns a list of results in the order of original
submission.
:return: list
"""
if self._ready or self.ready():
return [future.result() for future in self.futures]
else:
return None
def __init__(self, procs_per_worker=1):
"""
:param procs_per_worker: int
"""
try:
from mpi4py import MPI
from mpi4py.futures import MPIPoolExecutor
except ImportError:
raise ImportError(
'nested: MPIFuturesInterface: problem with importing from mpi4py.futures'
)
self.global_comm = MPI.COMM_WORLD
if procs_per_worker > 1:
print 'nested: MPIFuturesInterface: procs_per_worker reduced to 1; collective operations not yet ' \
'implemented'
self.procs_per_worker = 1
self.executor = MPIPoolExecutor()
self.rank = self.global_comm.rank
self.global_size = self.global_comm.size
self.num_workers = self.global_size - 1
self.apply_counter = 0
self.map = self.map_sync
self.apply = self.apply_sync
self.init_workers(disp=True)
def init_workers(self, disp=False):
"""
:param disp: bool
"""
futures = []
for task_id in xrange(1, self.global_size):
futures.append(
self.executor.submit(mpi_futures_init_worker, task_id, disp))
results = [future.result() for future in futures]
num_returned = len(set(results))
if num_returned != self.num_workers:
raise ValueError(
'nested: MPIFuturesInterface: %i / %i processes returned from init_workers'
% (num_returned, self.num_workers))
self.print_info()
def print_info(self):
"""
"""
print 'nested: MPIFuturesInterface: process id: %i; rank: %i / %i; num_workers: %i' % \
(os.getpid(), self.rank, self.global_size, self.num_workers)
sys.stdout.flush()
time.sleep(0.1)
def apply_sync(self, func, *args, **kwargs):
"""
mpi4py.futures lacks a native method to guarantee execution of a function on all workers. This method
implements a synchronous (blocking) apply operation that accepts **kwargs and returns values collected from each
worker.
:param func: callable
:param args: list
:param kwargs: dict
:return: dynamic
"""
apply_key = int(self.apply_counter)
self.apply_counter += 1
futures = []
for rank in xrange(1, self.global_size):
futures.append(
self.executor.submit(mpi_futures_apply_wrapper, func,
apply_key, args, kwargs))
results = [future.result() for future in futures]
return results
def map_sync(self, func, *sequences):
"""
This method wraps mpi4py.futures.MPIPoolExecutor.map to implement a synchronous (blocking) map operation.
Uses all available processes, and returns results as a list in the same order as the specified sequences.
:param func: callable
:param sequences: list
:return: list
"""
if not sequences:
return None
results = []
for result in self.executor.map(func, *sequences):
results.append(result)
return results
def map_async(self, func, *sequences):
"""
This method wraps mpi4py.futures.MPIPoolExecutor.submit to implement an asynchronous (non-blocking) map
operation. Uses all available processes, and returns results as a list in the same order as the specified
sequences. Returns an AsyncResultWrapper object to track progress of the submitted jobs.
:param func: callable
:param sequences: list
:return: list
"""
if not sequences:
return None
futures = []
for args in zip(*sequences):
futures.append(self.executor.submit(func, *args))
return self.AsyncResultWrapper(futures)
def get(self, object_name):
"""
mpi4py.futures lacks a native method to get the value of an object from all workers. This method implements a
synchronous (blocking) pull operation.
:param object_name: str
:return: dynamic
"""
return self.apply_sync(find_nested_object, object_name)
def start(self, disp=False):
pass
def stop(self):
self.executor.shutdown()
def ensure_controller(self):
"""
Exceptions in python on an MPI rank are not enough to end a job. Strange behavior results when an unhandled
Exception occurs on an MPI rank while under the control of an mpi4py.futures.MPIPoolExecutor. This method will
hard exit python if executed by any rank other than the master.
"""
if self.rank != 0:
os._exit(1)
def RunTMCMC(N,
AllPars,
Nm_steps_max,
Nm_steps_maxmax,
log_likelihood,
variables,
workdirMain,
seed,
calibrationData,
numExperiments,
covarianceMatrixList,
edpNamesList,
edpLengthsList,
scaleFactors,
shiftFactors,
run_type,
logFile,
MPI_size,
parallelizeMCMC=True):
""" Runs TMCMC Algorithm """
# Initialize (beta, effective sample size)
beta = 0
ESS = N
mytrace = []
totalNumberOfModelEvaluations = N
# Initialize other TMCMC variables
Nm_steps = Nm_steps_max
Adap_calc_Nsteps = 'yes' # yes or no
Adap_scale_cov = 'yes' # yes or no
scalem = 1 # cov scale factor
evidence = 1 # model evidence
stageNum = 0 # stage number of TMCMC
logFile.write('\n\n\t\t==========================')
logFile.write("\n\t\tStage number: {}".format(stageNum))
logFile.write("\n\t\tSampling from prior")
logFile.write("\n\t\tbeta = 0")
logFile.write("\n\t\tESS = %d" % ESS)
logFile.write("\n\t\tscalem = %.2f" % scalem)
logFile.write(
"\n\n\t\tNumber of model evaluations in this stage: {}".format(N))
logFile.flush()
os.fsync(logFile.fileno())
# initial samples
Sm = tmcmcFunctions.initial_population(N, AllPars)
# Evaluate posterior at Sm
Priorm = np.array([tmcmcFunctions.log_prior(s, AllPars)
for s in Sm]).squeeze()
Postm = Priorm # prior = post for beta = 0
# Evaluate log-likelihood at current samples Sm
logFile.write("\n\n\t\tRun type: {}".format(run_type))
if parallelizeMCMC:
if run_type == "runningLocal":
procCount = mp.cpu_count()
pool = Pool(processes=procCount)
logFile.write(
"\n\n\t\tCreated multiprocessing pool for runType: {}".format(
run_type))
logFile.write("\n\t\t\tNumber of processors being used: {}".format(
procCount))
Lmt = pool.starmap(
runFEM,
[(ind, Sm[ind], variables, workdirMain, log_likelihood,
calibrationData, numExperiments, covarianceMatrixList,
edpNamesList, edpLengthsList, scaleFactors, shiftFactors)
for ind in range(N)],
)
else:
from mpi4py.futures import MPIPoolExecutor
executor = MPIPoolExecutor(max_workers=MPI_size)
logFile.write(
"\n\n\t\tCreated mpi4py executor pool for runType: {}".format(
run_type))
logFile.write("\n\t\t\tmax_workers: {}".format(MPI_size))
iterables = [
(ind, Sm[ind], variables, workdirMain, log_likelihood,
calibrationData, numExperiments, covarianceMatrixList,
edpNamesList, edpLengthsList, scaleFactors, shiftFactors)
for ind in range(N)
]
Lmt = list(executor.starmap(runFEM, iterables))
Lm = np.array(Lmt).squeeze()
else:
logFile.write("\n\n\t\tNot parallelized")
logFile.write("\n\t\t\tNumber of processors being used: {}".format(1))
Lm = np.array([
runFEM(ind, Sm[ind], variables, workdirMain, log_likelihood,
calibrationData, numExperiments, covarianceMatrixList,
edpNamesList, edpLengthsList, scaleFactors, shiftFactors)
for ind in range(N)
]).squeeze()
logFile.write(
"\n\n\t\tTotal number of model evaluations so far: {}".format(
totalNumberOfModelEvaluations))
# Write the results of the first stage to a file named dakotaTabPrior.out for quoFEM to be able to read the results
logFile.write(
"\n\n\t\tWriting prior samples to 'dakotaTabPrior.out' for quoFEM to read the results"
)
tabFilePath = os.path.join(workdirMain, "dakotaTabPrior.out")
writeOutputs = True
# Create the headings, which will be the first line of the file
logFile.write("\n\t\t\tCreating headings")
headings = 'eval_id\tinterface\t'
for v in variables['names']:
headings += '{}\t'.format(v)
if writeOutputs: # create headings for outputs
for i, edp in enumerate(edpNamesList):
if edpLengthsList[i] == 1:
headings += '{}\t'.format(edp)
else:
for comp in range(edpLengthsList[i]):
headings += '{}_{}\t'.format(edp, comp + 1)
headings += '\n'
# Get the data from the first stage
logFile.write("\n\t\t\tGetting data from first stage")
dataToWrite = Sm
logFile.write("\n\t\t\tWriting to file {}".format(tabFilePath))
with open(tabFilePath, "w") as f:
f.write(headings)
for i in range(N):
string = "{}\t{}\t".format(i + 1, 1)
for j in range(len(variables['names'])):
string += "{}\t".format(dataToWrite[i, j])
if writeOutputs: # write the output data
workdirString = ("workdir." + str(i + 1))
prediction = np.atleast_2d(
np.genfromtxt(
os.path.join(workdirMain, workdirString,
'results.out'))).reshape((1, -1))
for predNum in range(np.shape(prediction)[1]):
string += "{}\t".format(prediction[0, predNum])
string += "\n"
f.write(string)
logFile.write('\n\t\t==========================')
logFile.flush()
os.fsync(logFile.fileno())
while beta < 1:
# adaptively compute beta s.t. ESS = N/2 or ESS = 0.95*prev_ESS
# plausible weights of Sm corresponding to new beta
beta, Wm, ESS = tmcmcFunctions.compute_beta(beta,
Lm,
ESS,
threshold=0.95)
# beta, Wm, ESS = tmcmcFunctions.compute_beta(beta, Lm, ESS, threshold=0.5)
stageNum += 1
# seed to reproduce results
ss = SeedSequence(seed)
child_seeds = ss.spawn(N + 1)
# update model evidence
evidence = evidence * (sum(Wm) / N)
# Calculate covariance matrix using Wm_n
Cm = np.cov(Sm, aweights=Wm / sum(Wm), rowvar=False)
# logFile.write("\nCovariance matrix: {}".format(Cm))
# Resample ###################################################
# Resampling using plausible weights
# SmcapIDs = np.random.choice(range(N), N, p=Wm / sum(Wm))
rng = default_rng(child_seeds[-1])
SmcapIDs = rng.choice(range(N), N, p=Wm / sum(Wm))
# SmcapIDs = resampling.stratified_resample(Wm_n)
Smcap = Sm[SmcapIDs]
Lmcap = Lm[SmcapIDs]
Postmcap = Postm[SmcapIDs]
# save to trace
# stage m: samples, likelihood, weights, next stage ESS, next stage beta, resampled samples
mytrace.append([Sm, Lm, Wm, ESS, beta, Smcap])
# Write Data to '.csv' files
dataToWrite = mytrace[stageNum - 1][0]
logFile.write(
"\n\n\t\tWriting samples from stage {} to csv file".format(
stageNum - 1))
stringToAppend = 'resultsStage{}.csv'.format(stageNum - 1)
resultsFilePath = os.path.join(os.path.abspath(workdirMain),
stringToAppend)
with open(resultsFilePath, 'w', newline='') as csvfile:
csvWriter = csv.writer(csvfile)
csvWriter.writerows(dataToWrite)
logFile.write("\n\t\t\tWrote to file {}".format(resultsFilePath))
# Finished writing data
logFile.write('\n\n\t\t==========================')
logFile.write("\n\t\tStage number: {}".format(stageNum))
if beta < 1e-7:
logFile.write("\n\t\tbeta = %9.6e" % beta)
else:
logFile.write("\n\t\tbeta = %9.8f" % beta)
logFile.write("\n\t\tESS = %d" % ESS)
logFile.write("\n\t\tscalem = %.2f" % scalem)
# Perturb ###################################################
# perform MCMC starting at each Smcap (total: N) for Nm_steps
Em = (scalem**2) * Cm # Proposal dist covariance matrix
numProposals = N * Nm_steps
totalNumberOfModelEvaluations += numProposals
logFile.write(
"\n\n\t\tNumber of model evaluations in this stage: {}".format(
numProposals))
logFile.flush()
os.fsync(logFile.fileno())
numAccepts = 0
if parallelizeMCMC:
if run_type == "runningLocal":
logFile.write("\n\n\t\tLocal run - MCMC steps")
logFile.write(
"\n\t\t\tNumber of processors being used: {}".format(
procCount))
results = pool.starmap(
tmcmcFunctions.MCMC_MH,
[(j1, Em, Nm_steps, Smcap[j1], Lmcap[j1], Postmcap[j1],
beta, numAccepts, AllPars, log_likelihood, variables,
workdirMain, default_rng(child_seeds[j1]),
calibrationData, numExperiments, covarianceMatrixList,
edpNamesList, edpLengthsList, scaleFactors, shiftFactors)
for j1 in range(N)],
)
else:
logFile.write("\n\n\t\tRemote run - MCMC steps")
logFile.write("\n\t\t\tmax_workers: {}".format(MPI_size))
iterables = [
(j1, Em, Nm_steps, Smcap[j1], Lmcap[j1], Postmcap[j1],
beta, numAccepts, AllPars, log_likelihood, variables,
workdirMain, default_rng(child_seeds[j1]),
calibrationData, numExperiments, covarianceMatrixList,
edpNamesList, edpLengthsList, scaleFactors, shiftFactors)
for j1 in range(N)
]
results = list(
executor.starmap(tmcmcFunctions.MCMC_MH, iterables))
else:
logFile.write("\n\n\t\tLocal run - MCMC steps, not parallelized")
logFile.write(
"\n\t\t\tNumber of processors being used: {}".format(1))
results = [
tmcmcFunctions.MCMC_MH(j1, Em, Nm_steps, Smcap[j1], Lmcap[j1],
Postmcap[j1], beta, numAccepts, AllPars,
log_likelihood, variables, workdirMain,
default_rng(child_seeds[j1]),
calibrationData, numExperiments,
covarianceMatrixList, edpNamesList,
edpLengthsList, scaleFactors,
shiftFactors) for j1 in range(N)
]
Sm1, Lm1, Postm1, numAcceptsS, all_proposals, all_PLP = zip(*results)
Sm1 = np.asarray(Sm1)
Lm1 = np.asarray(Lm1)
Postm1 = np.asarray(Postm1)
numAcceptsS = np.asarray(numAcceptsS)
numAccepts = sum(numAcceptsS)
all_proposals = np.asarray(all_proposals)
all_PLP = np.asarray(all_PLP)
logFile.write(
"\n\n\t\tTotal number of model evaluations so far: {}".format(
totalNumberOfModelEvaluations))
# total observed acceptance rate
R = numAccepts / numProposals
if R < 1e-5:
logFile.write("\n\n\t\tacceptance rate = %9.5e" % R)
else:
logFile.write("\n\n\t\tacceptance rate = %.6f" % R)
# Calculate Nm_steps based on observed acceptance rate
if Adap_calc_Nsteps == 'yes':
# increase max Nmcmc with stage number
Nm_steps_max = min(Nm_steps_max + 1, Nm_steps_maxmax)
logFile.write("\n\t\tadapted max MCMC steps = %d" % Nm_steps_max)
acc_rate = max(1. / numProposals, R)
Nm_steps = min(Nm_steps_max,
1 + int(np.log(1 - 0.99) / np.log(1 - acc_rate)))
logFile.write("\n\t\tnext MCMC Nsteps = %d" % Nm_steps)
logFile.write('\n\t\t==========================')
# scale factor based on observed acceptance ratio
if Adap_scale_cov == 'yes':
scalem = (1 / 9) + ((8 / 9) * R)
# for next beta
Sm, Postm, Lm = Sm1, Postm1, Lm1
# save to trace
mytrace.append([Sm, Lm, np.ones(len(Wm)), 'notValid', 1, 'notValid'])
# Write last stage data to '.csv' file
dataToWrite = mytrace[stageNum][0]
logFile.write(
"\n\n\t\tWriting samples from stage {} to csv file".format(stageNum))
stringToAppend = 'resultsStage{}.csv'.format(stageNum)
resultsFilePath = os.path.join(os.path.abspath(workdirMain),
stringToAppend)
with open(resultsFilePath, 'w', newline='') as csvfile:
csvWriter = csv.writer(csvfile)
csvWriter.writerows(dataToWrite)
logFile.write("\n\t\t\tWrote to file {}".format(resultsFilePath))
if parallelizeMCMC == 'yes':
if run_type == "runningLocal":
pool.close()
logFile.write(
"\n\tClosed multiprocessing pool for runType: {}".format(
run_type))
else:
executor.shutdown()
logFile.write(
"\n\tShutdown mpi4py executor pool for runType: {}".format(
run_type))
return mytrace