def test_update_p_cat(self):
popex = PoPEx(ncmax=self.ncmax,
nmtype=self.nmtype,
path_res=PATH_TEST)
for imod in range(self.nmod_init):
mod = generate_m()
p_lik = compute_log_p_lik(mod)
log_p_pri = compute_log_p_pri(mod)
log_p_gen = compute_log_p_gen(mod)
popex.add_model(imod, mod, p_lik, True, log_p_pri, log_p_gen,
ncmod=(0, 0, 0))
w_pred = utl.compute_w_pred(popex)
p_cat_upd = utl.compute_cat_prob(popex, w_pred)
m_new = []
for imod in range(self.nmod_sup):
mod = generate_m()
m_new.append(mod)
p_lik = compute_log_p_lik(mod)
log_p_pri = compute_log_p_pri(mod)
log_p_gen = compute_log_p_gen(mod)
popex.add_model(imod+self.nmod_init, mod,
p_lik, True, log_p_pri, log_p_gen,
ncmod=(0, 0, 0))
w_pred = utl.compute_w_pred(popex)
p_cat_cmp = utl.compute_cat_prob(popex, w_pred)
sum_w_old = np.sum(w_pred[:self.nmod_init])
w_new = w_pred[self.nmod_init:]
utl.update_cat_prob(p_cat_upd, m_new, w_new, sum_w_old)
for imtype in self.ncond_mtype:
self.assertTrue(p_cat_upd[imtype] is None)
for imtype in self.cond_mtype:
self.assertTrue(isinstance(p_cat_upd[imtype], MType))
self.assertTrue(isinstance(p_cat_upd[imtype], CatProb))
self.assertTrue(
p_cat_upd[imtype].param_val.shape[0] == p_cat_upd[imtype].nparam)
self.assertTrue(
p_cat_upd[imtype].param_val.shape[1] == p_cat_upd[imtype].ncat)
for imtype in self.cond_mtype:
diff = p_cat_cmp[imtype].param_val - p_cat_upd[imtype].param_val
self.assertTrue(np.all(np.abs(diff) < NP_MIN_TOL))
def test_compute_p_cat(self):
popex = PoPEx(ncmax=self.ncmax,
nmtype=self.nmtype,
path_res=PATH_TEST)
for imod in range(self.nmod_init):
mod = generate_m()
p_lik = compute_log_p_lik(mod)
log_p_pri = compute_log_p_pri(mod)
log_p_gen = compute_log_p_gen(mod)
popex.add_model(0, mod, p_lik, True, log_p_pri, log_p_gen, (2, 5, 0))
w_pred = utl.compute_w_pred(popex)
p_cat = utl.compute_cat_prob(popex, w_pred)
for imtype in self.ncond_mtype:
self.assertTrue(p_cat[imtype] is None)
for imtype in self.cond_mtype:
self.assertTrue(isinstance(p_cat[imtype], MType))
self.assertTrue(isinstance(p_cat[imtype], CatProb))
self.assertTrue(
p_cat[imtype].param_val.shape[0] == p_cat[imtype].nparam)
self.assertTrue(
p_cat[imtype].param_val.shape[1] == p_cat[imtype].ncat)
def pred_popex_mp(pred, path_res, nmp=1):
""" `pred_popex_mp` is the main implementation for computing predictions
from a PoPEx sampling.
Notes
-----
The computations are supposed to be independent, such that they can be
computed in parallel.
Parameters
----------
pred : Prediction
Defines the prediction functions and parameters
path_res : str
Path for loading the PoPEx results
nmp : int
Number of parallel processes
Returns
-------
None
"""
# Start PoPEx predictions
tst_pred = time.time()
# Load PoPEx results
with open(path_res + 'popex.pop', 'rb') as file:
popex = pickle.load(file) # type: PoPEx
# Compute prediction indices
w_pred = utl.compute_w_pred(popex=popex,
nw_min=pred.nw_min,
meth=pred.meth_w_pred)
ind_pred = utl.compute_subset_ind(pred.wfrac_pred, w_pred)
npred = len(ind_pred)
# Write some information
print("\n START 'PRED_POPEX_MP'\n")
print(' nmp = {:>6}'.format(nmp))
print(' npred = {:>6}'.format(npred))
print(" pred_meth = '{}'\n".format(pred.meth_w_pred['name']))
# Main prediction loop
stop = False
ipred = 0
ndone = 0
proc_mngr = deque([], maxlen=nmp)
print(' > predict data...', end='')
with multiprocessing.Pool(processes=nmp, maxtasksperchild=100) as pool:
while not stop:
if len(proc_mngr) < nmp and ipred < npred:
# Create new process
imod = ind_pred[ipred]
args = (deepcopy(pred), deepcopy(popex), imod)
proc_mngr.append(pool.apply_async(_pred_process, args))
ipred += 1
# Check if the process at the head of the queue has finished
if proc_mngr[0].ready():
# Remove process
proc = proc_mngr.popleft()
ipred_proc = proc.get()
ndone += 1
# Write some information
t_pred = time.time() - tst_pred
_write_pred_sum(path_res, ipred_proc, ndone, npred, t_pred)
# Update stopping condition
if len(proc_mngr) == 0 and ipred >= npred:
stop = True
else:
# Send process to the back of the queue
proc_mngr.rotate(-1)
print('done')
# Save Prediction
print(' > save prediction...', end='')
with open('{}pred.pred'.format(popex.path_res), 'wb') as file:
pickle.dump(pred, file)
print('done')
print("\n END 'PRED_POPEX_MP'\n")
def _write_run_sum(pb, popex, nmax, t_popex, t_mod):
""" `_write_run_sum` writes a document that summarized the overall
progress of the popex run at 'popex.path_res'.
The file structure is as followes:
| <popex.path_res>$
| └- run_progress.txt
Parameters
----------
pb : Problem
Problem definition
popex : PoPEx
PoPEx main structure
nmax : int
Number of maximal models
t_popex : float
Total time elapsed
t_mod : float
Average time per model
Returns
-------
None
"""
# Generic constants
bar_width = 40
# Path of the result location
path_res = popex.path_res
# Compute n_e diagnostics
with warnings.catch_warnings(record=True) as _:
ne_l = isampl.ne(utl.compute_w_lik(popex=popex, meth={'name':
'exact'}))
ne_w_hd = isampl.ne(utl.compute_w_lik(popex=popex, meth=pb.meth_w_hd))
ne_w_l = isampl.ne(
utl.compute_w_pred(popex=popex, nw_min=0, meth={'name': 'exact'}))
# Compute time information (t_popex, t_mod, t_est)
progress = popex.nmod / nmax
m_popex, s_popex = divmod(t_popex, 60)
h_popex, m_popex = divmod(m_popex, 60)
m_mod, s_mod = divmod(t_mod, 60)
h_mod, m_mod = divmod(m_mod, 60)
t_est = t_popex / progress
m_est, s_est = divmod(t_est, 60)
h_est, m_est = divmod(m_est, 60)
# Write into a file 'iteration_info.txt'
with open('{}/run_progress.txt'.format(path_res), 'w+') as file:
# Start with empty line
file.write('\n')
# Write title
file.write('RUN PROGRESS SUMMARY\n')
file.write('--------------------\n')
file.write('\n')
# Print status bar
nfull = int(bar_width * progress)
nempty = bar_width - nfull
file.write(' Status\n')
file.write(' n_mod = {:7d} / {:7d}\n'.format(
popex.nmod, nmax))
file.write(' [{:s}{:s}]'.format('-' * nfull, ' ' * nempty))
file.write(' {:3.0f}%\n'.format(progress * 100))
file.write('\n')
# Write n_e
file.write(' Diagnostics\n')
file.write(' ne(L) = {:9.1f}\n'.format(ne_l))
file.write(' ne(w_hd) = {:9.1f}\n'.format(ne_w_hd))
file.write(' ne(w_pred) = {:9.1f}\n'.format(ne_w_l))
file.write('\n')
# Write compute/predict stats
n_comp = np.sum(popex.cmp_log_p_lik)
n_pred = popex.nmod - n_comp
file.write(' Computed or Predicted\n')
file.write(' n_cmp = {:7d}\n'.format(n_comp))
file.write(' n_prd = {:7d}\n'.format(n_pred))
file.write('\n')
# Write times
file.write(' Time\n')
file.write(
' T_el = {:4.0f}[h] {:2.0f}[m] {:5.2f}[s]\n'.format(
h_popex, m_popex, s_popex))
file.write(
' T / mod = {:4.0f}[h] {:2.0f}[m] {:5.2f}[s]\n'.format(
h_mod, m_mod, s_mod))
file.write(
' T_tot = {:4.0f}[h] {:2.0f}[m] {:5.2f}[s]\n'.format(
h_est, m_est, s_est))
# End with empty line
file.write('\n')
def opt_fun(fsigma):
weights = utils.compute_w_pred(popex=popex, meth={'name':'soft', 'fsigma':fsigma})
return (ne(weights) - theta) ** 2 + l_reg*fsigma