def __init__(self,
likelihood_module,
prior_type='uniform',
prior_means=None,
prior_sigmas=None,
width_scale=1,
sigma_scale=1,
bound='multi',
sample='auto',
use_mpi=False,
use_pool=None):
"""
:param likelihood_module: likelihood_module like in likelihood.py (should be callable)
:param prior_type: 'uniform' of 'gaussian', for converting the unit hypercube to param cube
:param prior_means: if prior_type is 'gaussian', mean for each param
:param prior_sigmas: if prior_type is 'gaussian', std dev for each param
:param width_scale: scale the widths of the parameters space by this factor
:param sigma_scale: if prior_type is 'gaussian', scale the gaussian sigma by this factor
:param bound: specific to Dynesty, see https://dynesty.readthedocs.io
:param sample: specific to Dynesty, see https://dynesty.readthedocs.io
:param use_mpi: Use MPI computing if `True`
:param use_pool: specific to Dynesty, see https://dynesty.readthedocs.io
"""
self._check_install()
super(DynestySampler,
self).__init__(likelihood_module, prior_type, prior_means,
prior_sigmas, width_scale, sigma_scale)
# create the Dynesty sampler
if use_mpi:
from schwimmbad import MPIPool
import sys
pool = MPIPool(
use_dill=True
) # use_dill=True not supported for some versions of schwimmbad
if not pool.is_master():
pool.wait()
sys.exit(0)
self._sampler = self._dynesty.DynamicNestedSampler(
self.log_likelihood,
self.prior,
self.n_dims,
bound=bound,
sample=sample,
pool=pool,
use_pool=use_pool)
else:
self._sampler = self._dynesty.DynamicNestedSampler(
self.log_likelihood,
self.prior,
self.n_dims,
bound=bound,
sample=sample)
self._has_warned = False
def __init__(self, func, low, high, particleCount=25, threads=1):
self.threads = threads
pool = MPIPool()
if not pool.is_master():
pool.wait()
sys.exit(0)
ParticleSwarmOptimizer.__init__(self,
func,
low,
high,
particleCount=particleCount,
pool=pool)
def run_MCMC_mpi(self, Nwalker, Nsteps, outfile_MCMC=None, save_step_size=2):
Ndim = len(self.active_par_key)
starting_point = [self.par_fid[item] for item in self.active_par_key]
std = [self.par_std[item] for item in self.active_par_key]
p0_walkers = emcee.utils.sample_ball(starting_point, std, size=Nwalker)
with MPIPool() as pool:
if not pool.is_master():
pool.wait()
sys.exit(0)
sampler = emcee.EnsembleSampler(Nwalker, Ndim, self.cal_loglike, a=2.0, pool=pool)
step_size = save_step_size
steps_taken = 0
##### start long sampling
Tstart = time.time()
while steps_taken < Nsteps:
posInfo = sampler.run_mcmc(p0_walkers, step_size, progress=True)
p0_walkers = posInfo.coords
steps_taken+=step_size
print("steps_taken", steps_taken)
self.chainInfo = self.save_chain(sampler=sampler, outfile_MCMC=outfile_MCMC)
Time_MCMC = (time.time()-Tstart)/60.
print("Total MCMC time (mins):", Time_MCMC)
return self.chainInfo
def pool(request):
multimode = 'None'
# multimode = 'Serial'
# multimode = 'Multi'
# multimode = 'MPI'
# setup code
pool = None
if multimode == 'Serial':
from schwimmbad import SerialPool
pool = SerialPool()
elif multimode == 'Multi':
from schwimmbad import MultiPool
pool = MultiPool()
elif multimode == 'MPI':
from schwimmbad import MPIPool
pool = MPIPool()
if not pool.is_master():
pool.wait()
import sys
sys.exit(0)
# inject class variables
request.cls.pool = pool
yield
# tear down
if multimode == 'Multi' or multimode == 'MPI':
pool.close()
def mcmc_std(mapa,mpi=False):
def lnlike(theta, x0, y0, N):
n1,n2,kc = theta
x1,y1=sigma_resolution(Noise(N,n1,n2,kc))
y1=y1/x1**2
y1+=np.mean(y0)-np.mean(y1)
error=np.sum((y1-y0)**2)
plt.plot(x0,y0)
plt.plot(x1,y1)
plt.title(str(error))
plt.xscale('log')
plt.savefig('Temp_Figures/n1_%04.2f' %n1+'_n2_%04.2f' %n2+'_kc_%04d' %int(kc)+'.png')
plt.close()
return -0.5*error
def lnprior(theta, N):
n1, n2, kc = theta
if 0 < n1 < 5 and n1 < n2 < 9.0 and 1 < kc < N:
return 0.0
return -1e32
def lnprob(theta, x0, y0, N):
lp = lnprior(theta, N)
if not np.isfinite(lp):
return -1e32
return lp + lnlike(theta, x0,y0,N)
new_map=mapa/np.std(mapa)
x0,y0=sigma_resolution(new_map)
y0=y0/x0**2
N=len(new_map)
if mpi:
with MPIPool() as pool:
if not pool.is_master():
pool.wait()
sys.exit(0)
ndim, nwalkers = 3, 10
pos = [[1.33,4,N/4] + np.random.randn(ndim) for i in range(nwalkers)]
nsteps = 10
sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob, args=(x0,y0,N),pool=pool)
start = time.time()
sampler.run_mcmc(pos, nsteps)
end = time.time()
return sampler
else:
ndim, nwalkers = 3, 20
pos = [[1.33,4,N] + np.random.randn(ndim) for i in range(nwalkers)]
for p in pos:
p[2]*=np.random.uniform(0,1)
print(pos)
nsteps = 100
sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob, args=(x0,y0,N))
start = time.time()
sampler.run_mcmc(pos, nsteps)
end = time.time()
return sampler
def mcmc_emcee(self, n_walkers, n_run, n_burn, mean_start, sigma_start, mpi=False, progress=False, threadCount=1):
numParam, _ = self.chain.param.num_param()
p0 = sampling_util.sample_ball(mean_start, sigma_start, n_walkers)
time_start = time.time()
if mpi is True:
pool = MPIPool()
if not pool.is_master():
pool.wait()
sys.exit(0)
is_master_pool = pool.is_master()
sampler = emcee.EnsembleSampler(n_walkers, numParam, self.chain.logL, pool=pool)
else:
is_master_pool = True
if threadCount > 1 :
pool = Pool(processes=threadCount)
else :
pool = None
sampler = emcee.EnsembleSampler(n_walkers, numParam, self.chain.likelihood, pool=pool)
sampler.run_mcmc(p0, n_burn + n_run, progress=progress)
flat_samples = sampler.get_chain(discard=n_burn, thin=1, flat=True)
dist = sampler.get_log_prob(flat=True, discard=n_burn, thin=1)
if is_master_pool:
print('Computing the MCMC...')
print('Number of walkers = ', n_walkers)
print('Burn-in iterations: ', n_burn)
print('Sampling iterations:', n_run)
time_end = time.time()
print(time_end - time_start, 'time taken for MCMC sampling')
return flat_samples, dist
def sample(self, verbosity=0, use_mpi=False):
import emcee
if use_mpi:
from schwimmbad import MPIPool
pool = MPIPool()
print("Using MPI")
pool_use = pool
else:
pool = DumPool()
print("Not using MPI")
pool_use = None
if not pool.is_master():
pool.wait()
sys.exit(0)
fname_chain = self.prefix_out + "chain"
found_file = os.path.isfile(fname_chain + '.txt')
if (not found_file) or self.rerun:
pos_ini = (np.array(self.p0)[None, :] +
0.001 * np.random.randn(self.nwalkers, self.ndim))
nsteps_use = self.nsteps
else:
print("Restarting from previous run")
old_chain = np.loadtxt(fname_chain + '.txt')
if np.ndim(old_chain) == 1:
old_chain = np.atleast_2d(old_chain).T
pos_ini = old_chain[-self.nwalkers:, :]
nsteps_use = max(self.nsteps - len(old_chain) // self.nwalkers, 0)
print(self.nsteps - len(old_chain) // self.nwalkers)
chain_file = SampleFileUtil(self.prefix_out + "chain",
rerun=self.rerun)
sampler = emcee.EnsembleSampler(self.nwalkers,
self.ndim,
self.lnprob,
pool=pool_use)
counter = 1
for pos, prob, _ in sampler.sample(pos_ini, iterations=nsteps_use):
if pool.is_master():
chain_file.persistSamplingValues(pos, prob)
if counter % 10 == 0:
print(f"Finished sample {counter}")
counter += 1
pool.close()
return sampler
def postprocess_run(jobdir,
savename,
exp_type,
fields,
save_beta=False,
comm=None,
return_dframe=True):
# Distribute postprocessing across ranks, if desired
if comm is not None:
# Collect all .h5 files
if comm.rank == 0:
data_files = grab_files(jobdir, '*.dat', exp_type)
data_files = [(d) for d in data_files]
print(len(data_files))
else:
data_files = None
rank = comm.rank
size = comm.size
# print('Rank %d' % rank)
master = StreamWorker(savename)
worker = PostprocessWorker(jobdir, fields, rank, size)
pool = MPIPool(comm)
pool.map(worker,
data_files,
callback=master.stream,
track_results=False)
if not pool.is_master():
pool.wait()
sys.exit(0)
pool.close()
if rank == 0:
master.close()
else:
worker = PostprocessWorker(jobdir, fields, savename)
for i, data_file in enumerate(data_files):
t0 = time.time()
result = worker(data_file)
worker.extend(result)
print(time.time() - t0)
dframe = worker.save(save_beta)
if return_dframe:
return dframe
def test_MPI(self):
if MPIPool.enabled():
# mpi should work
mpi_pool = choose_pool('mpi')
self.assertTrue(isinstance(mpi_pool, MPIPool))
elif find_spec('mpi4py') is None:
# mpi4py not installed
self.assertRaises(ImportError, choose_pool, 'mpi')
else:
# mpi4py installed but only one process available
self.assertRaises(ValueError, choose_pool, 'mpi')
def MPIpool_evaluate(representations, ndim, fid, iids, num_reps, budget=None):
from schwimmbad import MPIPool
budget = Config.ES_budget_factor * ndim if budget is None else budget
run_es = partial(runCustomizedES, ndim=ndim, fid=fid, budget=budget)
func = partial(helper, func=run_es)
tasks = product(representations, iids, range(num_reps))
with MPIPool() as pool:
results = pool.map(func, tasks)
return results
def main_fit():
pool = MPIPool()
if not pool.is_master():
pool.wait()
sys.exit(0)
# Create an initial point
psize = pos_lim_max - pos_lim_min
p0 = [pos_lim_min + psize * np.random.rand(ndim) for i in range(nwalkers)]
# Set a sample
sampler = emcee.EnsembleSampler(nwalkers, ndim, lnprob, pool=pool)
# Run MCMC
sampler.run_mcmc(p0, burnin)
# Flat sample and discard 10% sample
flat_samples = sampler.get_chain(discard=int(0.1 * burnin), flat=True)
pool.close()
return flat_samples
def megafit_emcee(self):
nsamples = 300
atm = []
for j in range(len(self.names)):
prod_name = os.path.join(parameters.PROD_DIRECTORY, parameters.PROD_NAME)
atmgrid = AtmosphereGrid(
filename=(prod_name + '/' + self.names[j].split('/')[-1]).replace('sim', 'reduc').replace(
'spectrum.txt', 'atmsim.fits'))
atm.append(atmgrid)
def matrice_data():
y = self.data - self.order2
nb_spectre = len(self.names)
nb_bin = len(self.data)
D = np.zeros(nb_bin * nb_spectre)
for j in range(nb_spectre):
D[j * nb_bin: (j + 1) * nb_bin] = y[:, j]
return D
def Atm(atm, ozone, eau, aerosols):
nb_spectre = len(self.names)
nb_bin = len(self.data)
M = np.zeros((nb_spectre, nb_bin, nb_bin))
M_p = np.zeros((nb_spectre * nb_bin, nb_bin))
for j in range(nb_spectre):
Atmo = np.zeros(len(self.new_lambda))
Lambdas = np.arange(self.Bin[0],self.Bin[-1],0.2)
Atm = atm[j].simulate(ozone, eau, aerosols)(Lambdas)
for i in range(len(self.new_lambda)):
Atmo[i] = np.mean(Atm[i*int(self.binwidths/0.2):(i+1)*int(self.binwidths/0.2)])
a = np.diagflat(Atmo)
M[j, :, :] = a
M_p[nb_bin * j:nb_bin * (j+1),:] = a
return M, M_p
def log_likelihood(params_fit, atm):
nb_spectre = len(self.names)
nb_bin = len(self.data)
ozone, eau, aerosols = params_fit[-3], params_fit[-2], params_fit[-1]
D = matrice_data()
M, M_p = Atm(atm, ozone, eau, aerosols)
prod = np.zeros((nb_bin, nb_spectre * nb_bin))
for spec in range(nb_spectre):
prod[:,spec * nb_bin : (spec+1) * nb_bin] = M[spec] @ self.INVCOV[spec]
COV = inv(prod @ M_p)
A = COV @ prod @ D
chi2 = 0
for spec in range(nb_spectre):
mat = D[spec * nb_bin : (spec+1) * nb_bin] - M[spec] @ A
chi2 += mat @ self.INVCOV[spec] @ mat
n = np.random.randint(0, 100)
if n > 97:
print(chi2 / (nb_spectre * nb_bin))
print(ozone, eau, aerosols)
return -0.5 * chi2
def log_prior(params_fit):
ozone, eau, aerosols = params_fit[-3], params_fit[-2], params_fit[-1]
if 100 < ozone < 700 and 0 < eau < 10 and 0 < aerosols < 0.1:
return 0
else:
return -np.inf
def log_probability(params_fit, atm):
lp = log_prior(params_fit)
if not np.isfinite(lp):
return -np.inf
return lp + log_likelihood(params_fit, atm)
if self.sim:
filename = "sps/" + self.disperseur + "_"+ "sim_"+ parameters.PROD_NUM + "_emcee.h5"
else:
filename = "sps/" + self.disperseur + "_" + "reduc_" + parameters.PROD_NUM + "_emcee.h5"
p_ozone = 300
p_eau = 5
p_aerosols = 0.03
p0 = np.array([p_ozone, p_eau, p_aerosols])
walker = 10
init_ozone = p0[0] + p0[0] / 5 * np.random.randn(walker)
init_eau = p0[1] + p0[1] / 5 * np.random.randn(walker)
init_aerosols = p0[2] + p0[2] / 5 * np.random.randn(walker)
p0 = np.array([[init_ozone[i], init_eau[i], init_aerosols[i]] for i in range(walker)])
nwalkers, ndim = p0.shape
backend = emcee.backends.HDFBackend(filename)
try:
pool = MPIPool()
if not pool.is_master():
pool.wait()
sys.exit(0)
sampler = emcee.EnsembleSampler(nwalkers, ndim, log_probability,
args=(atm,), pool=pool, backend=backend)
if backend.iteration > 0:
p0 = backend.get_last_sample()
if nsamples - backend.iteration > 0:
sampler.run_mcmc(p0, nsteps=max(0, nsamples - backend.iteration), progress=True)
pool.close()
except ValueError:
sampler = emcee.EnsembleSampler(nwalkers, ndim, log_probability,
args=(atm,),
threads=multiprocessing.cpu_count(), backend=backend)
if backend.iteration > 0:
p0 = sampler.get_last_sample()
for _ in sampler.sample(p0, iterations=max(0, nsamples - backend.iteration), progress=True, store=True):
continue
flat_samples = sampler.get_chain(discard=100, thin=1, flat=True)
ozone, d_ozone = np.mean(flat_samples[:, -3]), np.std(flat_samples[:, -3])
eau, d_eau = np.mean(flat_samples[:, -2]), np.std(flat_samples[:, -2])
aerosols, d_aerosols = np.mean(flat_samples[:, -1]), np.std(flat_samples[:, -1])
print(ozone, d_ozone)
print(eau, d_eau)
print(aerosols, d_aerosols)
self.params_atmo = np.array([ozone, eau, aerosols])
self.err_params_atmo = np.array([d_ozone, d_eau, d_aerosols])
nb_spectre = len(self.names)
nb_bin = len(self.data)
M, M_p = Atm(atm, ozone, eau, aerosols)
prod = np.zeros((nb_bin, nb_spectre * nb_bin))
chi2 = 0
for spec in range(nb_spectre):
prod[:, spec * nb_bin: (spec + 1) * nb_bin] = M[spec] @ self.INVCOV[spec]
COV = inv(prod @ M_p)
D = matrice_data()
Tinst = COV @ prod @ D
Tinst_err = np.array([np.sqrt(COV[i,i]) for i in range(len(Tinst))])
if self.disperseur == 'HoloAmAg' and self.sim == False:
a, b = np.argmin(abs(self.new_lambda - 537.5)), np.argmin(abs(self.new_lambda - 542.5))
Tinst_err[a], Tinst_err[b] = Tinst_err[a-1], Tinst_err[b+1]
for spec in range(nb_spectre):
mat = D[spec * nb_bin: (spec + 1) * nb_bin] - M[spec] @ Tinst
chi2 += mat @ self.INVCOV[spec] @ mat
print(chi2 / (nb_spectre * nb_bin))
err = np.zeros_like(D)
for j in range(len(self.names)):
for i in range(len(self.data)):
if self.disperseur == 'HoloAmAg' and self.sim == False:
if self.new_lambda[i] == 537.5 or self.new_lambda[i] == 542.5:
err[j * len(self.data) + i] = 1
else:
err[j * len(self.data) + i] = np.sqrt(self.cov[j][i, i])
else:
err[j * len(self.data) + i] = np.sqrt(self.cov[j][i, i])
model = M_p @ Tinst
Err = (D - model) / err
if parameters.plot_residuals:
self.Plot_residuals(model, D, Err, COV)
return Tinst, Tinst_err
def main(path2config, time_likelihood):
# Read params from yaml
config = yaml.load(open(path2config))
mode = config['modus_operandi']
default_params = config['default_params']
power_params = config['power_params']
template_params = config['template_params']
cl_params = config['cl_params']
ch_config_params = config['ch_config_params']
fit_params = config['fit_params']
# parameters to fit
nparams = len(fit_params.keys())
param_mapping = {}
params = np.zeros((nparams, 4))
for key in fit_params.keys():
param_mapping[key] = fit_params[key][0]
params[fit_params[key][0], :] = fit_params[key][1:]
# Cosmology
if set(COSMO_PARAM_KEYS) == set(default_params.keys()):
print("We are NOT varying the cosmology")
#cosmo_dict = {}
#for key in COSMO_PARAM_KEYS:
#cosmo_dict[key] = default_params[key]
cosmo = ccl.Cosmology(**default_params)
else:
print("We ARE varying the cosmology")
cosmo = None
if power_params['lmax'] == 'kmax':
lmax = kmax2lmax(power_params['kmax'], power_params['z'], cosmo)
power_params['lmax'] = lmax
if power_params['lmin'] == 'kmax':
lmin = 0.
power_params['lmin'] = lmin
# read data parameters
Data = PowerData(mode, power_params)
Data.setup()
# read theory parameters
Theory = PowerTheory(mode, Data.x, Data.z, template_params, cl_params,
power_params, default_params, param_mapping)
Theory.setup()
# initialize the Cl templates
if mode == 'Cl' and cosmo != None:
Theory.init_cl_ij(cosmo)
# Make path to output
if not os.path.isdir(os.path.expanduser(ch_config_params['path2output'])):
try:
os.makedirs(os.path.expanduser(ch_config_params['path2output']))
except:
pass
# MPI option
if ch_config_params['use_mpi']:
from schwimmbad import MPIPool
pool = MPIPool()
print("Using MPI")
pool_use = pool
else:
pool = DumPool()
print("Not using MPI")
pool_use = None
if not pool.is_master():
pool.wait()
sys.exit(0)
# just time the likelihood calculation
if time_likelihood:
time_lnprob(params, Data, Theory)
return
# emcee parameters
nwalkers = nparams * ch_config_params['walkersRatio']
nsteps = ch_config_params['burninIterations'] + ch_config_params[
'sampleIterations']
# where to record
prefix_chain = os.path.join(
os.path.expanduser(ch_config_params['path2output']),
ch_config_params['chainsPrefix'])
# fix initial conditions
found_file = os.path.isfile(prefix_chain + '.txt')
if (not found_file) or (not ch_config_params['rerun']):
p_initial = params[:, 0] + np.random.normal(
size=(nwalkers, nparams)) * params[:, 3][None, :]
nsteps_use = nsteps
else:
print("Restarting from a previous run")
old_chain = np.loadtxt(prefix_chain + '.txt')
p_initial = old_chain[-nwalkers:, :]
nsteps_use = max(nsteps - len(old_chain) // nwalkers, 0)
# initializing sampler
chain_file = SampleFileUtil(prefix_chain,
carry_on=ch_config_params['rerun'])
sampler = emcee.EnsembleSampler(nwalkers,
nparams,
lnprob,
args=(params, Data, Theory),
pool=pool_use)
start = time.time()
print("Running %d samples" % nsteps_use)
# record every iteration
counter = 1
for pos, prob, _ in sampler.sample(p_initial, iterations=nsteps_use):
if pool.is_master():
print('Iteration done. Persisting.')
chain_file.persistSamplingValues(pos, prob)
if counter % 10:
print(f"Finished sample {counter}")
counter += 1
pool.close()
end = time.time()
print("Took ", (end - start), " seconds")
def ensemble(likelihood, prior, MPI = True, **kwargs):
""" Initialise `emcee <http://dfm.io/emcee/current/>`_ and sample.
:param likelihood: An instance of :class:`~.Likelihood.Likelihood`.
:param prior: An instance of :class:`~.Prior.Prior`.
:param bool MPI: Parallelise with MPI? If calling script not lauched with
an MPI directive, sampling will not commence because there
is only one process. Default is ``True`` since only in
testing is it justifiable to use a single process.
:param kwargs: Passed to initialisers of appropriate classes.
* boolean to resume, under keyword :obj:`resume`;
* number of steps, under keyword :obj:`nsteps`;
* number of walkers, under keyword :obj:`nwalkers`;
* moments of initial walker multivariate Gaussian distribution, under
keyword :obj:`walker_dist_moments` (can be ``None``);
* root directory for output, under keyword :obj:`root_dir`;
The above objects are used to instantiate :class:`~.Posterior.Posterior`.
:return: An instance of :class:`emcee.backends.HDFBackend`.
"""
# use the globally scoped variable
global posterior
from xpsi.Posterior import Posterior
# Callable instance of the posterior
posterior = Posterior(likelihood,
prior,
**kwargs)
if MPI or global_imports._size > 1:
try:
from schwimmbad import MPIPool
except ImportError:
raise ImportError('Check your schwimmbad package installation.')
# Initialize the MPI-based pool used for parallelisation.
with MPIPool() as pool:
if not pool.is_master():
# Wait for instructions from the master process.
pool.wait()
_sys.exit(0)
from xpsi.EnsembleSampler import EnsembleSampler
# Initialise emcee sampler
sampler = EnsembleSampler(ndims = len(likelihood),
posterior = func,
pool = pool,
**kwargs)
# Commence emcee sampling process
sampler()
else:
from xpsi.EnsembleSampler import EnsembleSampler
# Initialise emcee sampler
sampler = EnsembleSampler(ndims = len(likelihood),
posterior = posterior,
pool = None,
**kwargs)
# Commence emcee sampling process
sampler()
return sampler.backend
n = dt.now()
print(f'starting lag {tau}, time now: {n.strftime("%Y %b %d %H:%M:%S")}')
cf = dict(zip(sorted(dims),fin.apply(lambda col: col.autocorr(tau))))
print(f'computed lag {tau}, seconds taken: {(dt.now()-n).total_seconds():.2f}')
return cf
## make up a function for batch computing autocorrelations
def batch_autocorrelation(taus):
n = dt.now()
print(f'starting batch of lags {taus}, time now: {n.strftime("%Y %b %d %H:%M:%S")}')
cf = [dict(zip(sorted(dims),fin.apply(lambda col: col.autocorr(t)))) for t in taus]
print(f'computed batch of lags {taus}, seconds taken: {(dt.now()-n).total_seconds():.2f}', flush=True)
return cf
## shut down all processes except the master one that will map tasks to others
pool = MPIPool()
if not pool.is_master():
print('one worker on standby')
pool.wait()
sys.exit(0)
print('MPI master proceeding to map lags to workers...')
print(f'There are {len(lag_batches)} total batches for parallelization')
## do the actual parallel computation of the autocorrelation function
print(f'{dt.now().strftime("%Y %b %d %H:%M:%S")}, computing from {options["file"]}')
if 'batch' in list(options.keys()) : acf = flatten(pool.map(batch_autocorrelation, lag_batches))
else : acf = pool.map(one_autocorrelation, lags)
print(f'done with parallel computation, {dt.now().strftime("%Y %b %d %H:%M:%S")}')
## convert to dataframe
initcosmo("halofit")
initbins(25, 30.0, 15000.0, 4000.0, 21.0, 10, 10)
initsurvey("WFIRST")
initgalaxies(file_source_z, file_lens_z, "gaussian", "gaussian", "source")
initclusters()
initia("none", "none")
initpriors("none", "none", "PhotoBAO", "none")
# test also with
#initpriors("none","none","none","Planck")
#initpriors("none","none","none","random")
initprobes("all_2pt")
initdatainv(cov_file, data_file)
#sample_params=sample_LCDM_only()
#sample_params= sample_cosmology_only()
#sample_params = sample_cosmology_shear_nuisance(get_N_tomo_shear())
sample_params = sample_cosmology_2pt_shear_nuisance(get_N_tomo_shear(),
get_N_tomo_clustering())
#sample_params = sample_cosmology_2pt_nuisance(get_N_tomo_shear(),get_N_tomo_clustering())
#sample_params = sample_cosmology_2pt_nuisance_IA_marg(get_N_tomo_shear(),get_N_tomo_clustering())
#sample_params = sample_cosmology_2pt_cluster_nuisance(get_N_tomo_shear(),get_N_tomo_clustering())
sample_main(sample_params,
1000,
560,
1,
chain_file,
blind=False,
pool=MPIPool())
#nbunch = 25
nstep = 50000
# (total_number_of_cores - 1)*2, this should be equal to ndim x integer
nwalkers = (size - 1) * 2
#nwalkers = 30
if nwalkers < ndim:
nwalkers = ndim * 2
coords = np.random.randn(nwalkers, ndim)
pos = [pinit + 1e-4 * np.random.randn(ndim) for j in range(nwalkers)]
# run MCMC
with MPIPool() as pool:
if not pool.is_master():
pool.wait()
sys.exit(0)
filename_backend = os.path.join(dirname, "backend.h5")
backend = emcee.backends.HDFBackend(filename_backend)
backend.reset(nwalkers, ndim)
sampler = emcee.EnsembleSampler(nwalkers,
ndim,
lnprob_global,
pool=pool,
backend=backend)
index = 0
autocorr = np.empty(nstep)
def runMPI(runFunction, arguments):
from schwimmbad import MPIPool
with MPIPool() as pool:
results = pool.map(runFunction, arguments)
return results
else:
if anlz.pmt1 and anlz.pmt2:
saveName = ("/N/slate/frangonz/outputSing"+str(runListAll[0])+".txt")
elif anlz.pmt1:
saveName = ("/N/slate/frangonz/outputPMT1"+str(runListAll[0])+".txt")
else:
saveName = ("/N/slate/frangonz/outputPMT2"+str(runListAll[0])+".txt")
#saveName = ("/N/u/frangonz/BigRed3/New_MCA_Analysis/outputs/outputSing"+str(iniRun)+".txt")
if len(ctsTrunc) == 0:
ctsTrunc = ctsSing
saveName = "outputDebug.txt"
nwalkers,ndim,walker_init = optimized_likelihood_fit(runListAll,ctsTrunc,nMon,bkgsH,bkgsT,anlz)
try: # I've moved this down here so that we pickle the updated global variables
pool = MPIPool() # This is the master MPI check
if not pool.is_master(): # All the pools will get called after the master loads
pool.wait() # Wait if you're not the master
sys.exit(0)
# Now let's do some quick writeouts for debug
print("Environment:")
print(" Config:",os.environ['ANALYSIS_CONFIG'])
print(" Data:",os.environ['DATA_DIR'])
print("RUNS:",runListAll)
if anlz.sing:
if anlz.pmt1 and anlz.pmt2:
print("PMT 1+2")
elif anlz.pmt1:
print("PMT 1")
else:
def MCMC(self, nwalkers=50, nburn=200, nMCMC=1000, use_MPI=False, chain_file='chain.dat', fig_name='./MCMC_corner.png', plot_corner=False, **kwargs): # The function to carry out MCMC. # parameters: # nwalkers: int, optional # the number of walkers in MCMC, which must be even. # default: 50 # nburn: int, optional # the number of burn-in steps in MCMC. # default: 200 # nMCMC: int, optional # the number of final MCMC steps in MCMC. # default: 1000 # use_MPI: Boolean, optional # whether to use MPI. # default: False # returns: # p_best: array_like # best fitting parameter set. # Initialize the walkers with a set of initial points, p0. E0 = np.random.normal(0.5, 0.3, size=nwalkers) T0 = np.random.normal(0.5, 0.3, size=nwalkers) a0 = np.random.normal(0, 0.7, size=nwalkers) covEE = truncnorm.rvs(0, 1, loc=0.3, scale=0.1, size=nwalkers) covTT = truncnorm.rvs(0 ,1, loc=0.3, scale=0.1, size=nwalkers) covaa = truncnorm.rvs(0, 1, loc=0.1, scale=0.1, size=nwalkers) covEa = truncnorm.rvs(0, 1, loc=0.1, scale=0.1, size=nwalkers) p0 = [[E0[i], T0[i], a0[i], covEE[i], covTT[i], covaa[i], covEa[i]] for i in range(nwalkers)] print 'start MCMC.' if not use_MPI: sampler = EnsembleSampler(nwalkers, self.ndim, lnprob, **kwargs) # sampler = EnsembleSampler(nwalkers, self.ndim, lnprob, \ # args=(self.E_grid, self.T_grid, self.a_grid, self.ba_lgSMA_bins, self.bin_obs, self.num_obs), **kwargs) # When using MPI, we differentiate between different processes. else: pool = MPIPool() if not pool.is_master(): pool.wait() sys.exit(0) # sampler = EnsembleSampler(nwalkers, self.ndim, lnprob, \ # args=(self.E_grid, self.T_grid, self.a_grid, self.ba_lgSMA_bins, self.bin_obs, self.num_obs), pool=pool, **kwargs) sampler = EnsembleSampler(nwalkers, self.ndim, lnprob, pool=pool, **kwargs) # burn-in phase pos, prob, state = sampler.run_mcmc(p0, nburn, chain_file=chain_file) sampler.reset() # MCMC phase sampler.run_mcmc(pos, nMCMC, chain_file=chain_file) if use_MPI: pool.close() # If we want to make classic corner plots... if plot_corner: samples = sampler.chain[:, nMCMC / 2:, :].reshape((-1, self.ndim)) fig = corner.corner(samples, labels=['E', 'T', 'a', 'covEE', 'covTT', 'covaa', 'covEa']) fig.savefig(fig_name) # Get the best fitting parameters. We take the median parameter value for the ensemble # of steps with log-probabilities within the largest 30% among the whole ensemble as the # best parameters. samples = sampler.flatchain lnp = sampler.flatlnprobability crit_lnp = np.percentile(lnp, 70) good = np.where(lnp > crit_lnp) p_best = [np.median(samples[good, i]) for i in range(self.ndim)] return np.array(p_best)
def sample(self, carry_on=False, verbosity=0, use_mpi=False):
"""
Sample the posterior distribution
Args:
carry_on (bool): if True, the sampler will restart from
its last iteration.
verbosity (int): if >0, progress will be reported.
use_mpi (bool): set to True to parallelize with MPI
Returns:
:obj:`emcee.EnsembleSampler`: sampler with chain.
"""
import emcee
if use_mpi:
from schwimmbad import MPIPool
pool = MPIPool()
print("Using MPI")
pool_use = pool
else:
pool = DumPool()
print("Not using MPI")
pool_use = None
if not pool.is_master():
pool.wait()
sys.exit(0)
fname_chain = self.prefix_out+"chain"
found_file = os.path.isfile(fname_chain+'.txt')
counter = 1
if (not found_file) or (not carry_on):
pos_ini = (np.array(self.p0)[None, :] +
0.001 * np.random.randn(self.nwalkers, self.ndim))
nsteps_use = self.nsteps
else:
print("Restarting from previous run")
with warnings.catch_warnings():
warnings.simplefilter("ignore")
old_chain = np.loadtxt(fname_chain+'.txt')
if old_chain.size != 0:
pos_ini = old_chain[-self.nwalkers:, :]
nsteps_use = max(self.nsteps-len(old_chain) // self.nwalkers, 0)
counter = len(old_chain) // self.nwalkers
# print(self.nsteps - len(old_chain) // self.nwalkers)
else:
pos_ini = (np.array(self.p0)[None, :] +
0.001 * np.random.randn(self.nwalkers, self.ndim))
nsteps_use = self.nsteps
chain_file = SampleFileUtil(self.prefix_out+"chain", carry_on=carry_on)
sampler = emcee.EnsembleSampler(self.nwalkers,
self.ndim,
self.lnprob,
pool=pool_use)
for pos, prob, _ in sampler.sample(pos_ini, iterations=nsteps_use):
if pool.is_master():
if verbosity > 0:
print('Iteration done. Persisting.')
chain_file.persistSamplingValues(pos, prob)
if (counter % 10) == 0:
print(f"Finished sample {counter}")
counter += 1
pool.close()
return sampler
flux_df = pd.read_csv('../'+args.fluxes)
filter_df = pd.read_csv('../filters.csv')
#Read in and zip up dataframes
sCM20_df = pd.read_hdf('models.h5','sCM20')
lCM20_df = pd.read_hdf('models.h5','lCM20')
aSilM5_df = pd.read_hdf('models.h5','aSilM5')
wavelength_df = pd.read_hdf('models.h5','wavelength')
pandas_dfs = [sCM20_df,lCM20_df,
aSilM5_df,wavelength_df]
if args.mpi:
mpi_pool = MPIPool()
if not mpi_pool.is_master():
mpi_pool.wait()
sys.exit(0)
mpi_pool.map( main,
range(len(flux_df)) )
mpi_pool.close()
else:
for gal_row in range(len(flux_df)):
main(gal_row)
totsum = np.sum(chunksum, axis=0)
weightsum = np.sum(chunkweightsum, axis=0)
finalstack = totsum / weightsum
finalstack.dump('%s.npy' % outname)
print(time.time() - starttime)
if __name__ == "__main__":
import sys
from schwimmbad import MPIPool
from astropy.io import fits
import glob
# set up schwimmbad MPI pool
pool = MPIPool()
# if current instantiation is not the master, wait for tasks
if not pool.is_master():
pool.wait()
sys.exit(0)
sample_name = 'xdqso_specz'
imsize = 160
reso = 1.5
stack_maps = False
stack_noise = False
nbootstacks = 10
nrandoms = 0
temperature = False
def MCMC(self,pool):
""" The MCMC method. Construct starting points of the chains around
the best solution found by the 'DE' method.
The objective function is :func:`chichi_MCMC`. Telescope flux (fs and g), can be optimized thanks to MCMC if
flux_estimation_MCMC is 'MCMC', either they are derived through np.polyfit.
Based on the emcee python package :
" emcee: The MCMC Hammer" (Foreman-Mackey et al. 2013).
Have a look here : http://dan.iel.fm/emcee/current/
:return: a tuple containing (MCMC_chains, MCMC_probabilities)
:rtype: tuple
**WARNING** :
nwalkers is set to 4 times the len of pazynski_parameters
nlinks is set to 4000
5*nwalkers*nlinks MCMC steps in total
"""
# start = python_time.time()
if len(self.model.parameters_guess) == 0:
differential_evolution_estimation = self.differential_evolution(pool)[0]
self.DE_population_size = 10
self.guess = differential_evolution_estimation
else:
self.guess = list(self.model.parameters_guess)
self.guess += self.find_fluxes(self.guess, self.model)
# Best solution
if self.fluxes_MCMC_method != 'MCMC':
limit_parameters = len(self.model.parameters_boundaries)
best_solution = self.guess[:limit_parameters]
else:
limit_parameters = len(self.guess)
best_solution = self.guess
nwalkers = 2*len(best_solution)
nlinks = 5*1000
# Initialize the population of MCMC
population = []
count_walkers = 0
while count_walkers < nwalkers:
# Construct an individual of the population around the best solution.
individual = []
for parameter_key in list(self.model.model_dictionnary.keys())[:limit_parameters]:
parameter_trial = microlguess.MCMC_parameters_initialization(parameter_key,
self.model.model_dictionnary,
best_solution)
if parameter_trial:
for parameter in parameter_trial:
individual.append(parameter)
#if self.fluxes_MCMC_method == 'MCMC':
# fluxes = self.find_fluxes(individual, self.model)
# individual += (fluxes*np.random.uniform(0.99,1.01,len(fluxes))+np.random.uni).tolist()
chichi = self.chichi_MCMC(individual)
if chichi != -np.inf:
# np.array(individual)
# print count_walkers
population.append(np.array(individual))
count_walkers += 1
print('pre MCMC done')
number_of_parameters = len(individual)
try:
# create a new MPI pool
from schwimmbad import MPIPool
pool = MPIPool()
if not pool.is_master():
pool.wait()
sys.exit(0)
except:
pass
sampler = emcee.EnsembleSampler(nwalkers, number_of_parameters, self.chichi_MCMC,
a=2.0, pool= pool)
# First estimation using population as a starting points.
#final_positions, final_probabilities, state = sampler.run_mcmc(population, nlinks, progress=True)
#print('MCMC preburn done')
#sampler.reset()
sampler.run_mcmc(population, nlinks, progress=True)
MCMC_chains = np.c_[sampler.get_chain().reshape(nlinks*nwalkers,number_of_parameters),sampler.get_log_prob().reshape(nlinks*nwalkers)]
# Final estimation using the previous output.
#for positions, probabilities, states in sampler.sample(final_positions, iterations= nlinks,
# storechain=True):
# chains = np.c_[positions, probabilities]
# if MCMC_chains is not None:
# MCMC_chains = np.r_[MCMC_chains, chains]
# else:
# MCMC_chains = chains
print(sys._getframe().f_code.co_name, ' : MCMC fit SUCCESS')
return MCMC_chains
file_source_z = os.path.join(dirname, "zdistris/zdistribution_DESY1_source")
file_lens_z = os.path.join(dirname, "zdistris/zdistribution_DESY1_lens")
data_file = os.path.join(dirname, "datav/DES_all_2pt_fid_opti")
cov_file = os.path.join(dirname, "cov/DES_cov_3x2pt_inv")
chain_file = os.path.join(dirname, "like/like_DES_ocelote_3x2pt_LCDM")
initcosmo("halofit")
initbins(25,30.0,15000.0,4000.0,21.0,4,5)
initpriors("photo_opti","shear_opti","none","none")
initsurvey("WFIRST")
initgalaxies(file_source_z,file_lens_z,"gaussian","gaussian","DES_Y1")
initclusters()
initia("none","none")
# test also with
#initpriors("none","none","none","Planck")
#initpriors("none","none","none","random")
initprobes("all_2pt")
initdatainv(cov_file ,data_file)
sample_params=sample_LCDM_only()
#sample_params=sample_LCDM_2pt_nuisance()
#sample_params= sample_cosmology_only()
#sample_params = sample_cosmology_shear_nuisance(get_N_tomo_shear())
#sample_params = sample_cosmology_2pt_nuisance(get_N_tomo_shear(),get_N_tomo_clustering())
#sample_params = sample_cosmology_2pt_nuisance_IA_marg(get_N_tomo_shear(),get_N_tomo_clustering())
#sample_params = sample_cosmology_2pt_cluster_nuisance(get_N_tomo_shear(),get_N_tomo_clustering())
sample_main(sample_params,2000,560,1,chain_file, blind=False, pool=MPIPool())
def main(path2config, time_likelihood):
# load the yaml parameters
config = yaml.load(open(path2config))
sim_params = config['sim_params']
HOD_params = config['HOD_params']
clustering_params = config['clustering_params']
data_params = config['data_params']
ch_config_params = config['ch_config_params']
fit_params = config['fit_params']
# create a new abacushod object and load the subsamples
newBall = AbacusHOD(sim_params, HOD_params, clustering_params)
# read data parameters
newData = PowerData(data_params, HOD_params)
# parameters to fit
nparams = len(fit_params.keys())
param_mapping = {}
param_tracer = {}
params = np.zeros((nparams, 4))
for key in fit_params.keys():
mapping_idx = fit_params[key][0]
tracer_type = fit_params[key][-1]
param_mapping[key] = mapping_idx
param_tracer[key] = tracer_type
params[mapping_idx, :] = fit_params[key][1:-1]
# Make path to output
if not os.path.isdir(os.path.expanduser(ch_config_params['path2output'])):
try:
os.makedirs(os.path.expanduser(ch_config_params['path2output']))
except:
pass
# MPI option
if ch_config_params['use_mpi']:
from schwimmbad import MPIPool
pool = MPIPool()
print("Using MPI")
pool_use = pool
else:
pool = DumPool()
print("Not using MPI")
pool_use = None
if not pool.is_master():
pool.wait()
sys.exit(0)
# just time the likelihood calculation
if time_likelihood:
time_lnprob(params, param_mapping, param_tracer, newData, newBall)
return
# emcee parameters
nwalkers = nparams * ch_config_params['walkersRatio']
nsteps = ch_config_params['burninIterations'] + ch_config_params[
'sampleIterations']
# where to record
prefix_chain = os.path.join(
os.path.expanduser(ch_config_params['path2output']),
ch_config_params['chainsPrefix'])
# fix initial conditions
found_file = os.path.isfile(prefix_chain + '.txt')
if (not found_file) or (not ch_config_params['rerun']):
p_initial = params[:, 0] + np.random.normal(
size=(nwalkers, nparams)) * params[:, 3][None, :]
nsteps_use = nsteps
else:
print("Restarting from a previous run")
old_chain = np.loadtxt(prefix_chain + '.txt')
p_initial = old_chain[-nwalkers:, :]
nsteps_use = max(nsteps - len(old_chain) // nwalkers, 0)
# initializing sampler
chain_file = SampleFileUtil(prefix_chain,
carry_on=ch_config_params['rerun'])
sampler = emcee.EnsembleSampler(nwalkers,
nparams,
lnprob,
args=(params, param_mapping, param_tracer,
newData, newBall),
pool=pool_use)
start = time.time()
print("Running %d samples" % nsteps_use)
# record every iteration
counter = 1
for pos, prob, _ in sampler.sample(p_initial, iterations=nsteps_use):
if pool.is_master():
print('Iteration done. Persisting.')
chain_file.persistSamplingValues(pos, prob)
if counter % 10:
print(f"Finished sample {counter}")
counter += 1
pool.close()
end = time.time()
print("Took ", (end - start), " seconds")
def fit_events(self,
events=[],
models=[],
max_time='',
time_list=[],
time_unit=None,
band_list=[],
band_systems=[],
band_instruments=[],
band_bandsets=[],
band_sampling_points=17,
iterations=10000,
num_walkers=None,
num_temps=1,
parameter_paths=['parameters.json'],
fracking=True,
frack_step=50,
burn=None,
post_burn=None,
gibbs=False,
smooth_times=-1,
extrapolate_time=0.0,
limit_fitting_mjds=False,
exclude_bands=[],
exclude_instruments=[],
exclude_systems=[],
exclude_sources=[],
exclude_kinds=[],
output_path='',
suffix='',
upload=False,
write=False,
upload_token='',
check_upload_quality=False,
variance_for_each=[],
user_fixed_parameters=[],
user_released_parameters=[],
convergence_type=None,
convergence_criteria=None,
save_full_chain=False,
draw_above_likelihood=False,
maximum_walltime=False,
start_time=False,
print_trees=False,
maximum_memory=np.inf,
speak=False,
return_fits=True,
extra_outputs=None,
walker_paths=[],
catalogs=[],
exit_on_prompt=False,
download_recommended_data=False,
local_data_only=False,
method=None,
seed=None,
**kwargs):
"""Fit a list of events with a list of models."""
global model
if start_time is False:
start_time = time.time()
self._seed = seed
if seed is not None:
np.random.seed(seed)
self._start_time = start_time
self._maximum_walltime = maximum_walltime
self._maximum_memory = maximum_memory
self._debug = False
self._speak = speak
self._download_recommended_data = download_recommended_data
self._local_data_only = local_data_only
self._draw_above_likelihood = draw_above_likelihood
prt = self._printer
event_list = listify(events)
model_list = listify(models)
if len(model_list) and not len(event_list):
event_list = ['']
# Exclude catalogs not included in catalog list.
self._fetcher.add_excluded_catalogs(catalogs)
if not len(event_list) and not len(model_list):
prt.message('no_events_models', warning=True)
# If the input is not a JSON file, assume it is either a list of
# transients or that it is the data from a single transient in tabular
# form. Try to guess the format first, and if that fails ask the user.
self._converter = Converter(prt, require_source=upload)
event_list = self._converter.generate_event_list(event_list)
event_list = [x.replace('‑', '-') for x in event_list]
entries = [[] for x in range(len(event_list))]
ps = [[] for x in range(len(event_list))]
lnprobs = [[] for x in range(len(event_list))]
# Load walker data if provided a list of walker paths.
walker_data = []
if len(walker_paths):
try:
pool = MPIPool()
except (ImportError, ValueError):
pool = SerialPool()
if pool.is_master():
prt.message('walker_file')
wfi = 0
for walker_path in walker_paths:
if os.path.exists(walker_path):
prt.prt(' {}'.format(walker_path))
with codecs.open(walker_path, 'r',
encoding='utf-8') as f:
all_walker_data = json.load(
f, object_pairs_hook=OrderedDict)
# Support both the format where all data stored in a
# single-item dictionary (the OAC format) and the older
# MOSFiT format where the data was stored in the
# top-level dictionary.
if ENTRY.NAME not in all_walker_data:
all_walker_data = all_walker_data[list(
all_walker_data.keys())[0]]
models = all_walker_data.get(ENTRY.MODELS, [])
choice = None
if len(models) > 1:
model_opts = [
'{}-{}-{}'.format(x['code'], x['name'],
x['date']) for x in models
]
choice = prt.prompt('select_model_walkers',
kind='select',
message=True,
options=model_opts)
choice = model_opts.index(choice)
elif len(models) == 1:
choice = 0
if choice is not None:
walker_data.extend([[
wfi, x[REALIZATION.PARAMETERS],
x.get(REALIZATION.WEIGHT)
] for x in models[choice][MODEL.REALIZATIONS]])
for i in range(len(walker_data)):
if walker_data[i][2] is not None:
walker_data[i][2] = float(walker_data[i][2])
if not len(walker_data):
prt.message('no_walker_data')
else:
prt.message('no_walker_data')
if self._offline:
prt.message('omit_offline')
raise RuntimeError
wfi = wfi + 1
for rank in range(1, pool.size + 1):
pool.comm.send(walker_data, dest=rank, tag=3)
else:
walker_data = pool.comm.recv(source=0, tag=3)
pool.wait()
if pool.is_master():
pool.close()
self._event_name = 'Batch'
self._event_path = ''
self._event_data = {}
try:
pool = MPIPool()
except (ImportError, ValueError):
pool = SerialPool()
if pool.is_master():
fetched_events = self._fetcher.fetch(
event_list,
offline=self._offline,
prefer_cache=self._prefer_cache)
for rank in range(1, pool.size + 1):
pool.comm.send(fetched_events, dest=rank, tag=0)
pool.close()
else:
fetched_events = pool.comm.recv(source=0, tag=0)
pool.wait()
for ei, event in enumerate(fetched_events):
if event is not None:
self._event_name = event.get('name', 'Batch')
self._event_path = event.get('path', '')
if not self._event_path:
continue
self._event_data = self._fetcher.load_data(event)
if not self._event_data:
continue
if model_list:
lmodel_list = model_list
else:
lmodel_list = ['']
entries[ei] = [None for y in range(len(lmodel_list))]
ps[ei] = [None for y in range(len(lmodel_list))]
lnprobs[ei] = [None for y in range(len(lmodel_list))]
if (event is not None and
(not self._event_data or ENTRY.PHOTOMETRY
not in self._event_data[list(self._event_data.keys())[0]])):
prt.message('no_photometry', [self._event_name])
continue
for mi, mod_name in enumerate(lmodel_list):
for parameter_path in parameter_paths:
try:
pool = MPIPool()
except (ImportError, ValueError):
pool = SerialPool()
self._model = Model(model=mod_name,
data=self._event_data,
parameter_path=parameter_path,
output_path=output_path,
wrap_length=self._wrap_length,
test=self._test,
printer=prt,
fitter=self,
pool=pool,
print_trees=print_trees)
if not self._model._model_name:
prt.message('no_models_avail', [self._event_name],
warning=True)
continue
if not event:
prt.message('gen_dummy')
self._event_name = mod_name
gen_args = {
'name': mod_name,
'max_time': max_time,
'time_list': time_list,
'band_list': band_list,
'band_systems': band_systems,
'band_instruments': band_instruments,
'band_bandsets': band_bandsets
}
self._event_data = self.generate_dummy_data(**gen_args)
success = False
alt_name = None
while not success:
self._model.reset_unset_recommended_keys()
success = self._model.load_data(
self._event_data,
event_name=self._event_name,
smooth_times=smooth_times,
extrapolate_time=extrapolate_time,
limit_fitting_mjds=limit_fitting_mjds,
exclude_bands=exclude_bands,
exclude_instruments=exclude_instruments,
exclude_systems=exclude_systems,
exclude_sources=exclude_sources,
exclude_kinds=exclude_kinds,
time_list=time_list,
time_unit=time_unit,
band_list=band_list,
band_systems=band_systems,
band_instruments=band_instruments,
band_bandsets=band_bandsets,
band_sampling_points=band_sampling_points,
variance_for_each=variance_for_each,
user_fixed_parameters=user_fixed_parameters,
user_released_parameters=user_released_parameters,
pool=pool)
if not success:
break
if self._local_data_only:
break
# If our data is missing recommended keys, offer the
# user option to pull the missing data from online and
# merge with existing data.
urk = self._model.get_unset_recommended_keys()
ptxt = prt.text('acquire_recommended',
[', '.join(list(urk))])
while event and len(urk) and (
alt_name or self._download_recommended_data
or prt.prompt(ptxt, [', '.join(urk)],
kind='bool')):
try:
pool = MPIPool()
except (ImportError, ValueError):
pool = SerialPool()
if pool.is_master():
en = (alt_name
if alt_name else self._event_name)
extra_event = self._fetcher.fetch(
en,
offline=self._offline,
prefer_cache=self._prefer_cache)[0]
extra_data = self._fetcher.load_data(
extra_event)
for rank in range(1, pool.size + 1):
pool.comm.send(extra_data,
dest=rank,
tag=4)
pool.close()
else:
extra_data = pool.comm.recv(source=0, tag=4)
pool.wait()
if extra_data is not None:
extra_data = extra_data[list(
extra_data.keys())[0]]
for key in urk:
new_val = extra_data.get(key)
self._event_data[list(
self._event_data.keys())
[0]][key] = new_val
if new_val is not None and len(new_val):
prt.message('extra_value', [
key,
str(new_val[0].get(QUANTITY.VALUE))
])
success = False
prt.message('reloading_merged')
break
else:
text = prt.text('extra_not_found',
[self._event_name])
alt_name = prt.prompt(text, kind='string')
if not alt_name:
break
if success:
self._walker_data = walker_data
entry, p, lnprob = self.fit_data(
event_name=self._event_name,
method=method,
iterations=iterations,
num_walkers=num_walkers,
num_temps=num_temps,
burn=burn,
post_burn=post_burn,
fracking=fracking,
frack_step=frack_step,
gibbs=gibbs,
pool=pool,
output_path=output_path,
suffix=suffix,
write=write,
upload=upload,
upload_token=upload_token,
check_upload_quality=check_upload_quality,
convergence_type=convergence_type,
convergence_criteria=convergence_criteria,
save_full_chain=save_full_chain,
extra_outputs=extra_outputs)
if return_fits:
entries[ei][mi] = deepcopy(entry)
ps[ei][mi] = deepcopy(p)
lnprobs[ei][mi] = deepcopy(lnprob)
if pool.is_master():
pool.close()
# Remove global model variable and garbage collect.
try:
model
except NameError:
pass
else:
del (model)
del (self._model)
gc.collect()
return (entries, ps, lnprobs)
parser.add_argument('savepath')
args = parser.parse_args()
n = args.n
S_ = args.S
savepath = args.savepath
p = 250
comm = MPI.COMM_WORLD
rank = comm.rank
numproc = comm.Get_size()
# Keep sigma and gamma squared equal to each other
sigma_sq = 1
gamma_sq = 0.1
F = np.linspace(0, 2 * np.log(n), 10)
T = np.linspace(1, p, 25, dtype=int)
pool = MPIPool(comm)
worker = Worker(savepath, gamma_sq, sigma_sq, n, S_)
# Split F and T into tasks
tasks = itertools.product(T, F)
pool.map(worker, tasks, callback=worker.save)
pool.close()
# update epsilon based on median thresholding
eps.eps = np.median(pool.dists, axis=0)
print('eps%i' % pool.t, eps.eps)
print('----------------------------------------')
#if pool.ratio <0.2: break
abcpmc_sampler.close()
return None
if __name__ == "__main__":
if machine == 'siro':
# run with MPI
from mpi4py import MPI
from schwimmbad import MPIPool
pewl = MPIPool()
if not pewl.is_master():
pewl.wait()
sys.exit(0)
else:
pewl = None
name = sys.argv[7] # name of ABC run
niter = int(sys.argv[8]) # number of iterations
restart = (sys.argv[9] == 'True')
print('Runnin ABC with ...')
print('%s simulation' % sim)
print('%s DEM' % dem)
print('%s distance' % distance_method)
print('%s summary statistic' % statistic)
return
beam, tune, pol = cFrame.parseID()
if tune == 0:
tune += 1
aStand = 2 * (tune - 1) + pol
data[aStand, count[aStand] * 4096:(count[aStand] + 1) *
4096] = cFrame.data.iq
count[aStand] += 1
# Calculate the spectra for this block of data, in the unit of intensity
masterSpectra[i, 0, :] = (
(numpy.fft.fftshift(numpy.abs(numpy.fft.fft2(
data[:2, :]))[:, 1:])[:, Lfcl:Lfch])**2.).mean(0) / LFFT / 2.
masterSpectra[i, 1, :] = (
(numpy.fft.fftshift(numpy.abs(numpy.fft.fft2(
data[2:, :]))[:, 1:])[:, Hfcl:Hfch])**2.).mean(0) / LFFT / 2.
# Save the results to the various master arrays
outname = "%s_%i_fft_offset_%.9i_frames" % (filename, beam, offset)
log("Writing %s" % outname)
numpy.save(outname, masterSpectra)
if __name__ == "__main__":
from schwimmbad import MPIPool
pool = MPIPool()
if not pool.is_master(): # Workers wait here
pool.wait()
sys.exit(0)
main(pool)
