def initial_pool(prior_obj, eps0, N_particles, N_threads=1):
""" Initial Pool
"""
args_list = [[i, prior_obj, eps0, N_particles]
for i in xrange(N_particles)]
if N_threads > 1:
pool = InterruptiblePool(processes=N_threads)
mapfn = pool.map
results = mapfn(initial_pool_sampling, args_list)
pool.close()
pool.terminate()
pool.join()
else:
results = []
for arg in args_list:
results.append(initial_pool_sampling(arg))
results = np.array(results).T
theta_t = results[1:prior_obj.n_params + 1, :]
w_t = results[prior_obj.n_params + 1, :]
rhos = results[prior_obj.n_params + 2, :]
sig_t = covariance(theta_t, w_t)
return theta_t, w_t, rhos, sig_t
def log_multivariate_gaussian_Nthreads(x, mu, V, xcov, Nthreads=1):
"""
Use multiprocessing to calculate log likelihoods.
"""
n_samples = x.shape[0]
pool = InterruptiblePool(Nthreads)
mapfn = pool.map
Nchunk = np.ceil(1. / Nthreads * n_samples).astype(np.int)
arglist = [None] * Nthreads
for i in range(Nthreads):
s = i * Nchunk
e = s + Nchunk
arglist[i] = (x[s:e], mu, V, xcov[s:e])
result = list(mapfn(lmg, [args for args in arglist]))
logls = result[0]
for i in range(1, Nthreads):
logls = np.vstack((logls, result[i]))
pool.close()
pool.terminate()
pool.join()
return logls
def func_grad_lnX_Nthreads(self, params):
"""
Use multiprocessing to calculate negative-log-likelihood and gradinets
w.r.t lnX, plus the terms coming from th regularization terms.
"""
n_samples = self.N
self.lnX = params
#self.fl, self.f, self.g, self.H, Nthreads = args
Pool = InterruptiblePool(Nthreads)
mapfn = Pool.map
Nchunk = np.ceil(1. / Nthreads * n_samples).astype(np.int)
arglist = [None] * Nthreads
for i in range(Nthreads):
s = int(i * Nchunk)
e = int(s + Nchunk)
arglist[i] = (self.lnX, self.F, self.B, self.fl, self.f, self.g, self.H, s, e)
result = list(mapfn(fg, [ars for ars in arglist]))
nll, grad = result[0]
a = time.time()
for i in range(1, Nthreads):
nll += result[i][0]
grad += result[i][1]
#print "adding up nll's from individual threads", time.time() - a
Pool.close()
Pool.terminate()
Pool.join()
#computing the regularization term and its derivative w.r.t lnX
reg_func, reg_grad = self.reg_func_grad_lnX()
return nll + reg_func, grad + reg_grad
def pmc_abc(N_threads = N_threads):
# initial pool
theta_t, w_t, rhos, sig_t = initial_pool()
t = 0 # iternation number
plot_thetas(theta_t , w_t, t)
plt.savefig("/home/mj/public_html/scatter_hod_gaussian_t"+str(t)+".png")
plt.close()
while t < N_iter:
eps_t = np.percentile(rhos, 75)
print 'New Distance Threshold Eps_t = ', eps_t
theta_t_1 = theta_t.copy()
w_t_1 = w_t.copy()
sig_t_1 = sig_t.copy()
"""these lines are borrowed from initial sampling to double-check multiprocessing"""
#pool = InterruptiblePool(processes = N_threads)
#mapfn = pool.map
#args_list = [i for i in xrange(N_particles)]
#results = mapfn(initial_pool_sampling, args_list)
#pool.close()
#pool.terminate()
#pool.join()
pool = InterruptiblePool(processes = N_threads)
mapfn = pool.map
args_list = [[i, theta_t_1, w_t_1, sig_t_1, eps_t] for i in xrange(N_particles)]
#results = []
#for args in args_list:
# pool_sample = importance_pool_sampling(args)
# results.append( pool_sample )
results = mapfn(importance_pool_sampling, args_list)
pool.close()
pool.terminate()
pool.join()
sig_t = np.cov(theta_t)
results = np.array(results).T
theta_t = results[1:n_params+1,:]
w_t = results[n_params+1,:]
rhos = results[n_params+2,:]
sig_t = np.cov(theta_t)
t += 1
plot_thetas(theta_t, w_t , t)
plt.savefig("/home/mj/public_html/scatter_hod_gaussian_t"+str(t)+".png")
plt.close()
def initial_pool(prior_obj, eps0, N_particles, N_threads=1):
""" Initial Pool
"""
args_list = [[i, prior_obj, eps0, N_particles] for i in xrange(N_particles)]
if N_threads > 1:
pool = InterruptiblePool(processes = N_threads)
mapfn = pool.map
results = mapfn(initial_pool_sampling, args_list)
pool.close()
pool.terminate()
pool.join()
else:
results = []
for arg in args_list:
results.append(initial_pool_sampling(arg))
results = np.array(results).T
theta_t = results[1:prior_obj.n_params+1,:]
w_t = results[prior_obj.n_params+1,:]
rhos = results[prior_obj.n_params+2,:]
sig_t = covariance(theta_t , w_t)
return theta_t, w_t, rhos, sig_t
def pmc_abc(N_threads=N_threads):
# initial pool
theta_t, w_t, rhos, sig_t = initial_pool()
t = 0 # iternation number
plot_thetas(theta_t, w_t, t)
plt.savefig("/home/mj/public_html/scatter_hod_gaussian_t" + str(t) + ".png")
plt.close()
while t < N_iter:
eps_t = np.percentile(rhos, 75)
print "New Distance Threshold Eps_t = ", eps_t
theta_t_1 = theta_t.copy()
w_t_1 = w_t.copy()
sig_t_1 = sig_t.copy()
"""these lines are borrowed from initial sampling to double-check multiprocessing"""
# pool = InterruptiblePool(processes = N_threads)
# mapfn = pool.map
# args_list = [i for i in xrange(N_particles)]
# results = mapfn(initial_pool_sampling, args_list)
# pool.close()
# pool.terminate()
# pool.join()
pool = InterruptiblePool(processes=N_threads)
mapfn = pool.map
args_list = [[i, theta_t_1, w_t_1, sig_t_1, eps_t] for i in xrange(N_particles)]
# results = []
# for args in args_list:
# pool_sample = importance_pool_sampling(args)
# results.append( pool_sample )
results = mapfn(importance_pool_sampling, args_list)
pool.close()
pool.terminate()
pool.join()
sig_t = np.cov(theta_t)
results = np.array(results).T
theta_t = results[1 : n_params + 1, :]
w_t = results[n_params + 1, :]
rhos = results[n_params + 2, :]
sig_t = np.cov(theta_t)
t += 1
plot_thetas(theta_t, w_t, t)
plt.savefig("/home/mj/public_html/scatter_hod_gaussian_t" + str(t) + ".png")
plt.close()
def pmc_abc(prior_dict, N_particles=100, N_iter=30, eps0=20.0, N_threads=1):
"""
"""
prior_obj = Prior(prior_dict)
# initial pool
theta_t, w_t, rhos, sig_t = initial_pool(prior_obj,
eps0,
N_particles,
N_threads=N_threads)
t = 0 # iternation number
#plot_thetas(theta_t , w_t, prior_dict, t)
while t < N_iter:
eps_t = np.percentile(rhos, 75)
print 'New Distance Threshold Eps_t = ', eps_t
theta_t_1 = theta_t.copy()
w_t_1 = w_t.copy()
sig_t_1 = sig_t.copy()
args_list = [[i, prior_obj, theta_t_1, w_t_1, sig_t_1, eps_t]
for i in xrange(N_particles)]
if N_threads > 1:
pool = InterruptiblePool(processes=N_threads)
mapfn = pool.map
results = mapfn(importance_pool_sampling, args_list)
pool.close()
pool.terminate()
pool.join()
else:
results = []
for args in args_list:
pool_sample = importance_pool_sampling(args)
results.append(pool_sample)
results = np.array(results).T
theta_t = results[1:prior_obj.n_params + 1, :]
w_t = results[prior_obj.n_params + 1, :]
rhos = results[prior_obj.n_params + 2, :]
sig_t = covariance(theta_t, w_t)
t += 1
plot_thetas(theta_t, w_t, prior_dict, t)
def pmc_abc(N_threads=N_threads):
# initial pool
theta_t, w_t, rhos, sig_t = initial_pool()
w_t = w_t / np.sum(w_t)
t = 0 # iternation number
plot_thetas(theta_t, w_t, t)
while t < N_iter:
if t < 4:
eps_t = np.percentile(np.atleast_2d(rhos), 20, axis=1)
else:
eps_t = np.percentile(np.atleast_2d(rhos), 50, axis=1)
print 'New Distance Threshold Eps_t = ', eps_t, "t=", t
theta_t_1 = theta_t.copy()
w_t_1 = w_t.copy()
sig_t_1 = sig_t.copy()
args_list = [[i, theta_t_1, w_t_1, sig_t_1, eps_t]
for i in xrange(N_particles)]
"""serial"""
results = []
#for args in args_list:
# pool_sample = importance_pool_sampling(args)
# results.append( pool_sample )
"""parallel"""
pool = InterruptiblePool(processes=N_threads)
mapfn = pool.map
results = mapfn(importance_pool_sampling, args_list)
pool.close()
pool.terminate()
pool.join()
results = np.array(results).T
theta_t = results[1:n_params + 1, :]
w_t = results[n_params + 1, :]
w_t = w_t / np.sum(w_t)
rhos = results[n_params + 2:, :]
#sig_t = knn_sigma(theta_t , k = 10)
sig_t = 2. * covariance(theta_t, w_t)
t += 1
plot_thetas(theta_t, w_t, t)
def initial_pool():
pool = InterruptiblePool(processes = N_threads)
mapfn = pool.map
args_list = [i for i in xrange(N_particles)]
results = mapfn(initial_pool_sampling, args_list)
pool.close()
pool.terminate()
pool.join()
results = np.array(results).T
theta_t = results[1:n_params+1,:]
w_t = results[n_params+1,:]
rhos = results[n_params+2,:]
sig_t = np.cov(theta_t)
return theta_t, w_t, rhos, sig_t
def pmc_abc(N_threads = N_threads):
# initial pool
theta_t, w_t, rhos, sig_t = initial_pool()
w_t = w_t/np.sum(w_t)
t = 0 # iternation number
plot_thetas(theta_t , w_t, t)
while t < N_iter:
if t < 4 :
eps_t = np.percentile(np.atleast_2d(rhos), 20, axis=1)
else:
eps_t = np.percentile(np.atleast_2d(rhos), 50, axis=1)
print 'New Distance Threshold Eps_t = ', eps_t , "t=" , t
theta_t_1 = theta_t.copy()
w_t_1 = w_t.copy()
sig_t_1 = sig_t.copy()
args_list = [[i, theta_t_1, w_t_1, sig_t_1, eps_t] for i in xrange(N_particles)]
"""serial"""
results = []
#for args in args_list:
# pool_sample = importance_pool_sampling(args)
# results.append( pool_sample )
"""parallel"""
pool = InterruptiblePool(processes = N_threads)
mapfn = pool.map
results = mapfn(importance_pool_sampling, args_list)
pool.close()
pool.terminate()
pool.join()
results = np.array(results).T
theta_t = results[1:n_params+1,:]
w_t = results[n_params+1,:]
w_t = w_t/np.sum(w_t)
rhos = results[n_params+2:,:]
#sig_t = knn_sigma(theta_t , k = 10)
sig_t = 2. * covariance(theta_t , w_t)
t += 1
plot_thetas(theta_t, w_t , t)
def pmc_abc(prior_dict, N_particles=100, N_iter=30, eps0=20.0, N_threads = 1):
"""
"""
prior_obj = Prior(prior_dict)
# initial pool
theta_t, w_t, rhos, sig_t = initial_pool(prior_obj, eps0, N_particles, N_threads=N_threads)
t = 0 # iternation number
#plot_thetas(theta_t , w_t, prior_dict, t)
while t < N_iter:
eps_t = np.percentile(rhos, 75)
print 'New Distance Threshold Eps_t = ', eps_t
theta_t_1 = theta_t.copy()
w_t_1 = w_t.copy()
sig_t_1 = sig_t.copy()
args_list = [[i, prior_obj, theta_t_1, w_t_1, sig_t_1, eps_t] for i in xrange(N_particles)]
if N_threads > 1:
pool = InterruptiblePool(processes = N_threads)
mapfn = pool.map
results = mapfn(importance_pool_sampling, args_list)
pool.close()
pool.terminate()
pool.join()
else:
results = []
for args in args_list:
pool_sample = importance_pool_sampling(args)
results.append( pool_sample )
results = np.array(results).T
theta_t = results[1:prior_obj.n_params+1,:]
w_t = results[prior_obj.n_params+1,:]
rhos = results[prior_obj.n_params+2,:]
sig_t = covariance(theta_t , w_t)
t += 1
plot_thetas(theta_t , w_t, prior_dict, t)
def initial_pool():
pool = InterruptiblePool(processes=N_threads)
mapfn = pool.map
args_list = [i for i in xrange(N_particles)]
results = mapfn(initial_pool_sampling, args_list)
pool.close()
pool.terminate()
pool.join()
results = np.array(results).T
theta_t = results[1 : n_params + 1, :]
w_t = results[n_params + 1, :]
rhos = results[n_params + 2, :]
sig_t = np.cov(theta_t)
return theta_t, w_t, rhos, sig_t
def initial_pool():
args_list = np.arange(N_particles)
"""serial"""
#results = []
#for arg in args_list:
# results.append(initial_pool_sampling(arg))
"""parallel"""
pool = InterruptiblePool(processes=N_threads)
mapfn = pool.map
results = mapfn(initial_pool_sampling, args_list)
pool.close()
pool.terminate()
pool.join()
results = np.array(results).T
theta_t = results[1:n_params + 1, :]
w_t = results[n_params + 1, :]
w_t = w_t / np.sum(w_t)
rhos = results[n_params + 2:, :]
sig_t = covariance(theta_t, w_t)
return theta_t, w_t, rhos, sig_t
def pmc_abc(self):
"""
"""
self.rhos = self.initial_pool()
while self.t < self.T:
self.eps_t = np.percentile(self.rhos, 75)
print 'Epsilon t', self.eps_t
self.theta_t_1 = self.theta_t.copy()
self.w_t_1 = self.w_t.copy()
self.sig_t_1 = self.sig_t.copy()
pool = InterruptiblePool(self.Nthreads)
mapfn = pool.map
args_list = [ i for i in xrange(self.N) ]
results = mapfn(unwrap_self_importance_sampling, zip([self]*len(args_list), args_list))
pool.close()
pool.terminate()
pool.join()
pars = np.array(results).T
self.theta_t = pars[1:self.n_params+1,:].copy()
self.w_t = pars[self.n_params+1,:].copy()
self.rhos = pars[self.n_params+2,:].copy()
self.sig_t = 2.0 * np.cov(self.theta_t)
self.t += 1
self.writeout()
self.plotout()
return None
def parallel_bulkfit(path, num_splits=20, ncores=8, start_pt=0):
'''
Run bulk fitting in parallel. Results are outputted in chunks to make
restarting easier.
'''
spectra = [f for f in os.listdir(path) if f[-4:] == 'fits']
split_at = len(spectra) / num_splits
splits = [split_at * i for i in range(1, num_splits)]
splits.append(len(spectra))
splits = splits[start_pt:]
prev_split = 0
for i, split in enumerate(splits):
print("On split " + str(i + 1) + " of " + str(len(splits)))
print(str(datetime.now()))
split_spectra = spectra[prev_split:split]
pool = Pool(processes=ncores)
output = pool.map(do_specfit, split_spectra)
pool.close()
pool.join()
df = DataFrame(output[0], columns=split_spectra[:1])
for out, spec in zip(output[1:], split_spectra[1:]):
df[spec[:-5]] = out
df.to_csv("spectral_fitting_" + str(i + 1) + ".csv")
prev_split = split
offsets = [
31.697, 14.437, 85.452, 26.216, 9.330, -879.063, 23.698, 21.273,
20.728, 32.616, 35.219, 9.005, 10.124, 14.678
]
beamwidth_250 = [18.2] * len(fits250)
beamwidth_350 = [24.9] * len(fits350)
# Inputs (adjust to desired wavelength)
beamwidths = beamwidth_350 # + beamwidth_350
distances = distances # + distances
fits_files = fits350 # + fits350
print "Started at " + str(datetime.now())
if not MULTICORE:
for i, filename in enumerate(fits_files):
wrapper(filename,
distances[i],
beamwidths[i],
offsets[i],
verbose=False)
else:
pool = Pool(processes=NCORES)
pool.map(single_input, izip(fits_files, distances, beamwidths,
offsets))
pool.close()
# pool.join()
if arg1 == '3pcf':
nreal_i = int(Sys.argv[3])
nreal_f = int(Sys.argv[4])
nzbin = int(Sys.argv[5])
zstr = Sys.argv[6]
if zstr == 'z':
zbool = True
elif zstr == 'real':
zbool = False
else:
raise ValueError
nthreads = int(Sys.argv[7])
if nthreads > 1:
args_list = [(mneut, ireal, nzbin, zbool) for ireal in np.arange(nreal_i, nreal_f+1)]
pool = Pewl(processes=nthreads)
mapfn = pool.map
results = mapfn(_NeutHalo_pre3PCF, args_list)
pool.close()
pool.terminate()
pool.join()
else:
for ireal in range(nreal_i, nreal_f+1):
NeutHalo_pre3PCF(mneut, ireal, nzbin, zspace=zbool)
elif arg1 == 'plk':
nreal = int(Sys.argv[3])
nzbin = int(Sys.argv[4])
zstr = Sys.argv[5]
if zstr == 'z':
zbool = True
elif zstr == 'real':
