def abc_hodsim(Mr=21, b_normal=0.25):
prior_min, prior_max = PriorRange('first_try')
prior_range = np.zeros((len(prior_min),2))
prior_range[:,0] = prior_min
prior_range[:,1] = prior_max
abc_hod = ABC_HODsim()
data_hod_dict = Data.data_hod_param(Mr=21)
data_hod = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
print Data.data_nbar(Mr=Mr, b_normal=b_normal), Data.data_gmf(Mr=Mr, b_normal=b_normal)
print abc_hod(data_hod, prior_range=prior_range, observables=['nbar', 'gmf'])
def abc_hodsim(Mr=21, b_normal=0.25):
prior_min, prior_max = PriorRange('first_try')
prior_range = np.zeros((len(prior_min), 2))
prior_range[:, 0] = prior_min
prior_range[:, 1] = prior_max
abc_hod = ABC_HODsim()
data_hod_dict = Data.data_hod_param(Mr=21)
data_hod = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
print Data.data_nbar(Mr=Mr,
b_normal=b_normal), Data.data_gmf(Mr=Mr,
b_normal=b_normal)
print abc_hod(data_hod,
prior_range=prior_range,
observables=['nbar', 'gmf'])
def ABC_Coner(obvs, weighted=False):
''' Pretty corner plot
'''
if obvs == 'nbargmf':
result_dir = ''.join([
ut.dat_dir(),
'paper/ABC',
obvs,
'/run1/',
])
theta_file = lambda tt: ''.join(
[result_dir, 'nbar_gmf_theta_t',
str(tt), '.ABCnbargmf.dat'])
w_file = lambda tt: ''.join(
[result_dir, 'nbar_gmf_w_t',
str(tt), '.ABCnbargmf.dat'])
tf = 8
elif obvs == 'nbarxi':
result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/'])
theta_file = lambda tt: ''.join(
[result_dir, 'nbar_xi_theta_t',
str(tt), '.abc.dat'])
w_file = lambda tt: ''.join(
[result_dir, 'nbar_xi_w_t',
str(tt), '.abc.dat'])
tf = 9
else:
raise ValueError
theta = np.loadtxt(theta_file(tf))
if weighted:
weights = np.loadtxt(w_file(tf))
else:
weights = None
true_dict = Data.data_hod_param(Mr=21)
true_theta = [
true_dict['logM0'],
np.log(true_dict['sigma_logM']), true_dict['logMmin'],
true_dict['alpha'], true_dict['logM1']
]
prior_min, prior_max = PriorRange('first_try')
prior_range = np.zeros((len(prior_min), 2))
prior_range[:, 0] = np.array(prior_min)
prior_range[:, 1] = np.array(prior_max)
fig = corner.corner(theta,
weights=weights,
truths=true_theta,
truth_color='k',
labels=[
r'$\log\;\mathcal{M}_{0}}$',
r'$\log\;\sigma_\mathtt{log\;M}}$',
r'$\log\;\mathcal{M}_\mathtt{min}}$', r'$\alpha$',
r'$\log\;\mathcal{M}_{1}}$'
],
label_kwargs={'fontsize': 25},
range=prior_range,
quantiles=[0.16, 0.5, 0.84],
show_titles=True,
title_args={"fontsize": 12},
plot_datapoints=True,
fill_contours=True,
levels=[0.68, 0.95],
color='#ee6a50',
bins=20,
smooth=1.0)
fig_name = ''.join([ut.fig_dir(), 'paper.ABCcorner', '.', obvs, '.pdf'])
fig.savefig(fig_name, bbox_inches='tight', dpi=150)
plt.close()
return None
def plot_mcmc(Nwalkers, Niter=1000, Nchains_burn=200, Mr=21, truths=None,
observables=['nbar', 'xi'], plot_range=None):
'''
Plot MCMC chains
'''
if truths is None:
data_hod_dict = Data.data_hod_param(Mr=Mr)
truths = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
if plot_range is None:
prior_min, prior_max = PriorRange(None)
plot_range = np.zeros((len(prior_min),2))
plot_range[:,0] = prior_min
plot_range[:,1] = prior_max
# chain files
chain_file = ''.join([util.dat_dir(),
util.observable_id_flag(observables),
'_Mr', str(Mr), '.mcmc_chain.dat'])
#f = h5py.File(chain_file, 'r')
#sample = f['positions'][:]
sample = np.loadtxt(chain_file)
# Posterior Likelihood Corner Plot
fig = corner.corner(
sample[Nchains_burn*Nwalkers:],
truths=truths,
truth_color='#ee6a50',
labels=[
r'$\mathtt{\log\;M_{0}}$',
r'$\mathtt{\log\;\sigma_{\logM}}$',
r'$\mathtt{\log\;M_{min}}$',
r'$\mathtt{\alpha}$',
r'$\mathtt{\log\;M_{1}}$'
],
label_kwargs={'fontsize': 25},
range=plot_range,
quantiles=[0.16,0.5,0.84],
show_titles=True,
title_args={"fontsize": 12},
plot_datapoints=True,
fill_contours=True,
levels=[0.68, 0.95],
color='b',
bins=16,
smooth=1.0)
fig_file = ''.join([util.fig_dir(),
util.observable_id_flag(observables),
'_Mr', str(Mr), '.Niter', str(Niter),
'.Nburn', str(Nchains_burn), '.mcmc_samples.test.png'])
#print fig_file
plt.savefig(fig_file)
plt.close()
# MCMC Chain plot
Ndim = len(sample[0])
Nchain = len(sample)/Nwalkers
chain_ensemble = sample.reshape(Nchain, Nwalkers, Ndim)
fig , axes = plt.subplots(5, 1 , sharex=True, figsize=(10, 12))
labels=[
r'$\mathtt{\log\;M_{0}}$',
r'$\mathtt{\log\;\sigma_{\logM}}$',
r'$\mathtt{\log\;M_{min}}$',
r'$\mathtt{\alpha}$',
r'$\mathtt{\log\;M_{1}}$'
]
for i in xrange(5):
axes[i].plot(chain_ensemble[:, :, i], color="k", alpha=0.4)
axes[i].yaxis.set_major_locator(MaxNLocator(5))
axes[i].axhline(truths[i], color="#888888", lw=2)
axes[i].vlines(Nchains_burn, plot_range[i,0], plot_range[i,1], colors='#ee6a50', linewidth=4, alpha=1)
axes[i].set_ylim([plot_range[i,0], plot_range[i,1]])
axes[i].set_xlim(0, 6000)
axes[i].set_ylabel(labels[i], fontsize=25)
axes[4].set_xlabel("Step Number", fontsize=25)
fig.tight_layout(h_pad=0.0)
fig_file = ''.join([util.fig_dir(),
util.observable_id_flag(observables),
'_Mr', str(Mr), '.Niter', str(Niter),
'.Nburn', str(Nchains_burn), '.mcmc_time.test.png'])
plt.savefig(fig_file)
plt.close()
def mcmc_mpi(Nwalkers,
Nchains,
observables=['nbar', 'xi'],
data_dict={
'Mr': 21,
'b_normal': 0.25
},
prior_name='first_try',
mcmcrun=None):
'''
Standard MCMC implementaion
Parameters
-----------
- Nwalker :
Number of walkers
- Nchains :
Number of MCMC chains
- observables :
list of observables. Options are: ['nbar','xi'],['nbar','gmf'],['xi']
- data_dict : dictionary that specifies the observation keywords
'''
#Initializing the vector of observables and inverse covariance matrix
if observables == ['xi']:
fake_obs = Data.data_xi(**data_dict)
#fake_obs_icov = Data.data_inv_cov('xi', **data_dict)
fake_obs_icov = Data.data_cov(inference='mcmc', **data_dict)[1:16,
1:16]
if observables == ['nbar', 'xi']:
fake_obs = np.hstack(
[Data.data_nbar(**data_dict),
Data.data_xi(**data_dict)])
fake_obs_icov = Data.data_cov(inference='mcmc', **data_dict)[:16, :16]
if observables == ['nbar', 'gmf']:
##### FIRST BIN OF GMF DROPPED ###############
# CAUTION: hardcoded
fake_obs = np.hstack(
[Data.data_nbar(**data_dict),
Data.data_gmf(**data_dict)[1:]])
fake_obs_icov = np.zeros((10, 10))
#print Data.data_cov(**data_dict)[17: , 17:].shape
# Covariance matrix being adjusted accordingly
fake_obs_icov[1:, 1:] = Data.data_cov(inference='mcmc',
**data_dict)[17:, 17:]
fake_obs_icov[0, 1:] = Data.data_cov(inference='mcmc',
**data_dict)[0, 17:]
fake_obs_icov[1:, 0] = Data.data_cov(inference='mcmc',
**data_dict)[17:, 0]
fake_obs_icov[0, 0] = Data.data_cov(inference='mcmc', **data_dict)[0,
0]
# True HOD parameters
data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr'])
data_hod = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
Ndim = len(data_hod)
# Priors
prior_min, prior_max = PriorRange(prior_name)
prior_range = np.zeros((len(prior_min), 2))
prior_range[:, 0] = prior_min
prior_range[:, 1] = prior_max
# mcmc chain output file
chain_file = ''.join([
util.mcmc_dir(),
util.observable_id_flag(observables), '.', mcmcrun, '.mcmc_chain.dat'
])
#print chain_file
if os.path.isfile(chain_file) and continue_chain:
print 'Continuing previous MCMC chain!'
sample = np.loadtxt(chain_file)
Nchain = Niter - (len(sample) / Nwalkers
) # Number of chains left to finish
if Nchain > 0:
pass
else:
raise ValueError
print Nchain, ' iterations left to finish'
# Initializing Walkers from the end of the chain
pos0 = sample[-Nwalkers:]
else:
# new chain
f = open(chain_file, 'w')
f.close()
Nchain = Niter
# Initializing Walkers
random_guess = data_hod
pos0 = np.repeat(random_guess, Nwalkers).reshape(Ndim, Nwalkers).T + \
5.e-2 * np.random.randn(Ndim * Nwalkers).reshape(Nwalkers, Ndim)
#print pos0.shape
# Initializing MPIPool
pool = MPIPool()
if not pool.is_master():
pool.wait()
sys.exit(0)
# Initializing the emcee sampler
hod_kwargs = {
'prior_range': prior_range,
'data': fake_obs,
'data_icov': fake_obs_icov,
'observables': observables,
'Mr': data_dict['Mr']
}
sampler = emcee.EnsembleSampler(Nwalkers,
Ndim,
lnPost,
pool=pool,
kwargs=hod_kwargs)
# Initializing Walkers
for result in sampler.sample(pos0, iterations=Nchain, storechain=False):
position = result[0]
#print position
f = open(chain_file, 'a')
for k in range(position.shape[0]):
output_str = '\t'.join(position[k].astype('str')) + '\n'
f.write(output_str)
f.close()
pool.close()
for obv in observables:
if obv == 'nbar':
data_nbar, data_nbar_var = Data.data_nbar(**data_dict)
fake_obs.append(data_nbar)
if obv == 'gmf':
data_gmf, data_gmf_sigma = Data.data_gmf(**data_dict)
fake_obs.append(data_gmf)
if obv == 'xi':
# import xir and full covariance matrix of xir
data_xi, data_xi_cov = Data.data_xi_full_cov(**data_dict)
# take the inverse of the covariance matrix
data_xi_invcov = solve(np.eye(len(data_xi)), data_xi_cov)
fake_obs.append(data_xi)
# 'True' HOD parameters
data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr'])
data_hod = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
Ndim = len(data_hod)
# Priors
prior_min, prior_max = PriorRange(prior_name)
prior_range = np.zeros((len(prior_min), 2))
prior_range[:, 0] = prior_min
prior_range[:, 1] = prior_max
def ABCvsMCMC_contour(obvs, nwalkers=100, nburns=9000, sigma=False):
''' Plots that compare the ABC posteriors to the MCMC posteriors
'''
if obvs == 'nbargmf':
abc_dir = ''.join([
ut.dat_dir(),
'paper/ABC',
obvs,
'/run1/',
])
abc_theta_file = lambda tt: ''.join(
[abc_dir, 'nbar_gmf_theta_t',
str(tt), '.ABCnbargmf.dat'])
tf = 8
mcmc_dir = ''.join([ut.dat_dir(), 'paper/'])
mcmc_filename = ''.join([mcmc_dir, 'nbar_gmf.mcmc.mcmc_chain.p'])
elif obvs == 'nbarxi':
abc_dir = ''.join([
ut.dat_dir(),
'paper/ABC',
obvs,
'/',
])
abc_theta_file = lambda tt: ''.join(
[abc_dir, 'nbar_xi_theta_t',
str(tt), '.abc.dat'])
tf = 9
mcmc_dir = ''.join([ut.dat_dir(), 'paper/'])
mcmc_filename = ''.join([mcmc_dir, 'nbar_xi.mcmc.mcmc_chain.p'])
else:
raise ValueError
prior_min, prior_max = PriorRange('first_try')
prior_range = np.zeros((len(prior_min), 2))
prior_range[:, 0] = prior_min
prior_range[:, 1] = prior_max
# true HOD parameter
true_dict = Data.data_hod_param(Mr=21)
truths = [
true_dict['logM0'], # log M0
np.log(true_dict['sigma_logM']), # log(sigma)
true_dict['logMmin'], # log Mmin
true_dict['alpha'], # alpha
true_dict['logM1'] # log M1
]
mcmc_sample = pickle.load(open(mcmc_filename, 'rb'))[nburns * nwalkers:, :]
abc_sample = np.loadtxt(abc_theta_file(tf))
par_labels = [
r'$\mathtt{log}\;\mathcal{M}_{0}$',
r'$\mathtt{log}\;\sigma_\mathtt{log\;M}$',
r'$\mathtt{log}\;\mathcal{M}_\mathtt{min}$', r'$\alpha$',
r'$\mathtt{log}\;\mathcal{M}_{1}$'
]
prettyplot()
pretty_colors = prettycolors()
fig = plt.figure(1, figsize=(20, 6))
gs = gridspec.GridSpec(1, 3)
# first panel
for i in [0, 1, 2]:
plot_range = np.zeros((2, 2))
if i == 0:
col_pair = [2, 3]
plot_range[0, 0] = 12.5
plot_range[0, 1] = 13.0
plot_range[1, 0] = prior_range[3, 0]
plot_range[1, 1] = prior_range[3, 1]
elif i == 2:
col_pair = [4, 2]
plot_range[0, 0] = 13.6
plot_range[0, 1] = 14.2
plot_range[1, 0] = 12.5
plot_range[1, 1] = 13.0
elif i == 1:
col_pair = [3, 4]
plot_range[0, 0] = prior_range[3, 0]
plot_range[0, 1] = prior_range[3, 1]
plot_range[1, 0] = 13.6
plot_range[1, 1] = 14.2
if i == 2:
mcmc_label = r'$\mathcal{L}^\mathtt{Gauss}$ MCMC'
abc_label = 'ABC-PMC'
else:
mcmc_label = None
abc_label = None
mcmc_par1 = mcmc_sample[:, col_pair[0]]
mcmc_par2 = mcmc_sample[:, col_pair[1]]
abc_par1 = abc_sample[:, col_pair[0]]
abc_par2 = abc_sample[:, col_pair[1]]
ax = plt.subplot(gs[i])
if sigma:
lvls = [1 - np.exp(-0.5), 1 - np.exp(-0.125)]
else:
lvls = [0.68, 0.95]
corner.hist2d(mcmc_par1,
mcmc_par2,
bins=20,
range=plot_range,
ax=ax,
plot_datapoints=False,
levels=lvls,
color='#1F77B4',
fill_contours=True,
smooth=1.0,
label=mcmc_label)
corner.hist2d(abc_par1,
abc_par2,
bins=20,
range=plot_range,
ax=ax,
levels=lvls,
color='#FF7F0E',
fill_contours=True,
smooth=1.0,
label=abc_label)
ax.scatter(np.repeat(truths[col_pair[0]], 2),
np.repeat(truths[col_pair[1]], 2),
s=100,
marker='*',
c='k',
lw=0,
label=None)
#ax.axvline(truths[i_col], color='k', ls='--', linewidth=3)
#ax.set_xticklabels([])
ax.set_xlim([plot_range[0, 0], plot_range[0, 1]])
ax.set_ylim([plot_range[1, 0], plot_range[1, 1]])
if i == 2:
thick_line1 = mlines.Line2D([], [],
ls='-',
c='#FF7F0E',
linewidth=12,
alpha=0.5,
label='ABC-PMC')
ax.legend(loc='upper right',
handles=[thick_line1],
frameon=False,
fontsize=25,
handletextpad=0.1,
scatteryoffsets=[0.5])
elif i == 1:
thick_line2 = mlines.Line2D(
[], [],
ls='-',
c='#1F77B4',
linewidth=12,
alpha=0.5,
label='$\mathcal{L}^\mathtt{Gauss}$ \nMCMC')
ax.legend(loc='upper right',
handles=[thick_line2],
frameon=False,
fontsize=25,
handletextpad=0.1,
scatteryoffsets=[0.5])
ax.set_xlabel(par_labels[col_pair[0]], fontsize=25, labelpad=15)
ax.set_ylabel(par_labels[col_pair[1]], fontsize=25)
if sigma:
sigma_str = '.true1sigma'
else:
sigma_str = ''
fig.subplots_adjust(wspace=0.3)
fig_name = ''.join([
ut.fig_dir(), 'paper.ABCvsMCMC.contour', '.', obvs, sigma_str, '.pdf'
])
fig.savefig(fig_name, bbox_inches='tight', dpi=150)
return None
for obv in observables:
if obv == 'nbar':
data_nbar, data_nbar_var = Data.data_nbar(**data_dict)
fake_obs.append(data_nbar)
if obv == 'gmf':
data_gmf, data_gmf_sigma = Data.data_gmf(**data_dict)
fake_obs.append(data_gmf)
if obv == 'xi':
# import xir and full covariance matrix of xir
data_xi, data_xi_cov = Data.data_xi_full_cov(**data_dict)
# take the inverse of the covariance matrix
data_xi_invcov = solve(np.eye(len(data_xi)) , data_xi_cov)
fake_obs.append(data_xi)
# 'True' HOD parameters
data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr'])
data_hod = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
Ndim = len(data_hod)
# Priors
prior_min, prior_max = PriorRange(prior_name)
prior_range = np.zeros((len(prior_min),2))
prior_range[:,0] = prior_min
prior_range[:,1] = prior_max
def mcmc_ipython_par(Nwalkers,
Nchains_burn,
Nchains_pro,
observables=['nbar', 'xi'],
data_dict={
'Mr': 20,
'Nmock': 500
},
prior_name='first_try',
threads=1):
'''
Standard MCMC implementaion
Parameters
-----------
- Nwalker :
Number of walkers
- Nchains_burn :
Number of burn-in chains
- Nchains_pro :
Number of production chains
- observables :
list of observables. Options are 'nbar', 'gmf', 'xi'
- data_dict : dictionary that specifies the observation keywords
'''
# data observables
fake_obs = [] # list of observables
fake_obs_cov = []
for obv in observables:
if obv == 'nbar':
data_nbar, data_nbar_var = Data.data_nbar(**data_dict)
fake_obs.append(data_nbar)
fake_obs_cov.append(data_nbar_var)
if obv == 'gmf':
data_gmf, data_gmf_sigma = Data.data_gmf(**data_dict)
fake_obs.append(data_gmf)
fake_obs_cov.append(data_gmf)
if obv == 'xi':
# import xir and full covariance matrix of xir
data_xi, data_xi_cov = Data.data_xi_full_cov(**data_dict)
data_xi_invcov = Data.data_xi_inv_cov(**data_dict)
fake_obs.append(data_xi)
fake_obs_cov.append(data_xi_invcov)
# True HOD parameters
data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr'])
data_hod = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
Ndim = len(data_hod)
# Priors
prior_min, prior_max = PriorRange(prior_name)
prior_range = np.zeros((len(prior_min), 2))
prior_range[:, 0] = prior_min
prior_range[:, 1] = prior_max
# Initializing Walkers
random_guess = np.array([11., np.log(.4), 11.5, 1.0, 13.5])
pos0 = np.repeat(random_guess, Nwalkers).reshape(Ndim, Nwalkers).T + \
1e-1 * np.random.randn(Ndim * Nwalkers).reshape(Nwalkers, Ndim)
# Initializing the emcee sampler
hod_kwargs = {
'prior_range': prior_range,
'data': fake_obs,
'data_cov': fake_obs_cov,
'observables': observables,
'Mr': data_dict['Mr']
}
# Set up the interface to the ipcluster.
c = Client()
view = c[:]
view.push({"lnPost": lnPost})
# Modules necessary in posterior calculation should be called here
view.execute("import numpy as np")
view.execute("from hod_sim import HODsimulator")
# Setting up the Sampler
sampler = emcee.EnsembleSampler(Nwalkers,
Ndim,
lnPost,
kwargs=hod_kwargs,
pool=view)
# Setting up a file for saving the chains
chain_file = ''.join([
util.dat_dir(),
util.observable_id_flag(observables), '_Mr',
str(data_dict["Mr"]), '_theta.mcmc_chain.dat'
])
f = open(chain_file, "w")
f.close()
# Running the Sampler and writing out the chains
for result in sampler.sample(pos0,
iterations=Nchains_burn + Nchains_pro,
storechain=False):
position = result[0]
f = open(chain_file, "a")
for k in range(position.shape[0]):
output_str = '\t'.join(position[k].astype('str')) + '\n'
f.write(output_str)
f.close()
def ABCvsMCMC_contour(obvs, nwalkers=100, nburns=9000, sigma=False):
''' Plots that compare the ABC posteriors to the MCMC posteriors
'''
if obvs == 'nbargmf':
abc_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/run1/',])
abc_theta_file = lambda tt: ''.join([abc_dir, 'nbar_gmf_theta_t', str(tt), '.ABCnbargmf.dat'])
tf = 8
mcmc_dir = ''.join([ut.dat_dir(), 'paper/'])
mcmc_filename = ''.join([mcmc_dir, 'nbar_gmf.mcmc.mcmc_chain.p'])
elif obvs == 'nbarxi':
abc_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/',])
abc_theta_file = lambda tt: ''.join([abc_dir, 'nbar_xi_theta_t', str(tt), '.abc.dat'])
tf = 9
mcmc_dir = ''.join([ut.dat_dir(), 'paper/'])
mcmc_filename = ''.join([mcmc_dir, 'nbar_xi.mcmc.mcmc_chain.p'])
else:
raise ValueError
prior_min, prior_max = PriorRange('first_try')
prior_range = np.zeros((len(prior_min),2))
prior_range[:,0] = prior_min
prior_range[:,1] = prior_max
# true HOD parameter
true_dict = Data.data_hod_param(Mr=21)
truths = [
true_dict['logM0'], # log M0
np.log(true_dict['sigma_logM']), # log(sigma)
true_dict['logMmin'], # log Mmin
true_dict['alpha'], # alpha
true_dict['logM1'] # log M1
]
mcmc_sample = pickle.load(open(mcmc_filename, 'rb'))[nburns*nwalkers:,:]
abc_sample = np.loadtxt(abc_theta_file(tf))
par_labels = [
r'$\mathtt{log}\;\mathcal{M}_{0}$',
r'$\mathtt{log}\;\sigma_\mathtt{log\;M}$',
r'$\mathtt{log}\;\mathcal{M}_\mathtt{min}$',
r'$\alpha$',
r'$\mathtt{log}\;\mathcal{M}_{1}$'
]
prettyplot()
pretty_colors = prettycolors()
fig = plt.figure(1, figsize=(20,6))
gs = gridspec.GridSpec(1, 3)
# first panel
for i in [0, 1, 2]:
plot_range = np.zeros((2,2))
if i == 0:
col_pair = [2, 3]
plot_range[0, 0] = 12.5
plot_range[0, 1] = 13.0
plot_range[1, 0] = prior_range[3, 0]
plot_range[1, 1] = prior_range[3, 1]
elif i == 2:
col_pair = [4, 2]
plot_range[0, 0] = 13.6
plot_range[0, 1] = 14.2
plot_range[1, 0] = 12.5
plot_range[1, 1] = 13.0
elif i == 1:
col_pair = [3, 4]
plot_range[0, 0] = prior_range[3, 0]
plot_range[0, 1] = prior_range[3, 1]
plot_range[1, 0] = 13.6
plot_range[1, 1] = 14.2
if i == 2:
mcmc_label = r'$\mathcal{L}^\mathtt{Gauss}$ MCMC'
abc_label = 'ABC-PMC'
else:
mcmc_label = None
abc_label = None
mcmc_par1 = mcmc_sample[:,col_pair[0]]
mcmc_par2 = mcmc_sample[:,col_pair[1]]
abc_par1 = abc_sample[:, col_pair[0]]
abc_par2 = abc_sample[:, col_pair[1]]
ax = plt.subplot(gs[i])
if sigma:
lvls = [1-np.exp(-0.5), 1-np.exp(-0.125)]
else:
lvls = [0.68, 0.95]
corner.hist2d(mcmc_par1, mcmc_par2, bins=20, range=plot_range, ax = ax, plot_datapoints=False,
levels=lvls, color='#1F77B4', fill_contours=True, smooth=1.0, label=mcmc_label)
corner.hist2d(abc_par1, abc_par2, bins=20, range=plot_range, ax = ax,
levels=lvls, color='#FF7F0E', fill_contours=True, smooth=1.0, label=abc_label)
ax.scatter(np.repeat(truths[col_pair[0]],2), np.repeat(truths[col_pair[1]],2),
s=100, marker='*', c='k', lw=0, label=None)
#ax.axvline(truths[i_col], color='k', ls='--', linewidth=3)
#ax.set_xticklabels([])
ax.set_xlim([plot_range[0, 0] , plot_range[0, 1]])
ax.set_ylim([plot_range[1, 0] , plot_range[1, 1]])
if i == 2:
thick_line1 = mlines.Line2D([], [], ls='-', c='#FF7F0E', linewidth=12, alpha=0.5,
label='ABC-PMC')
ax.legend(loc='upper right', handles=[thick_line1],
frameon=False, fontsize=25, handletextpad=0.1, scatteryoffsets=[0.5])
elif i == 1:
thick_line2 = mlines.Line2D([], [], ls='-', c='#1F77B4', linewidth=12, alpha=0.5,
label='$\mathcal{L}^\mathtt{Gauss}$ \nMCMC')
ax.legend(loc='upper right', handles=[thick_line2],
frameon=False, fontsize=25, handletextpad=0.1, scatteryoffsets=[0.5])
ax.set_xlabel(par_labels[col_pair[0]], fontsize=25, labelpad=15)
ax.set_ylabel(par_labels[col_pair[1]], fontsize=25)
if sigma:
sigma_str = '.true1sigma'
else:
sigma_str = ''
fig.subplots_adjust(wspace=0.3)
fig_name = ''.join([ut.fig_dir(),
'paper.ABCvsMCMC.contour',
'.', obvs,
sigma_str,
'.pdf'])
fig.savefig(fig_name, bbox_inches='tight', dpi=150)
return None
def PoolEvolution(obvs):
''' Demostrative plot for the evolution of the pool. Illustrate the pool evolution for
log M_min versus log M_1, which has the starkest evolution from its prior.
'''
if obvs == 'nbargmf':
result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/run1/',])
theta_file = lambda tt: ''.join([result_dir, 'nbar_gmf_theta_t', str(tt), '.ABCnbargmf.dat'])
t_list = [0, 1, 2, 3, 5, 8]
elif obvs == 'nbarxi':
result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/'])
theta_file = lambda tt: ''.join([result_dir, 'nbar_xi_theta_t', str(tt), '.abc.dat'])
t_list = [0, 1, 2, 3, 7, 9]
else:
raise ValueError
prior_min, prior_max = PriorRange('first_try')
prior_range = np.zeros((2,2))
prior_range[:,0] = np.array([prior_min[2], prior_min[4]])
prior_range[:,1] = np.array([prior_max[2], prior_max[4]])
# true HOD parameter
true_dict = Data.data_hod_param(Mr=21)
true_pair = [true_dict['logMmin'], true_dict['logM1']]
prettyplot()
pretty_colors = prettycolors()
fig = plt.figure(figsize=(12,8))
all_fig = fig.add_subplot(111)
for i_t, t in enumerate(t_list):
sub = fig.add_subplot(2, len(t_list)/2, i_t+1)
theta_Mmin, theta_M1 = np.loadtxt(theta_file(t), unpack=True, usecols=[2, 4])
corner.hist2d(theta_Mmin, theta_M1, bins=20, range=prior_range,
levels=[0.68, 0.95], color='c', fill_contours=True, smooth=1.0)
t_label = r"$\mathtt{t = "+str(t)+"}$"
sub.text(13.0, 15.0, t_label, fontsize=25)
if i_t == len(t_list) - 1:
true_label = r'$``\mathtt{true}"$'
else:
true_label = None
plt.scatter(np.repeat(true_pair[0],2), np.repeat(true_pair[1],2),
s=75, marker='*', c='k', lw=0, label=true_label)
if i_t == len(t_list) - 1:
plt.legend(loc='lower left', scatterpoints=1, markerscale=2.5,
handletextpad=-0.25, scatteryoffsets=[0.5])
if i_t == 0:
sub.set_xticklabels([])
sub.set_yticklabels([13., 13.5, 14., 14.5, 15., 15.5])
elif (i_t > 0) and (i_t < len(t_list)/2):
sub.set_xticklabels([])
sub.set_yticklabels([])
elif i_t == len(t_list)/2:
sub.set_yticklabels([13., 13.5, 14., 14.5, 15.])
elif i_t > len(t_list)/2:
sub.set_yticklabels([])
all_fig.set_xticklabels([])
all_fig.set_yticklabels([])
all_fig.set_ylabel(
r'$\mathtt{log}\;\mathcal{M}_\mathtt{1}$',
fontsize=30, labelpad=50)
all_fig.set_xlabel(
r'$\mathtt{log}\;\mathcal{M}_\mathtt{min}$',
fontsize=30, labelpad=25)
fig.subplots_adjust(hspace=0.0)
fig_name = ''.join([ut.fig_dir(),
'paper_ABC_poolevolution',
'.', obvs,
'.pdf'])
fig.savefig(fig_name, bbox_inches='tight', dpi=150)
plt.close()
def ABCvsMCMC_histogram(obvs, nwalkers=100, nburns=9000):
''' Plots that compare the ABC posteriors to the MCMC posteriors
'''
if obvs == 'nbargmf':
abc_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/run1/',])
abc_theta_file = lambda tt: ''.join([abc_dir, 'nbar_gmf_theta_t', str(tt), '.ABCnbargmf.dat'])
tf = 8
mcmc_dir = ''.join([ut.dat_dir(), 'paper/'])
mcmc_filename = ''.join([mcmc_dir, 'nbar_gmf.mcmc.mcmc_chain.p'])
elif obvs == 'nbarxi':
abc_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/',])
abc_theta_file = lambda tt: ''.join([abc_dir, 'nbar_xi_theta_t', str(tt), '.abc.dat'])
tf = 9
mcmc_dir = ''.join([ut.dat_dir(), 'paper/'])
mcmc_filename = ''.join([mcmc_dir, 'nbar_xi.mcmc.mcmc_chain.p'])
else:
raise ValueError
prior_min, prior_max = PriorRange('first_try')
prior_range = np.zeros((len(prior_min),2))
prior_range[:,0] = prior_min
prior_range[:,1] = prior_max
# true HOD parameter
true_dict = Data.data_hod_param(Mr=21)
truths = [
true_dict['logM0'], # log M0
np.log(true_dict['sigma_logM']), # log(sigma)
true_dict['logMmin'], # log Mmin
true_dict['alpha'], # alpha
true_dict['logM1'] # log M1
]
mcmc_sample = pickle.load(open(mcmc_filename, 'rb'))[nburns*nwalkers:,:]
abc_sample = np.loadtxt(abc_theta_file(tf))
normie = norm()
sig1lo = normie.cdf(-1)
sig1hi = normie.cdf(1)
sig2lo = normie.cdf(-2)
sig2hi = normie.cdf(2)
nsamples_mcmc = len(mcmc_sample[:, 2])
nsamples_abc = len(abc_sample[:, 2])
sig1lo_mcmc = int(sig1lo * nsamples_mcmc)
sig2lo_mcmc = int(sig2lo * nsamples_mcmc)
sig1hi_mcmc = int(sig1hi * nsamples_mcmc)
sig2hi_mcmc = int(sig2hi * nsamples_mcmc)
sig1lo_abc = int(sig1lo * nsamples_abc)
sig2lo_abc = int(sig2lo * nsamples_abc)
sig1hi_abc = int(sig1hi * nsamples_abc)
sig2hi_abc = int(sig2hi * nsamples_abc)
par_labels = [
r'$\mathtt{log}\;\mathcal{M}_{0}$',
r'$\mathtt{log}\;\sigma_\mathtt{log\;M}$',
r'$\mathtt{log}\;\mathcal{M}_\mathtt{min}$',
r'$\alpha$',
r'$\mathtt{log}\;\mathcal{M}_{1}$'
]
prettyplot()
pretty_colors = prettycolors()
fig = plt.figure(1, figsize=(20,8))
gs = gridspec.GridSpec(2, 3, height_ratios=[2.75, 1])
# first panel
for i in [0, 1, 2]:
if i == 0:
i_col = 2
prior_range[i_col,0] = 12.5
prior_range[i_col,1] = 13.
plot_range = prior_range[i_col, :]
elif i == 1:
i_col = 3
plot_range = prior_range[i_col, :]
elif i == 2:
i_col = 4
prior_range[i_col,0] = 13.5
prior_range[i_col,1] = 14.25
plot_range = np.array([13.5, 14.5])
ax = plt.subplot(gs[i])
q = ax.hist(mcmc_sample[:,i_col], bins=20,
range=[prior_range[i_col,0], prior_range[i_col,1]],
normed=True,
alpha=0.75,
color=pretty_colors[1],
linewidth=2,
histtype='stepfilled',
edgecolor=None
)
qq = ax.hist(abc_sample[:,i_col], bins=20,
range=[prior_range[i_col,0], prior_range[i_col,1]],
normed=True,
alpha=0.75,
color=pretty_colors[3],
linewidth=2,
histtype='stepfilled',
edgecolor=None
)
ax.axvline(truths[i_col], color='k', ls='--', linewidth=3)
ax.set_xticklabels([])
ax.set_xlim([plot_range[0] , plot_range[1]])
if i == 2:
thick_line2 = mlines.Line2D([], [], ls='-', c=pretty_colors[1], linewidth=4,
label='$\mathcal{L}^\mathtt{Gauss}$ \nMCMC')
thick_line1 = mlines.Line2D([], [], ls='-', c=pretty_colors[3], linewidth=4,
label='ABC-PMC')
ax.legend(loc='upper right', handles=[thick_line2, thick_line1],
frameon=False, fontsize=25, handletextpad=-0.0)
# general box properties
boxprops = {'color': 'k'}
medianprops = {'alpha': 0.}
bplots1 = []
ax = plt.subplot(gs[i+3])
# stats dict for each box
bplots1.append({'med': np.median(mcmc_sample[:, i_col]),
'q1': np.sort(mcmc_sample[:, i_col])[sig1lo_mcmc],
'q3': np.sort(mcmc_sample[:, i_col])[sig1hi_mcmc],
'whislo': np.sort(mcmc_sample[:, i_col])[sig2lo_mcmc],
'whishi': np.sort(mcmc_sample[:, i_col])[sig2hi_mcmc],
'fliers': []})
bplots1.append({'med': np.median(abc_sample[:, i_col]),
'q1': np.sort(abc_sample[:, i_col])[sig1lo_abc],
'q3': np.sort(abc_sample[:, i_col])[sig1hi_abc],
'whislo': np.sort(abc_sample[:, i_col])[sig2lo_abc],
'whishi': np.sort(abc_sample[:, i_col])[sig2hi_abc],
'fliers': []})
whiskprop = dict(linestyle='-', linewidth=2, color='k')
boxprops = dict(linestyle='-', linewidth=2, color='k')
bxp1 = ax.bxp(bplots1, positions=[1,2], vert=False, patch_artist=True,
showfliers=False, boxprops=boxprops, medianprops=medianprops, whiskerprops=whiskprop)
for ibox, box in enumerate(bxp1['boxes']):
if ibox == 0:
box.set(facecolor=pretty_colors[1], alpha=0.75)
elif ibox == 1:
box.set(facecolor=pretty_colors[3], alpha=0.75)
ax.axvline(truths[i_col], color='k', ls='--', linewidth=3)
ax.set_xlim([plot_range[0] , plot_range[1]])
ax.set_xlabel(par_labels[i_col], fontsize=25, labelpad=15)
if i == 0:
ax.set_yticks([1,2])
ax.set_yticklabels([r"$\mathtt{MCMC}$", r"$\mathtt{ABC}$"])
else:
ax.set_yticks([])
fig.subplots_adjust(wspace=0.2, hspace=0.0)
fig_name = ''.join([ut.fig_dir(),
'paper.ABCvsMCMC',
'.', obvs,
'.pdf'])
print fig_name
fig.savefig(fig_name, bbox_inches='tight', dpi=150)
return None
def ABC_Convergence(weighted=False):
''' Plot the error bars on the parameters as a function of time step
'''
prettyplot()
pretty_colors = prettycolors()
fig = plt.figure(figsize=(20, 10))
for i_obv, obvs in enumerate(['nbargmf', 'nbarxi']):
if obvs == 'nbargmf':
result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/run1/',])
theta_file = lambda tt: ''.join([result_dir, 'nbar_gmf_theta_t', str(tt), '.ABCnbargmf.dat'])
w_file = lambda tt: ''.join([result_dir, 'nbar_gmf_w_t', str(tt), '.ABCnbargmf.dat'])
tf = 8
elif obvs == 'nbarxi':
result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/'])
theta_file = lambda tt: ''.join([result_dir, 'nbar_xi_theta_t', str(tt), '.abc.dat'])
w_file = lambda tt: ''.join([result_dir, 'nbar_xi_w_t', str(tt), '.abc.dat'])
tf = 9
else:
raise ValueError
t_list = range(tf+1)
columns = [
r"$\mathtt{log}\;\mathcal{M}_0$",
r"$\sigma_{\mathtt{log}\;\mathcal{M}}$",
r"$\mathtt{log}\;\mathcal{M}_\mathtt{min}$",
r"$\alpha$",
r"$\mathtt{log}\;\mathcal{M}_1$"
]
true_dict = Data.data_hod_param(Mr=21)
true_theta = [
true_dict['logM0'],
np.log(true_dict['sigma_logM']),
true_dict['logMmin'],
true_dict['alpha'],
true_dict['logM1']
]
prior_min, prior_max = PriorRange('first_try')
a_theta = np.zeros((len(t_list), 5))
b_theta = np.zeros((len(t_list), 5))
d_theta = np.zeros((len(t_list), 5))
e_theta = np.zeros((len(t_list), 5))
for i_t, tt in enumerate(t_list):
theta_i = np.loadtxt(theta_file(tt), unpack=True)
w_i = np.loadtxt(w_file(tt))
for i_par in range(len(theta_i)):
if not weighted:
a, b, d, e = np.percentile(theta_i[i_par], [2.5, 16, 84, 97.5], axis=0)
else:
a = ut.quantile_1D(theta_i[i_par], w_i, 0.025)
b = ut.quantile_1D(theta_i[i_par], w_i, 0.16)
d = ut.quantile_1D(theta_i[i_par], w_i, 0.84)
e = ut.quantile_1D(theta_i[i_par], w_i, 0.975)
a_theta[i_t, i_par] = a
b_theta[i_t, i_par] = b
d_theta[i_t, i_par] = d
e_theta[i_t, i_par] = e
keep_index = [2,3,4]
for ii, i in enumerate(keep_index):
if i == keep_index[-1]:
true_label = r'$``\mathtt{true}"$'
abc_1sig_label = r'ABC Pool'
else:
true_label = None
abc_1sig_label = None
sub = fig.add_subplot(2, len(keep_index), i_obv * len(keep_index) + ii+1)
sub.fill_between(t_list, a_theta[:, i], e_theta[:,i],
color=pretty_colors[3], alpha=0.3, edgecolor="none")
sub.fill_between(t_list, b_theta[:, i], d_theta[:,i],
color=pretty_colors[3], alpha=0.5, edgecolor="none",
label=abc_1sig_label)
sub.plot(t_list, np.repeat(true_theta[i], len(t_list)), c='k', ls='--', lw=2,
label=true_label)
if ii == 0:
if obvs == 'nbargmf':
sub.text(4.85, 13.4, r"$\bar{\mathtt{n}}$ and $\zeta(\mathtt{N})$", fontsize=25)
elif obvs == 'nbarxi':
sub.text(4.85, 13.4, r"$\bar{\mathtt{n}}$ and $\xi(\mathtt{r})$", fontsize=25)
sub.set_ylabel(columns[i], fontsize=25)
sub.set_ylim([prior_min[i], prior_max[i]])
sub.set_xlim([-1, 10])
if i_obv == 1:
sub.legend(loc='upper right', borderpad=1.)
sub.set_xlabel('iterations', fontsize=25)
if i == 4:
sub.set_yticklabels([13.0, 13.5, 14.0, 14.5, 15.])
else:
sub.set_xticklabels([])
fig.subplots_adjust(wspace=0.3, hspace=0.0)
if weighted:
weight_str = '.weighted'
else:
weight_str = ''
fig_name = ''.join([ut.fig_dir(),
'paper',
'.ABCconvergence',
weight_str,
'.pdf'])
fig.savefig(fig_name, bbox_inches='tight', dpi=150)
plt.close()
return None
def ABC_Coner(obvs, weighted=False):
''' Pretty corner plot
'''
if obvs == 'nbargmf':
result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/run1/',])
theta_file = lambda tt: ''.join([result_dir, 'nbar_gmf_theta_t', str(tt), '.ABCnbargmf.dat'])
w_file = lambda tt: ''.join([result_dir, 'nbar_gmf_w_t', str(tt), '.ABCnbargmf.dat'])
tf = 8
elif obvs == 'nbarxi':
result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/'])
theta_file = lambda tt: ''.join([result_dir, 'nbar_xi_theta_t', str(tt), '.abc.dat'])
w_file = lambda tt: ''.join([result_dir, 'nbar_xi_w_t', str(tt), '.abc.dat'])
tf = 9
else:
raise ValueError
theta = np.loadtxt(theta_file(tf))
if weighted:
weights = np.loadtxt(w_file(tf))
else:
weights = None
true_dict = Data.data_hod_param(Mr=21)
true_theta = [
true_dict['logM0'],
np.log(true_dict['sigma_logM']),
true_dict['logMmin'],
true_dict['alpha'],
true_dict['logM1']
]
prior_min, prior_max = PriorRange('first_try')
prior_range = np.zeros((len(prior_min),2))
prior_range[:,0] = np.array(prior_min)
prior_range[:,1] = np.array(prior_max)
fig = corner.corner(
theta,
weights=weights,
truths=true_theta,
truth_color='k',
labels=[
r'$\log\;\mathcal{M}_{0}}$',
r'$\log\;\sigma_\mathtt{log\;M}}$',
r'$\log\;\mathcal{M}_\mathtt{min}}$',
r'$\alpha$',
r'$\log\;\mathcal{M}_{1}}$'
],
label_kwargs={'fontsize': 25},
range=prior_range,
quantiles=[0.16,0.5,0.84],
show_titles=True,
title_args={"fontsize": 12},
plot_datapoints=True,
fill_contours=True,
levels=[0.68, 0.95],
color='#ee6a50',
bins=20,
smooth=1.0)
fig_name = ''.join([ut.fig_dir(),
'paper.ABCcorner',
'.', obvs,
'.pdf'])
fig.savefig(fig_name, bbox_inches='tight', dpi=150)
plt.close()
return None
def ABC_Convergence(weighted=False):
''' Plot the error bars on the parameters as a function of time step
'''
prettyplot()
pretty_colors = prettycolors()
fig = plt.figure(figsize=(20, 10))
for i_obv, obvs in enumerate(['nbargmf', 'nbarxi']):
if obvs == 'nbargmf':
result_dir = ''.join([
ut.dat_dir(),
'paper/ABC',
obvs,
'/run1/',
])
theta_file = lambda tt: ''.join(
[result_dir, 'nbar_gmf_theta_t',
str(tt), '.ABCnbargmf.dat'])
w_file = lambda tt: ''.join(
[result_dir, 'nbar_gmf_w_t',
str(tt), '.ABCnbargmf.dat'])
tf = 8
elif obvs == 'nbarxi':
result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/'])
theta_file = lambda tt: ''.join(
[result_dir, 'nbar_xi_theta_t',
str(tt), '.abc.dat'])
w_file = lambda tt: ''.join(
[result_dir, 'nbar_xi_w_t',
str(tt), '.abc.dat'])
tf = 9
else:
raise ValueError
t_list = range(tf + 1)
columns = [
r"$\mathtt{log}\;\mathcal{M}_0$",
r"$\sigma_{\mathtt{log}\;\mathcal{M}}$",
r"$\mathtt{log}\;\mathcal{M}_\mathtt{min}$", r"$\alpha$",
r"$\mathtt{log}\;\mathcal{M}_1$"
]
true_dict = Data.data_hod_param(Mr=21)
true_theta = [
true_dict['logM0'],
np.log(true_dict['sigma_logM']), true_dict['logMmin'],
true_dict['alpha'], true_dict['logM1']
]
prior_min, prior_max = PriorRange('first_try')
a_theta = np.zeros((len(t_list), 5))
b_theta = np.zeros((len(t_list), 5))
d_theta = np.zeros((len(t_list), 5))
e_theta = np.zeros((len(t_list), 5))
for i_t, tt in enumerate(t_list):
theta_i = np.loadtxt(theta_file(tt), unpack=True)
w_i = np.loadtxt(w_file(tt))
for i_par in range(len(theta_i)):
if not weighted:
a, b, d, e = np.percentile(theta_i[i_par],
[2.5, 16, 84, 97.5],
axis=0)
else:
a = ut.quantile_1D(theta_i[i_par], w_i, 0.025)
b = ut.quantile_1D(theta_i[i_par], w_i, 0.16)
d = ut.quantile_1D(theta_i[i_par], w_i, 0.84)
e = ut.quantile_1D(theta_i[i_par], w_i, 0.975)
a_theta[i_t, i_par] = a
b_theta[i_t, i_par] = b
d_theta[i_t, i_par] = d
e_theta[i_t, i_par] = e
keep_index = [2, 3, 4]
for ii, i in enumerate(keep_index):
if i == keep_index[-1]:
true_label = r'$``\mathtt{true}"$'
abc_1sig_label = r'ABC Pool'
else:
true_label = None
abc_1sig_label = None
sub = fig.add_subplot(2, len(keep_index),
i_obv * len(keep_index) + ii + 1)
sub.fill_between(t_list,
a_theta[:, i],
e_theta[:, i],
color=pretty_colors[3],
alpha=0.3,
edgecolor="none")
sub.fill_between(t_list,
b_theta[:, i],
d_theta[:, i],
color=pretty_colors[3],
alpha=0.5,
edgecolor="none",
label=abc_1sig_label)
sub.plot(t_list,
np.repeat(true_theta[i], len(t_list)),
c='k',
ls='--',
lw=2,
label=true_label)
if ii == 0:
if obvs == 'nbargmf':
sub.text(4.85,
13.4,
r"$\bar{\mathtt{n}}$ and $\zeta(\mathtt{N})$",
fontsize=25)
elif obvs == 'nbarxi':
sub.text(4.85,
13.4,
r"$\bar{\mathtt{n}}$ and $\xi(\mathtt{r})$",
fontsize=25)
sub.set_ylabel(columns[i], fontsize=25)
sub.set_ylim([prior_min[i], prior_max[i]])
sub.set_xlim([-1, 10])
if i_obv == 1:
sub.legend(loc='upper right', borderpad=1.)
sub.set_xlabel('iterations', fontsize=25)
if i == 4:
sub.set_yticklabels([13.0, 13.5, 14.0, 14.5, 15.])
else:
sub.set_xticklabels([])
fig.subplots_adjust(wspace=0.3, hspace=0.0)
if weighted:
weight_str = '.weighted'
else:
weight_str = ''
fig_name = ''.join(
[ut.fig_dir(), 'paper', '.ABCconvergence', weight_str, '.pdf'])
fig.savefig(fig_name, bbox_inches='tight', dpi=150)
plt.close()
return None
def plot_mcmc(Nwalkers,
Niter=1000,
Nchains_burn=200,
Mr=21,
truths=None,
observables=['nbar', 'xi'],
plot_range=None):
'''
Plot MCMC chains
'''
if truths is None:
data_hod_dict = Data.data_hod_param(Mr=Mr)
truths = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
if plot_range is None:
prior_min, prior_max = PriorRange(None)
plot_range = np.zeros((len(prior_min), 2))
plot_range[:, 0] = prior_min
plot_range[:, 1] = prior_max
# chain files
chain_file = ''.join([
util.dat_dir(),
util.observable_id_flag(observables), '_Mr',
str(Mr), '.mcmc_chain.dat'
])
#f = h5py.File(chain_file, 'r')
#sample = f['positions'][:]
sample = np.loadtxt(chain_file)
# Posterior Likelihood Corner Plot
fig = corner.corner(sample[Nchains_burn * Nwalkers:],
truths=truths,
truth_color='#ee6a50',
labels=[
r'$\mathtt{\log\;M_{0}}$',
r'$\mathtt{\log\;\sigma_{\logM}}$',
r'$\mathtt{\log\;M_{min}}$', r'$\mathtt{\alpha}$',
r'$\mathtt{\log\;M_{1}}$'
],
label_kwargs={'fontsize': 25},
range=plot_range,
quantiles=[0.16, 0.5, 0.84],
show_titles=True,
title_args={"fontsize": 12},
plot_datapoints=True,
fill_contours=True,
levels=[0.68, 0.95],
color='b',
bins=16,
smooth=1.0)
fig_file = ''.join([
util.fig_dir(),
util.observable_id_flag(observables), '_Mr',
str(Mr), '.Niter',
str(Niter), '.Nburn',
str(Nchains_burn), '.mcmc_samples.test.png'
])
#print fig_file
plt.savefig(fig_file)
plt.close()
# MCMC Chain plot
Ndim = len(sample[0])
Nchain = len(sample) / Nwalkers
chain_ensemble = sample.reshape(Nchain, Nwalkers, Ndim)
fig, axes = plt.subplots(5, 1, sharex=True, figsize=(10, 12))
labels = [
r'$\mathtt{\log\;M_{0}}$', r'$\mathtt{\log\;\sigma_{\logM}}$',
r'$\mathtt{\log\;M_{min}}$', r'$\mathtt{\alpha}$',
r'$\mathtt{\log\;M_{1}}$'
]
for i in xrange(5):
axes[i].plot(chain_ensemble[:, :, i], color="k", alpha=0.4)
axes[i].yaxis.set_major_locator(MaxNLocator(5))
axes[i].axhline(truths[i], color="#888888", lw=2)
axes[i].vlines(Nchains_burn,
plot_range[i, 0],
plot_range[i, 1],
colors='#ee6a50',
linewidth=4,
alpha=1)
axes[i].set_ylim([plot_range[i, 0], plot_range[i, 1]])
axes[i].set_xlim(0, 6000)
axes[i].set_ylabel(labels[i], fontsize=25)
axes[4].set_xlabel("Step Number", fontsize=25)
fig.tight_layout(h_pad=0.0)
fig_file = ''.join([
util.fig_dir(),
util.observable_id_flag(observables), '_Mr',
str(Mr), '.Niter',
str(Niter), '.Nburn',
str(Nchains_burn), '.mcmc_time.test.png'
])
plt.savefig(fig_file)
plt.close()
def ABCvsMCMC_histogram(obvs, nwalkers=100, nburns=9000):
''' Plots that compare the ABC posteriors to the MCMC posteriors
'''
if obvs == 'nbargmf':
abc_dir = ''.join([
ut.dat_dir(),
'paper/ABC',
obvs,
'/run1/',
])
abc_theta_file = lambda tt: ''.join(
[abc_dir, 'nbar_gmf_theta_t',
str(tt), '.ABCnbargmf.dat'])
tf = 8
mcmc_dir = ''.join([ut.dat_dir(), 'paper/'])
mcmc_filename = ''.join([mcmc_dir, 'nbar_gmf.mcmc.mcmc_chain.p'])
elif obvs == 'nbarxi':
abc_dir = ''.join([
ut.dat_dir(),
'paper/ABC',
obvs,
'/',
])
abc_theta_file = lambda tt: ''.join(
[abc_dir, 'nbar_xi_theta_t',
str(tt), '.abc.dat'])
tf = 9
mcmc_dir = ''.join([ut.dat_dir(), 'paper/'])
mcmc_filename = ''.join([mcmc_dir, 'nbar_xi.mcmc.mcmc_chain.p'])
else:
raise ValueError
prior_min, prior_max = PriorRange('first_try')
prior_range = np.zeros((len(prior_min), 2))
prior_range[:, 0] = prior_min
prior_range[:, 1] = prior_max
# true HOD parameter
true_dict = Data.data_hod_param(Mr=21)
truths = [
true_dict['logM0'], # log M0
np.log(true_dict['sigma_logM']), # log(sigma)
true_dict['logMmin'], # log Mmin
true_dict['alpha'], # alpha
true_dict['logM1'] # log M1
]
mcmc_sample = pickle.load(open(mcmc_filename, 'rb'))[nburns * nwalkers:, :]
abc_sample = np.loadtxt(abc_theta_file(tf))
normie = norm()
sig1lo = normie.cdf(-1)
sig1hi = normie.cdf(1)
sig2lo = normie.cdf(-2)
sig2hi = normie.cdf(2)
nsamples_mcmc = len(mcmc_sample[:, 2])
nsamples_abc = len(abc_sample[:, 2])
sig1lo_mcmc = int(sig1lo * nsamples_mcmc)
sig2lo_mcmc = int(sig2lo * nsamples_mcmc)
sig1hi_mcmc = int(sig1hi * nsamples_mcmc)
sig2hi_mcmc = int(sig2hi * nsamples_mcmc)
sig1lo_abc = int(sig1lo * nsamples_abc)
sig2lo_abc = int(sig2lo * nsamples_abc)
sig1hi_abc = int(sig1hi * nsamples_abc)
sig2hi_abc = int(sig2hi * nsamples_abc)
par_labels = [
r'$\mathtt{log}\;\mathcal{M}_{0}$',
r'$\mathtt{log}\;\sigma_\mathtt{log\;M}$',
r'$\mathtt{log}\;\mathcal{M}_\mathtt{min}$', r'$\alpha$',
r'$\mathtt{log}\;\mathcal{M}_{1}$'
]
prettyplot()
pretty_colors = prettycolors()
fig = plt.figure(1, figsize=(20, 8))
gs = gridspec.GridSpec(2, 3, height_ratios=[2.75, 1])
# first panel
for i in [0, 1, 2]:
if i == 0:
i_col = 2
prior_range[i_col, 0] = 12.5
prior_range[i_col, 1] = 13.
plot_range = prior_range[i_col, :]
elif i == 1:
i_col = 3
plot_range = prior_range[i_col, :]
elif i == 2:
i_col = 4
prior_range[i_col, 0] = 13.5
prior_range[i_col, 1] = 14.25
plot_range = np.array([13.5, 14.5])
ax = plt.subplot(gs[i])
q = ax.hist(mcmc_sample[:, i_col],
bins=20,
range=[prior_range[i_col, 0], prior_range[i_col, 1]],
normed=True,
alpha=0.75,
color=pretty_colors[1],
linewidth=2,
histtype='stepfilled',
edgecolor=None)
qq = ax.hist(abc_sample[:, i_col],
bins=20,
range=[prior_range[i_col, 0], prior_range[i_col, 1]],
normed=True,
alpha=0.75,
color=pretty_colors[3],
linewidth=2,
histtype='stepfilled',
edgecolor=None)
ax.axvline(truths[i_col], color='k', ls='--', linewidth=3)
ax.set_xticklabels([])
ax.set_xlim([plot_range[0], plot_range[1]])
if i == 2:
thick_line2 = mlines.Line2D(
[], [],
ls='-',
c=pretty_colors[1],
linewidth=4,
label='$\mathcal{L}^\mathtt{Gauss}$ \nMCMC')
thick_line1 = mlines.Line2D([], [],
ls='-',
c=pretty_colors[3],
linewidth=4,
label='ABC-PMC')
ax.legend(loc='upper right',
handles=[thick_line2, thick_line1],
frameon=False,
fontsize=25,
handletextpad=-0.0)
# general box properties
boxprops = {'color': 'k'}
medianprops = {'alpha': 0.}
bplots1 = []
ax = plt.subplot(gs[i + 3])
# stats dict for each box
bplots1.append({
'med': np.median(mcmc_sample[:, i_col]),
'q1': np.sort(mcmc_sample[:, i_col])[sig1lo_mcmc],
'q3': np.sort(mcmc_sample[:, i_col])[sig1hi_mcmc],
'whislo': np.sort(mcmc_sample[:, i_col])[sig2lo_mcmc],
'whishi': np.sort(mcmc_sample[:, i_col])[sig2hi_mcmc],
'fliers': []
})
bplots1.append({
'med': np.median(abc_sample[:, i_col]),
'q1': np.sort(abc_sample[:, i_col])[sig1lo_abc],
'q3': np.sort(abc_sample[:, i_col])[sig1hi_abc],
'whislo': np.sort(abc_sample[:, i_col])[sig2lo_abc],
'whishi': np.sort(abc_sample[:, i_col])[sig2hi_abc],
'fliers': []
})
whiskprop = dict(linestyle='-', linewidth=2, color='k')
boxprops = dict(linestyle='-', linewidth=2, color='k')
bxp1 = ax.bxp(bplots1,
positions=[1, 2],
vert=False,
patch_artist=True,
showfliers=False,
boxprops=boxprops,
medianprops=medianprops,
whiskerprops=whiskprop)
for ibox, box in enumerate(bxp1['boxes']):
if ibox == 0:
box.set(facecolor=pretty_colors[1], alpha=0.75)
elif ibox == 1:
box.set(facecolor=pretty_colors[3], alpha=0.75)
ax.axvline(truths[i_col], color='k', ls='--', linewidth=3)
ax.set_xlim([plot_range[0], plot_range[1]])
ax.set_xlabel(par_labels[i_col], fontsize=25, labelpad=15)
if i == 0:
ax.set_yticks([1, 2])
ax.set_yticklabels([r"$\mathtt{MCMC}$", r"$\mathtt{ABC}$"])
else:
ax.set_yticks([])
fig.subplots_adjust(wspace=0.2, hspace=0.0)
fig_name = ''.join([ut.fig_dir(), 'paper.ABCvsMCMC', '.', obvs, '.pdf'])
print fig_name
fig.savefig(fig_name, bbox_inches='tight', dpi=150)
return None
def mcmc_mpi(
Nwalkers,
Nchains,
observables=["nbar", "xi"],
data_dict={"Mr": 21, "b_normal": 0.25},
prior_name="first_try",
mcmcrun=None,
):
"""
Standard MCMC implementaion
Parameters
-----------
- Nwalker :
Number of walkers
- Nchains :
Number of MCMC chains
- observables :
list of observables. Options are: ['nbar','xi'],['nbar','gmf'],['xi']
- data_dict : dictionary that specifies the observation keywords
"""
# Initializing the vector of observables and inverse covariance matrix
if observables == ["xi"]:
fake_obs = Data.data_xi(**data_dict)
# fake_obs_icov = Data.data_inv_cov('xi', **data_dict)
fake_obs_icov = Data.data_cov(inference="mcmc", **data_dict)[1:16, 1:16]
if observables == ["nbar", "xi"]:
fake_obs = np.hstack([Data.data_nbar(**data_dict), Data.data_xi(**data_dict)])
fake_obs_icov = Data.data_cov(inference="mcmc", **data_dict)[:16, :16]
if observables == ["nbar", "gmf"]:
##### FIRST BIN OF GMF DROPPED ###############
# CAUTION: hardcoded
fake_obs = np.hstack([Data.data_nbar(**data_dict), Data.data_gmf(**data_dict)[1:]])
fake_obs_icov = np.zeros((10, 10))
# print Data.data_cov(**data_dict)[17: , 17:].shape
# Covariance matrix being adjusted accordingly
fake_obs_icov[1:, 1:] = Data.data_cov(inference="mcmc", **data_dict)[17:, 17:]
fake_obs_icov[0, 1:] = Data.data_cov(inference="mcmc", **data_dict)[0, 17:]
fake_obs_icov[1:, 0] = Data.data_cov(inference="mcmc", **data_dict)[17:, 0]
fake_obs_icov[0, 0] = Data.data_cov(inference="mcmc", **data_dict)[0, 0]
# True HOD parameters
data_hod_dict = Data.data_hod_param(Mr=data_dict["Mr"])
data_hod = np.array(
[
data_hod_dict["logM0"], # log M0
np.log(data_hod_dict["sigma_logM"]), # log(sigma)
data_hod_dict["logMmin"], # log Mmin
data_hod_dict["alpha"], # alpha
data_hod_dict["logM1"], # log M1
]
)
Ndim = len(data_hod)
# Priors
prior_min, prior_max = PriorRange(prior_name)
prior_range = np.zeros((len(prior_min), 2))
prior_range[:, 0] = prior_min
prior_range[:, 1] = prior_max
# mcmc chain output file
chain_file = "".join([util.mcmc_dir(), util.observable_id_flag(observables), ".", mcmcrun, ".mcmc_chain.dat"])
# print chain_file
if os.path.isfile(chain_file) and continue_chain:
print "Continuing previous MCMC chain!"
sample = np.loadtxt(chain_file)
Nchain = Niter - (len(sample) / Nwalkers) # Number of chains left to finish
if Nchain > 0:
pass
else:
raise ValueError
print Nchain, " iterations left to finish"
# Initializing Walkers from the end of the chain
pos0 = sample[-Nwalkers:]
else:
# new chain
f = open(chain_file, "w")
f.close()
Nchain = Niter
# Initializing Walkers
random_guess = data_hod
pos0 = np.repeat(random_guess, Nwalkers).reshape(Ndim, Nwalkers).T + 5.0e-2 * np.random.randn(
Ndim * Nwalkers
).reshape(Nwalkers, Ndim)
# print pos0.shape
# Initializing MPIPool
pool = MPIPool()
if not pool.is_master():
pool.wait()
sys.exit(0)
# Initializing the emcee sampler
hod_kwargs = {
"prior_range": prior_range,
"data": fake_obs,
"data_icov": fake_obs_icov,
"observables": observables,
"Mr": data_dict["Mr"],
}
sampler = emcee.EnsembleSampler(Nwalkers, Ndim, lnPost, pool=pool, kwargs=hod_kwargs)
# Initializing Walkers
for result in sampler.sample(pos0, iterations=Nchain, storechain=False):
position = result[0]
# print position
f = open(chain_file, "a")
for k in range(position.shape[0]):
output_str = "\t".join(position[k].astype("str")) + "\n"
f.write(output_str)
f.close()
pool.close()
def PoolEvolution(obvs):
''' Demostrative plot for the evolution of the pool. Illustrate the pool evolution for
log M_min versus log M_1, which has the starkest evolution from its prior.
'''
if obvs == 'nbargmf':
result_dir = ''.join([
ut.dat_dir(),
'paper/ABC',
obvs,
'/run1/',
])
theta_file = lambda tt: ''.join(
[result_dir, 'nbar_gmf_theta_t',
str(tt), '.ABCnbargmf.dat'])
t_list = [0, 1, 2, 3, 5, 8]
elif obvs == 'nbarxi':
result_dir = ''.join([ut.dat_dir(), 'paper/ABC', obvs, '/'])
theta_file = lambda tt: ''.join(
[result_dir, 'nbar_xi_theta_t',
str(tt), '.abc.dat'])
t_list = [0, 1, 2, 3, 7, 9]
else:
raise ValueError
prior_min, prior_max = PriorRange('first_try')
prior_range = np.zeros((2, 2))
prior_range[:, 0] = np.array([prior_min[2], prior_min[4]])
prior_range[:, 1] = np.array([prior_max[2], prior_max[4]])
# true HOD parameter
true_dict = Data.data_hod_param(Mr=21)
true_pair = [true_dict['logMmin'], true_dict['logM1']]
prettyplot()
pretty_colors = prettycolors()
fig = plt.figure(figsize=(12, 8))
all_fig = fig.add_subplot(111)
for i_t, t in enumerate(t_list):
sub = fig.add_subplot(2, len(t_list) / 2, i_t + 1)
theta_Mmin, theta_M1 = np.loadtxt(theta_file(t),
unpack=True,
usecols=[2, 4])
corner.hist2d(theta_Mmin,
theta_M1,
bins=20,
range=prior_range,
levels=[0.68, 0.95],
color='c',
fill_contours=True,
smooth=1.0)
t_label = r"$\mathtt{t = " + str(t) + "}$"
sub.text(13.0, 15.0, t_label, fontsize=25)
if i_t == len(t_list) - 1:
true_label = r'$``\mathtt{true}"$'
else:
true_label = None
plt.scatter(np.repeat(true_pair[0], 2),
np.repeat(true_pair[1], 2),
s=75,
marker='*',
c='k',
lw=0,
label=true_label)
if i_t == len(t_list) - 1:
plt.legend(loc='lower left',
scatterpoints=1,
markerscale=2.5,
handletextpad=-0.25,
scatteryoffsets=[0.5])
if i_t == 0:
sub.set_xticklabels([])
sub.set_yticklabels([13., 13.5, 14., 14.5, 15., 15.5])
elif (i_t > 0) and (i_t < len(t_list) / 2):
sub.set_xticklabels([])
sub.set_yticklabels([])
elif i_t == len(t_list) / 2:
sub.set_yticklabels([13., 13.5, 14., 14.5, 15.])
elif i_t > len(t_list) / 2:
sub.set_yticklabels([])
all_fig.set_xticklabels([])
all_fig.set_yticklabels([])
all_fig.set_ylabel(r'$\mathtt{log}\;\mathcal{M}_\mathtt{1}$',
fontsize=30,
labelpad=50)
all_fig.set_xlabel(r'$\mathtt{log}\;\mathcal{M}_\mathtt{min}$',
fontsize=30,
labelpad=25)
fig.subplots_adjust(hspace=0.0)
fig_name = ''.join(
[ut.fig_dir(), 'paper_ABC_poolevolution', '.', obvs, '.pdf'])
fig.savefig(fig_name, bbox_inches='tight', dpi=150)
plt.close()
def mcmc_ipython_par(Nwalkers, Nchains_burn, Nchains_pro, observables=['nbar', 'xi'],
data_dict={'Mr':20, 'Nmock':500}, prior_name = 'first_try', threads=1):
'''
Standard MCMC implementaion
Parameters
-----------
- Nwalker :
Number of walkers
- Nchains_burn :
Number of burn-in chains
- Nchains_pro :
Number of production chains
- observables :
list of observables. Options are 'nbar', 'gmf', 'xi'
- data_dict : dictionary that specifies the observation keywords
'''
# data observables
fake_obs = [] # list of observables
fake_obs_cov = []
for obv in observables:
if obv == 'nbar':
data_nbar, data_nbar_var = Data.data_nbar(**data_dict)
fake_obs.append(data_nbar)
fake_obs_cov.append(data_nbar_var)
if obv == 'gmf':
data_gmf, data_gmf_sigma = Data.data_gmf(**data_dict)
fake_obs.append(data_gmf)
fake_obs_cov.append(data_gmf)
if obv == 'xi':
# import xir and full covariance matrix of xir
data_xi, data_xi_cov = Data.data_xi_full_cov(**data_dict)
data_xi_invcov = Data.data_xi_inv_cov(**data_dict)
fake_obs.append(data_xi)
fake_obs_cov.append(data_xi_invcov)
# True HOD parameters
data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr'])
data_hod = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
Ndim = len(data_hod)
# Priors
prior_min, prior_max = PriorRange(prior_name)
prior_range = np.zeros((len(prior_min),2))
prior_range[:,0] = prior_min
prior_range[:,1] = prior_max
# Initializing Walkers
random_guess = np.array([11. , np.log(.4) , 11.5 , 1.0 , 13.5])
pos0 = np.repeat(random_guess, Nwalkers).reshape(Ndim, Nwalkers).T + \
1e-1 * np.random.randn(Ndim * Nwalkers).reshape(Nwalkers, Ndim)
# Initializing the emcee sampler
hod_kwargs = {
'prior_range': prior_range,
'data': fake_obs,
'data_cov': fake_obs_cov,
'observables': observables,
'Mr': data_dict['Mr']
}
# Set up the interface to the ipcluster.
c = Client()
view = c[:]
view.push({"lnPost": lnPost})
# Modules necessary in posterior calculation should be called here
view.execute("import numpy as np")
view.execute("from hod_sim import HODsimulator")
# Setting up the Sampler
sampler = emcee.EnsembleSampler(Nwalkers, Ndim, lnPost, kwargs=hod_kwargs, pool = view)
# Setting up a file for saving the chains
chain_file = ''.join([util.dat_dir(),
util.observable_id_flag(observables),
'_Mr', str(data_dict["Mr"]), '_theta.mcmc_chain.dat'])
f = open(chain_file, "w")
f.close()
# Running the Sampler and writing out the chains
for result in sampler.sample(pos0, iterations=Nchains_burn + Nchains_pro, storechain=False):
position = result[0]
f = open(chain_file, "a")
for k in range(position.shape[0]):
output_str = '\t'.join(position[k].astype('str')) + '\n'
f.write(output_str)
f.close()
def ABCpmc_HOD(T, eps_val, N_part=1000, prior_name='first_try', observables=['nbar', 'xi'],
abcrun=None, data_dict={'Mr':21, 'b_normal':0.25}):
'''
ABC-PMC implementation.
Parameters
----------
- T : Number of iterations
- eps_val :
- N_part : Number of particles
- observables : list of observables. Options are 'nbar', 'gmf', 'xi'
- data_dict : dictionary that specifies the observation keywords
'''
if abcrun is None:
raise ValueError("Specify the name of the abcrun!")
#Initializing the vector of observables and inverse covariance matrix
fake_obs, Cii_list = getObvs(observables, **data_dict)
# True HOD parameters
data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr'])
data_hod = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
# Priors
prior_min, prior_max = PriorRange(prior_name)
prior = abcpmc.TophatPrior(prior_min, prior_max)
prior_range = np.zeros((len(prior_min),2))
prior_range[:,0] = prior_min
prior_range[:,1] = prior_max
# Simulator
our_model = ABC_HODsim(Mr=data_dict['Mr'], b_normal=data_dict['b_normal']) # initialize model
kwargs = {'prior_range': prior_range, 'observables': observables}
def simz(tt):
sim = our_model(tt, **kwargs)
if sim is None:
pickle.dump(tt, open(util.crash_dir()+"simz_crash_theta.p", 'wb'))
pickle.dump(kwargs, open(util.crash_dir()+'simz_crash_kwargs.p', 'wb'))
raise ValueError('Simulator is giving NonetType')
return sim
def multivariate_rho(model, datum):
dists = []
if observables == ['nbar','xi']:
nbar_Cii = Cii_list[0]
xi_Cii = Cii_list[1]
dist_nbar = (datum[0] - model[0])**2. / nbar_Cii
dist_xi = np.sum((datum[1:] - model[1:])**2. / xi_Cii)
dists = [dist_nbar , dist_xi]
elif observables == ['nbar','gmf']:
nbar_Cii = Cii_list[0]
gmf_Cii = Cii_list[1]
dist_nbar = (datum[0] - model[0])**2. / nbar_Cii
# omitting the first GMF bin in the model ([1:])
dist_gmf = np.sum((datum[1:] - model[1][1:])**2. / gmf_Cii)
dists = [dist_nbar , dist_gmf]
elif observables == ['xi']:
xi_Cii = Cii_list[0]
dist_xi = np.sum((datum- model)**2. / xi_Cii)
dists = [dist_xi]
return np.array(dists)
tolerance_file = lambda name: ''.join([util.abc_dir(), "abc_tolerance", '.', name, '.dat'])
theta_file = lambda tt, name: ''.join([util.abc_dir(),
util.observable_id_flag(observables), '_theta_t', str(tt), '.', name, '.dat'])
w_file = lambda tt, name: ''.join([util.abc_dir(),
util.observable_id_flag(observables), '_w_t', str(tt), '.', name, '.dat'])
dist_file = lambda tt, name: ''.join([util.abc_dir(),
util.observable_id_flag(observables), '_dist_t', str(tt), '.', name, '.dat'])
def launch(eps_start, init_pool=None):
print eps_start
eps = abcpmc.ConstEps(T, eps_start)
mpi_pool = mpi_util.MpiPool()
pools = []
abcpmc_sampler = abcpmc.Sampler(
N=N_part, #N_particles
Y=fake_obs, #data
postfn=simz, #simulator
dist=multivariate_rho, #distance function
pool=mpi_pool)
abcpmc_sampler.particle_proposal_cls = abcpmc.ParticleProposal
f = open(tolerance_file(abcrun), "w")
f.close()
eps_str = ''
for pool in abcpmc_sampler.sample(prior, eps):
#while pool.ratio > 0.01:
new_eps_str = '\t'.join(np.array(pool.eps).astype('str'))+'\n'
if eps_str != new_eps_str: # if eps is different, open fiel and append
f = open(tolerance_file(abcrun) , "a")
eps_str = new_eps_str
f.write(eps_str)
f.close()
print("T:{0},ratio: {1:>.4f}".format(pool.t, pool.ratio))
print pool.eps
# write theta, w, and rhos to file
np.savetxt(theta_file(pool.t, abcrun), pool.thetas)
np.savetxt(w_file(pool.t, abcrun), pool.ws)
np.savetxt(dist_file(pool.t, abcrun) , pool.dists)
# plot theta
plot_thetas(pool.thetas, pool.ws , pool.t,
truths=data_hod, plot_range=prior_range,
theta_filename=theta_file(pool.t, abcrun),
output_dir=util.abc_dir())
eps.eps = np.median(np.atleast_2d(pool.dists), axis = 0)
pools.append(pool)
abcpmc_sampler.close()
return pools
print "Initial launch of the sampler"
pools = launch(eps_val)
def overlay_pdfs_contours(abc_filename , mcmc_filename , nwalkers , nburns , Mr):
import matplotlib as mpl
label_size = 35
mpl.rcParams['xtick.labelsize'] = label_size
mpl.rcParams['ytick.labelsize'] = label_size
MP_LINEWIDTH = 2.4
MP_TICKSIZE = 10.
thick_line1 = mlines.Line2D([], [], ls = '-', c = 'blue', linewidth=4,
label='ABC-PMC')
thick_line2 = mlines.Line2D([], [], ls = '-', c = 'red', linewidth=4,
label=r'Gaussian $\mathcal{L}$ +MCMC')
mpl.rc('axes', linewidth=MP_LINEWIDTH)
data_hod_dict = Data.data_hod_param(Mr=Mr)
truths = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
prior_min, prior_max = PriorRange(None)
plot_range = np.zeros((len(prior_min),2))
plot_range[:,0] = prior_min
plot_range[:,1] = prior_max
mcmc_sample = np.loadtxt(mcmc_filename)[nburns*nwalkers:,:]
abc_sample = np.loadtxt(abc_filename)
# ##################### scatter plots ##########################
# fig, axes = plt.subplots(2, 3, figsize=(48, 13))
# fig.subplots_adjust(wspace=0.4, hspace=0.2)
#
# ax = axes[0]
# ax_list = corner.hist2d(mcmc_sample[:,3],mcmc_sample[:,2],bins =20,levels=[0.68,0.95], ax = ax, plot_datapoints = False ,
# fill_contours = True, alpha = 10. , color = 'r', smooth = 1. , linewidth=4, range = [plot_range[3,:],plot_range[2,:]])
# ax_list = corner.hist2d(abc_sample[:,3],abc_sample[:,2],bins =20,levels=[0.68,0.95], ax = ax, plot_datapoints = False ,
# fill_contours = True,alpha = 0.1, color = 'b', smooth = 1. , linewidth=4, range = [plot_range[3,:],plot_range[2,:]])
# ax.plot(truths[3] , truths[2] , marker="*", markersize=25 , color = "yellow")
# ax.set_ylabel(r'$\log M_{\rm min}$', fontsize = 50)
# ax.set_xlabel(r'$\alpha$', fontsize = 50)
# ax.set_xlim([plot_range[3,0] , plot_range[3,1]])
# ax.set_ylim([plot_range[2,0] , plot_range[2,1]])
#
#
# ax = axes[1]
# ax_list = corner.hist2d(mcmc_sample[:,2],mcmc_sample[:,4],bins =20,levels=[0.68,0.95], ax = ax, plot_datapoints = False ,
# fill_contours = True, color = 'r', smooth = 1. , linewidth=4, range = [plot_range[2,:],plot_range[4,:]])
# ax_list = corner.hist2d(abc_sample[:,2],abc_sample[:,4],bins =20,levels=[0.68,0.95], ax = ax, plot_datapoints = False ,
# fill_contours = True, color = 'b', smooth = 1. , linewidth=4, range = [plot_range[2,:],plot_range[4,:]])
# ax.plot(truths[2] , truths[4] , marker="*", markersize=25 , color = "yellow")
# ax.set_xlabel(r'$\log M_{\rm min}$', fontsize = 50)
# ax.set_ylabel(r'$\log M_{1}$', fontsize = 50)
# ax.set_xlim([plot_range[2,0] , plot_range[2,1]])
# ax.set_ylim([plot_range[4,0] , plot_range[4,1]])
#
#
#
# ax = axes[2]
# ax_list = corner.hist2d(mcmc_sample[:,3],mcmc_sample[:,4],bins =20,levels=[0.68,0.95], ax = ax, plot_datapoints = False ,
# fill_contours = True, color = 'r', smooth = 1. , linewidth=4, range = [plot_range[3,:],plot_range[4,:]])
# ax_list = corner.hist2d(abc_sample[:,3], abc_sample[:,4], bins =20,levels=[0.68,0.95], ax = ax, plot_datapoints = False ,
# fill_contours = True, color = 'b', smooth = 1. , linewidth=4, range = [plot_range[3,:],plot_range[4,:]])
# ax.plot(truths[3] , truths[4] , marker="*", markersize=25 , color = "yellow")
# ax.set_ylabel(r'$\log M_{1}$', fontsize = 50)
# ax.set_xlabel(r'$\alpha$', fontsize = 50)
# ax.set_xlim([plot_range[3,0] , plot_range[3,1]])
# ax.set_ylim([plot_range[4,0] , plot_range[4,1]])
#
#
# plt.legend(handles=[thick_line2, thick_line1], frameon=False, loc='best', fontsize=50)
#
# plt.savefig("contours_nbarxi2.pdf")
################## HISTOGRAMS#########################################
# fig, axes = plt.subplots(1, 3, figsize=(48, 13))
# fig.subplots_adjust(wspace=0.4, hspace=0.2)
#
# ax = axes[0]
# q = ax.hist(mcmc_sample[:,2], bins =20, range = [plot_range[2,:].min(),plot_range[2,:].max()] , normed = True , alpha = 1. , color = 'r', linewidth=4 , histtype='step')
# qq = ax.hist(abc_sample[:,2], bins =20, range = [plot_range[2,:].min(),plot_range[2,:].max()] ,normed = True , alpha = 1. , color = 'b', linewidth=4 , histtype='step')
# ax.vlines(truths[2], 0, max(q[0].max(),qq[0].max()), color = "k", linewidth = 5)
# ax.set_xlabel(r'$\log M_{\rm min}$', fontsize = 50)
# ax.set_xlim([plot_range[2,0] , plot_range[2,1]])
#
#
# ax = axes[1]
# q = ax.hist(mcmc_sample[:,3], bins =20, range = [plot_range[3,:].min(),plot_range[3,:].max()] ,normed = True , alpha = 1. , color = 'r', linewidth=4, histtype='step')
# qq = ax.hist(abc_sample[:,3], bins =20, range = [plot_range[3,:].min(),plot_range[3,:].max()] ,normed = True , alpha = 1. , color = 'b', linewidth=4, histtype='step')
# ax.vlines(truths[3], 0, max(q[0].max(),qq[0].max()), color = "k" , linewidth = 5)
# ax.set_xlabel(r'$\alpha$', fontsize = 50)
# ax.set_xlim([plot_range[3,0] , plot_range[3,1]])
#
#
# ax = axes[2]
# q = ax.hist(mcmc_sample[:,4], bins =20, range = [plot_range[4,:].min(),plot_range[4,:].max()] ,normed = True , alpha = 1. , color = 'r', linewidth=4, histtype='step')
# qq = ax.hist(abc_sample[:,4], bins =20, range = [plot_range[4,:].min(),plot_range[4,:].max()] ,normed = True , alpha = 1. , color = 'b', linewidth=4 , histtype='step')
# ax.vlines(truths[4] , 0, max(q[0].max(),qq[0].max()), colors='k' , linewidth = 5)
# ax.set_xlabel(r'$\log M_{1}$', fontsize = 50)
# ax.set_xlim([plot_range[4,0] , plot_range[4,1]])
#
#
# plt.legend(handles=[thick_line2, thick_line1], frameon=False, loc='best', fontsize=30)
#
# plt.savefig("histograms_nbarxi2.pdf")
# return None
# get the 68% and 95% confidence interval indices for sorted chain
normie = norm()
sig1lo = normie.cdf(-1)
sig1hi = normie.cdf(1)
sig2lo = normie.cdf(-2)
sig2hi = normie.cdf(2)
nsamples_mcmc = len(mcmc_sample[:, 2])
nsamples_abc = len(abc_sample[:, 2])
sig1lo_mcmc = int(sig1lo * nsamples_mcmc)
sig2lo_mcmc = int(sig2lo * nsamples_mcmc)
sig1hi_mcmc = int(sig1hi * nsamples_mcmc)
sig2hi_mcmc = int(sig2hi * nsamples_mcmc)
sig1lo_abc = int(sig1lo * nsamples_abc)
sig2lo_abc = int(sig2lo * nsamples_abc)
sig1hi_abc = int(sig1hi * nsamples_abc)
sig2hi_abc = int(sig2hi * nsamples_abc)
# choose colours here
abclr = 'darkcyan'
mcmclr = 'darkorchid'
fig = plt.figure(1, figsize=(50, 25))
gs = gridspec.GridSpec(2, 3, height_ratios=[3, 1])
ax = plt.subplot(gs[0])
q = ax.hist(mcmc_sample[:,2], bins=20,
range=[plot_range[2,:].min(),
plot_range[2,:].max()],
normed=True, alpha=1., color=mcmclr,
linewidth=4, histtype='step')
qq = ax.hist(abc_sample[:,2], bins=20,
range=[plot_range[2,:].min(),
plot_range[2,:].max()],
normed=True, alpha=1., color=abclr,
linewidth=4, histtype='step')
# ax.vlines(truths[2], 0, max(q[0].max(),qq[0].max()),
# color = "k", linewidth = 5)
ax.axvline(truths[2], color='k', linewidth=5)
ax.set_xticklabels([])
ax.set_xlim([plot_range[2,0] , plot_range[2,1]])
ax = plt.subplot(gs[1])
q = ax.hist(mcmc_sample[:,3], bins=20,
range=[plot_range[3,:].min(),
plot_range[3,:].max()],
normed=True, alpha=1., color=mcmclr,
linewidth=4, histtype='step')
qq = ax.hist(abc_sample[:,3], bins=20,
range=[plot_range[3,:].min(),
plot_range[3,:].max()],
normed=True, alpha=1., color=abclr,
linewidth=4, histtype='step')
# ax.vlines(truths[3], 0, max(q[0].max(),qq[0].max()),
# color="k", linewidth=5)
ax.axvline(truths[3], color='k', linewidth=5)
ax.set_xticklabels([])
ax.set_xlim([plot_range[3,0], plot_range[3,1]])
ax = plt.subplot(gs[2])
q = ax.hist(mcmc_sample[:,4], bins=20,
range=[plot_range[4,:].min(),
plot_range[4,:].max()],
normed=True, alpha=1., color=mcmclr,
linewidth=4, histtype='step')
qq = ax.hist(abc_sample[:,4], bins=20,
range=[plot_range[4,:].min(),
plot_range[4,:].max()],
normed=True, alpha=1., color=abclr,
linewidth=4, histtype='step')
# ax.vlines(truths[4], 0, max(q[0].max(),qq[0].max()),
# colors='k', linewidth=5)
ax.axvline(truths[4], color='k', linewidth=5)
ax.set_xticklabels([])
ax.set_xlim([plot_range[4,0], plot_range[4,1]])
thick_line1 = mlines.Line2D([], [], ls='-', c=abclr, linewidth=4,
label='ABC-PMC')
thick_line2 = mlines.Line2D([], [], ls='-', c=mcmclr, linewidth=4,
label=r'Gaussian $\mathcal{L}$ +MCMC')
ax.legend(handles=[thick_line2, thick_line1],
frameon=False, loc='best', fontsize=30)
# and now the box plots below
# general box properties
boxprops = {'color': 'k'}
medianprops = {'alpha': 0.}
bplots1 = []
ax = plt.subplot(gs[3])
# bplots.append(ax.boxplot(mcmc_sample[:,2],
# vert=False, patch_artist=True))
# bplots.append(ax.boxplot(abc_sample[:,2],
# vert=False, patch_artist=True))
# stats dict for each box
bplots1.append({'med': np.median(mcmc_sample[:, 2]),
'q1': np.sort(mcmc_sample[:, 2])[sig1lo_mcmc],
'q3': np.sort(mcmc_sample[:, 2])[sig1hi_mcmc],
'whislo': np.sort(mcmc_sample[:, 2])[sig2lo_mcmc],
'whishi': np.sort(mcmc_sample[:, 2])[sig2hi_mcmc],
'fliers': []})
bplots1.append({'med': np.median(abc_sample[:, 2]),
'q1': np.sort(abc_sample[:, 2])[sig1lo_abc],
'q3': np.sort(abc_sample[:, 2])[sig1hi_abc],
'whislo': np.sort(abc_sample[:, 2])[sig2lo_abc],
'whishi': np.sort(abc_sample[:, 2])[sig2hi_abc],
'fliers': []})
bxp1 = ax.bxp(bplots1, positions=[1,2], vert=False, patch_artist=True,
showfliers=False, boxprops=boxprops, medianprops=medianprops)
# boxprops=boxprops, medianprops=medianprops, sym='')
for i, box in enumerate(bxp1['boxes']):
if i == 0:
box.set(facecolor=mcmclr, alpha=0.5)
elif i == 1:
box.set(facecolor=abclr, alpha=0.5)
# for patch in bplot['boxes']:
# patch.set_facecolor(abclr)
# patch.set_alpha(0.5)
# ax.vlines(truths[2], 0, ax.get_ylim(), # max(q[0].max(),qq[0].max()),
# color = "k", linewidth = 5)
ax.axvline(truths[2], color='k', linewidth=5)
ax.set_xlim([plot_range[2,0] , plot_range[2,1]])
ax.set_xlabel(r'$\log M_{\rm min}$', fontsize = 50)
ax.set_yticks([1,2])
ax.set_yticklabels(["MCMC", "ABC"])
ax = plt.subplot(gs[4])
# bplots.append(ax.boxplot(mcmc_sample[:,3],
# vert=False, patch_artist=True))
# bplots.append(ax.boxplot(abc_sample[:,3],
# vert=False, patch_artist=True))
bplots2 = []
bplots2.append({'med': np.median(mcmc_sample[:, 3]),
'q1': np.sort(mcmc_sample[:, 3])[sig1lo_mcmc],
'q3': np.sort(mcmc_sample[:, 3])[sig1hi_mcmc],
'whislo': np.sort(mcmc_sample[:, 3])[sig2lo_mcmc],
'whishi': np.sort(mcmc_sample[:, 3])[sig2hi_mcmc],
'fliers': []})
bplots2.append({'med': np.median(abc_sample[:, 3]),
'q1': np.sort(abc_sample[:, 3])[sig1lo_abc],
'q3': np.sort(abc_sample[:, 3])[sig1hi_abc],
'whislo': np.sort(abc_sample[:, 3])[sig2lo_abc],
'whishi': np.sort(abc_sample[:, 3])[sig2hi_abc],
'fliers': []})
# for i, bplot in enumerate(bplots[2:]):
# if i == 0:
# for patch in bplot['boxes']:
# patch.set_facecolor(mcmclr)
# patch.set_alpha(0.5)
# elif i == 1:
# for patch in bplot['boxes']:
# patch.set_facecolor(abclr)
# patch.set_alpha(0.5)
bxp2 = ax.bxp(bplots2, positions=[1,2], vert=False, patch_artist=True,
showfliers=False, boxprops=boxprops, medianprops=medianprops)
for i, box in enumerate(bxp2['boxes']):
if i == 0:
box.set(facecolor=mcmclr, alpha=0.5)
elif i == 1:
box.set(facecolor=abclr, alpha=0.5)
# ax.vlines(truths[3], 0, ax.get_ylim(), # max(q[0].max(),qq[0].max()),
# color = "k" , linewidth = 5)
ax.axvline(truths[3], color='k', linewidth=5)
ax.set_xlim([plot_range[3,0], plot_range[3,1]])
ax.set_xlabel(r'$\alpha$', fontsize = 50)
ax.set_yticks([])
ax = plt.subplot(gs[5])
# bplots.append(ax.boxplot(mcmc_sample[:,4],
# vert=False, patch_artist=True))
# bplots.append(ax.boxplot(abc_sample[:,4],
# vert=False, patch_artist=True))
# for i, bplot in enumerate(bplots[4:]):
# if i == 0:
# for patch in bplot['boxes']:
# patch.set_facecolor(mcmclr)
# patch.set_alpha(0.5)
# elif i == 1:
# for patch in bplot['boxes']:
# patch.set_facecolor(abclr)
# patch.set_alpha(0.5)
bplots3 = []
bplots3.append({'med': np.median(mcmc_sample[:, 4]),
'q1': np.sort(mcmc_sample[:, 4])[sig1lo_mcmc],
'q3': np.sort(mcmc_sample[:, 4])[sig1hi_mcmc],
'whislo': np.sort(mcmc_sample[:, 4])[sig2lo_mcmc],
'whishi': np.sort(mcmc_sample[:, 4])[sig2hi_mcmc],
'fliers': []})
bplots3.append({'med': np.median(abc_sample[:, 4]),
'q1': np.sort(abc_sample[:, 4])[sig1lo_abc],
'q3': np.sort(abc_sample[:, 4])[sig1hi_abc],
'whislo': np.sort(abc_sample[:, 4])[sig2lo_abc],
'whishi': np.sort(abc_sample[:, 4])[sig2hi_abc],
'fliers': []})
bxp3 = ax.bxp(bplots3, positions=[1,2], vert=False, patch_artist=True,
showfliers=False, boxprops=boxprops, medianprops=medianprops)
for i, box in enumerate(bxp3['boxes']):
if i == 0:
box.set(facecolor=mcmclr, alpha=0.5)
elif i == 1:
box.set(facecolor=abclr, alpha=0.5)
# ax.vlines(truths[4] , 0, ax.get_ylim(), # max(q[0].max(),qq[0].max()),
# colors='k' , linewidth = 5)
ax.axvline(truths[4], color='k', linewidth=5)
ax.set_xlim([plot_range[4,0], plot_range[4,1]])
ax.set_xlabel(r'$\log M_{1}$', fontsize = 50)
ax.set_yticks([])
fig.subplots_adjust(wspace=0.05, hspace=0.0)
plt.savefig("histograms_nbarxi2_boxplot_fix.pdf")
return None