def getObvs(observables, **data_dict):
''' Given the list of observable strings return data vector and
covariance matrices
'''
if observables == ['xi']:
fake_obs = Data.data_xi(**data_dict)
fake_obs_cov = Data.data_cov(inference='abc', **data_dict)[1:16 , 1:16]
xi_Cii = np.diag(fake_obs_cov)
Cii_list = [xi_Cii]
elif observables == ['nbar','xi']:
fake_obs = np.hstack([Data.data_nbar(**data_dict), Data.data_xi(**data_dict)])
fake_obs_cov = Data.data_cov(inference='abc', **data_dict)[:16 , :16]
Cii = np.diag(fake_obs_cov)
xi_Cii = Cii[1:]
nbar_Cii = Cii[0]
Cii_list = [nbar_Cii, xi_Cii]
elif observables == ['nbar','gmf']:
##### FIRST BIN OF GMF DROPPED ###############
# CAUTION: hardcoded
fake_obs = np.hstack([Data.data_nbar(**data_dict), Data.data_gmf(**data_dict)[1:]])
# CAUTION: Covariance matrix starts at 17 instead
fake_obs_cov = Data.data_cov(inference='abc', **data_dict)
Cii = np.diag(fake_obs_cov)
gmf_Cii = Cii[17:]
nbar_Cii = Cii[0]
Cii_list = [nbar_Cii, gmf_Cii]
return fake_obs, Cii_list
def abc_hodsim(Mr=21, b_normal=0.25):
prior_min, prior_max = PriorRange('first_try')
prior_range = np.zeros((len(prior_min),2))
prior_range[:,0] = prior_min
prior_range[:,1] = prior_max
abc_hod = ABC_HODsim()
data_hod_dict = Data.data_hod_param(Mr=21)
data_hod = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
print Data.data_nbar(Mr=Mr, b_normal=b_normal), Data.data_gmf(Mr=Mr, b_normal=b_normal)
print abc_hod(data_hod, prior_range=prior_range, observables=['nbar', 'gmf'])
def abc_hodsim(Mr=21, b_normal=0.25):
prior_min, prior_max = PriorRange('first_try')
prior_range = np.zeros((len(prior_min), 2))
prior_range[:, 0] = prior_min
prior_range[:, 1] = prior_max
abc_hod = ABC_HODsim()
data_hod_dict = Data.data_hod_param(Mr=21)
data_hod = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
print Data.data_nbar(Mr=Mr,
b_normal=b_normal), Data.data_gmf(Mr=Mr,
b_normal=b_normal)
print abc_hod(data_hod,
prior_range=prior_range,
observables=['nbar', 'gmf'])
def test_nbar(Mr=21, b_normal=0.25):
print Data.data_nbar(Mr=Mr, b_normal=b_normal)
def mcmc_mpi(Nwalkers,
Nchains,
observables=['nbar', 'xi'],
data_dict={
'Mr': 21,
'b_normal': 0.25
},
prior_name='first_try',
mcmcrun=None):
'''
Standard MCMC implementaion
Parameters
-----------
- Nwalker :
Number of walkers
- Nchains :
Number of MCMC chains
- observables :
list of observables. Options are: ['nbar','xi'],['nbar','gmf'],['xi']
- data_dict : dictionary that specifies the observation keywords
'''
#Initializing the vector of observables and inverse covariance matrix
if observables == ['xi']:
fake_obs = Data.data_xi(**data_dict)
#fake_obs_icov = Data.data_inv_cov('xi', **data_dict)
fake_obs_icov = Data.data_cov(inference='mcmc', **data_dict)[1:16,
1:16]
if observables == ['nbar', 'xi']:
fake_obs = np.hstack(
[Data.data_nbar(**data_dict),
Data.data_xi(**data_dict)])
fake_obs_icov = Data.data_cov(inference='mcmc', **data_dict)[:16, :16]
if observables == ['nbar', 'gmf']:
##### FIRST BIN OF GMF DROPPED ###############
# CAUTION: hardcoded
fake_obs = np.hstack(
[Data.data_nbar(**data_dict),
Data.data_gmf(**data_dict)[1:]])
fake_obs_icov = np.zeros((10, 10))
#print Data.data_cov(**data_dict)[17: , 17:].shape
# Covariance matrix being adjusted accordingly
fake_obs_icov[1:, 1:] = Data.data_cov(inference='mcmc',
**data_dict)[17:, 17:]
fake_obs_icov[0, 1:] = Data.data_cov(inference='mcmc',
**data_dict)[0, 17:]
fake_obs_icov[1:, 0] = Data.data_cov(inference='mcmc',
**data_dict)[17:, 0]
fake_obs_icov[0, 0] = Data.data_cov(inference='mcmc', **data_dict)[0,
0]
# True HOD parameters
data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr'])
data_hod = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
Ndim = len(data_hod)
# Priors
prior_min, prior_max = PriorRange(prior_name)
prior_range = np.zeros((len(prior_min), 2))
prior_range[:, 0] = prior_min
prior_range[:, 1] = prior_max
# mcmc chain output file
chain_file = ''.join([
util.mcmc_dir(),
util.observable_id_flag(observables), '.', mcmcrun, '.mcmc_chain.dat'
])
#print chain_file
if os.path.isfile(chain_file) and continue_chain:
print 'Continuing previous MCMC chain!'
sample = np.loadtxt(chain_file)
Nchain = Niter - (len(sample) / Nwalkers
) # Number of chains left to finish
if Nchain > 0:
pass
else:
raise ValueError
print Nchain, ' iterations left to finish'
# Initializing Walkers from the end of the chain
pos0 = sample[-Nwalkers:]
else:
# new chain
f = open(chain_file, 'w')
f.close()
Nchain = Niter
# Initializing Walkers
random_guess = data_hod
pos0 = np.repeat(random_guess, Nwalkers).reshape(Ndim, Nwalkers).T + \
5.e-2 * np.random.randn(Ndim * Nwalkers).reshape(Nwalkers, Ndim)
#print pos0.shape
# Initializing MPIPool
pool = MPIPool()
if not pool.is_master():
pool.wait()
sys.exit(0)
# Initializing the emcee sampler
hod_kwargs = {
'prior_range': prior_range,
'data': fake_obs,
'data_icov': fake_obs_icov,
'observables': observables,
'Mr': data_dict['Mr']
}
sampler = emcee.EnsembleSampler(Nwalkers,
Ndim,
lnPost,
pool=pool,
kwargs=hod_kwargs)
# Initializing Walkers
for result in sampler.sample(pos0, iterations=Nchain, storechain=False):
position = result[0]
#print position
f = open(chain_file, 'a')
for k in range(position.shape[0]):
output_str = '\t'.join(position[k].astype('str')) + '\n'
f.write(output_str)
f.close()
pool.close()
def mcmc_ipython_par(Nwalkers, Nchains_burn, Nchains_pro, observables=['nbar', 'xi'],
data_dict={'Mr':20, 'Nmock':500}, prior_name = 'first_try', threads=1):
'''
Standard MCMC implementaion
Parameters
-----------
- Nwalker :
Number of walkers
- Nchains_burn :
Number of burn-in chains
- Nchains_pro :
Number of production chains
- observables :
list of observables. Options are 'nbar', 'gmf', 'xi'
- data_dict : dictionary that specifies the observation keywords
'''
# data observables
fake_obs = [] # list of observables
fake_obs_cov = []
for obv in observables:
if obv == 'nbar':
data_nbar, data_nbar_var = Data.data_nbar(**data_dict)
fake_obs.append(data_nbar)
fake_obs_cov.append(data_nbar_var)
if obv == 'gmf':
data_gmf, data_gmf_sigma = Data.data_gmf(**data_dict)
fake_obs.append(data_gmf)
fake_obs_cov.append(data_gmf)
if obv == 'xi':
# import xir and full covariance matrix of xir
data_xi, data_xi_cov = Data.data_xi_full_cov(**data_dict)
data_xi_invcov = Data.data_xi_inv_cov(**data_dict)
fake_obs.append(data_xi)
fake_obs_cov.append(data_xi_invcov)
# True HOD parameters
data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr'])
data_hod = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
Ndim = len(data_hod)
# Priors
prior_min, prior_max = PriorRange(prior_name)
prior_range = np.zeros((len(prior_min),2))
prior_range[:,0] = prior_min
prior_range[:,1] = prior_max
# Initializing Walkers
random_guess = np.array([11. , np.log(.4) , 11.5 , 1.0 , 13.5])
pos0 = np.repeat(random_guess, Nwalkers).reshape(Ndim, Nwalkers).T + \
1e-1 * np.random.randn(Ndim * Nwalkers).reshape(Nwalkers, Ndim)
# Initializing the emcee sampler
hod_kwargs = {
'prior_range': prior_range,
'data': fake_obs,
'data_cov': fake_obs_cov,
'observables': observables,
'Mr': data_dict['Mr']
}
# Set up the interface to the ipcluster.
c = Client()
view = c[:]
view.push({"lnPost": lnPost})
# Modules necessary in posterior calculation should be called here
view.execute("import numpy as np")
view.execute("from hod_sim import HODsimulator")
# Setting up the Sampler
sampler = emcee.EnsembleSampler(Nwalkers, Ndim, lnPost, kwargs=hod_kwargs, pool = view)
# Setting up a file for saving the chains
chain_file = ''.join([util.dat_dir(),
util.observable_id_flag(observables),
'_Mr', str(data_dict["Mr"]), '_theta.mcmc_chain.dat'])
f = open(chain_file, "w")
f.close()
# Running the Sampler and writing out the chains
for result in sampler.sample(pos0, iterations=Nchains_burn + Nchains_pro, storechain=False):
position = result[0]
f = open(chain_file, "a")
for k in range(position.shape[0]):
output_str = '\t'.join(position[k].astype('str')) + '\n'
f.write(output_str)
f.close()
def mcmc_mpi(
Nwalkers,
Nchains,
observables=["nbar", "xi"],
data_dict={"Mr": 21, "b_normal": 0.25},
prior_name="first_try",
mcmcrun=None,
):
"""
Standard MCMC implementaion
Parameters
-----------
- Nwalker :
Number of walkers
- Nchains :
Number of MCMC chains
- observables :
list of observables. Options are: ['nbar','xi'],['nbar','gmf'],['xi']
- data_dict : dictionary that specifies the observation keywords
"""
# Initializing the vector of observables and inverse covariance matrix
if observables == ["xi"]:
fake_obs = Data.data_xi(**data_dict)
# fake_obs_icov = Data.data_inv_cov('xi', **data_dict)
fake_obs_icov = Data.data_cov(inference="mcmc", **data_dict)[1:16, 1:16]
if observables == ["nbar", "xi"]:
fake_obs = np.hstack([Data.data_nbar(**data_dict), Data.data_xi(**data_dict)])
fake_obs_icov = Data.data_cov(inference="mcmc", **data_dict)[:16, :16]
if observables == ["nbar", "gmf"]:
##### FIRST BIN OF GMF DROPPED ###############
# CAUTION: hardcoded
fake_obs = np.hstack([Data.data_nbar(**data_dict), Data.data_gmf(**data_dict)[1:]])
fake_obs_icov = np.zeros((10, 10))
# print Data.data_cov(**data_dict)[17: , 17:].shape
# Covariance matrix being adjusted accordingly
fake_obs_icov[1:, 1:] = Data.data_cov(inference="mcmc", **data_dict)[17:, 17:]
fake_obs_icov[0, 1:] = Data.data_cov(inference="mcmc", **data_dict)[0, 17:]
fake_obs_icov[1:, 0] = Data.data_cov(inference="mcmc", **data_dict)[17:, 0]
fake_obs_icov[0, 0] = Data.data_cov(inference="mcmc", **data_dict)[0, 0]
# True HOD parameters
data_hod_dict = Data.data_hod_param(Mr=data_dict["Mr"])
data_hod = np.array(
[
data_hod_dict["logM0"], # log M0
np.log(data_hod_dict["sigma_logM"]), # log(sigma)
data_hod_dict["logMmin"], # log Mmin
data_hod_dict["alpha"], # alpha
data_hod_dict["logM1"], # log M1
]
)
Ndim = len(data_hod)
# Priors
prior_min, prior_max = PriorRange(prior_name)
prior_range = np.zeros((len(prior_min), 2))
prior_range[:, 0] = prior_min
prior_range[:, 1] = prior_max
# mcmc chain output file
chain_file = "".join([util.mcmc_dir(), util.observable_id_flag(observables), ".", mcmcrun, ".mcmc_chain.dat"])
# print chain_file
if os.path.isfile(chain_file) and continue_chain:
print "Continuing previous MCMC chain!"
sample = np.loadtxt(chain_file)
Nchain = Niter - (len(sample) / Nwalkers) # Number of chains left to finish
if Nchain > 0:
pass
else:
raise ValueError
print Nchain, " iterations left to finish"
# Initializing Walkers from the end of the chain
pos0 = sample[-Nwalkers:]
else:
# new chain
f = open(chain_file, "w")
f.close()
Nchain = Niter
# Initializing Walkers
random_guess = data_hod
pos0 = np.repeat(random_guess, Nwalkers).reshape(Ndim, Nwalkers).T + 5.0e-2 * np.random.randn(
Ndim * Nwalkers
).reshape(Nwalkers, Ndim)
# print pos0.shape
# Initializing MPIPool
pool = MPIPool()
if not pool.is_master():
pool.wait()
sys.exit(0)
# Initializing the emcee sampler
hod_kwargs = {
"prior_range": prior_range,
"data": fake_obs,
"data_icov": fake_obs_icov,
"observables": observables,
"Mr": data_dict["Mr"],
}
sampler = emcee.EnsembleSampler(Nwalkers, Ndim, lnPost, pool=pool, kwargs=hod_kwargs)
# Initializing Walkers
for result in sampler.sample(pos0, iterations=Nchain, storechain=False):
position = result[0]
# print position
f = open(chain_file, "a")
for k in range(position.shape[0]):
output_str = "\t".join(position[k].astype("str")) + "\n"
f.write(output_str)
f.close()
pool.close()
def mcmc_ipython_par(Nwalkers,
Nchains_burn,
Nchains_pro,
observables=['nbar', 'xi'],
data_dict={
'Mr': 20,
'Nmock': 500
},
prior_name='first_try',
threads=1):
'''
Standard MCMC implementaion
Parameters
-----------
- Nwalker :
Number of walkers
- Nchains_burn :
Number of burn-in chains
- Nchains_pro :
Number of production chains
- observables :
list of observables. Options are 'nbar', 'gmf', 'xi'
- data_dict : dictionary that specifies the observation keywords
'''
# data observables
fake_obs = [] # list of observables
fake_obs_cov = []
for obv in observables:
if obv == 'nbar':
data_nbar, data_nbar_var = Data.data_nbar(**data_dict)
fake_obs.append(data_nbar)
fake_obs_cov.append(data_nbar_var)
if obv == 'gmf':
data_gmf, data_gmf_sigma = Data.data_gmf(**data_dict)
fake_obs.append(data_gmf)
fake_obs_cov.append(data_gmf)
if obv == 'xi':
# import xir and full covariance matrix of xir
data_xi, data_xi_cov = Data.data_xi_full_cov(**data_dict)
data_xi_invcov = Data.data_xi_inv_cov(**data_dict)
fake_obs.append(data_xi)
fake_obs_cov.append(data_xi_invcov)
# True HOD parameters
data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr'])
data_hod = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
Ndim = len(data_hod)
# Priors
prior_min, prior_max = PriorRange(prior_name)
prior_range = np.zeros((len(prior_min), 2))
prior_range[:, 0] = prior_min
prior_range[:, 1] = prior_max
# Initializing Walkers
random_guess = np.array([11., np.log(.4), 11.5, 1.0, 13.5])
pos0 = np.repeat(random_guess, Nwalkers).reshape(Ndim, Nwalkers).T + \
1e-1 * np.random.randn(Ndim * Nwalkers).reshape(Nwalkers, Ndim)
# Initializing the emcee sampler
hod_kwargs = {
'prior_range': prior_range,
'data': fake_obs,
'data_cov': fake_obs_cov,
'observables': observables,
'Mr': data_dict['Mr']
}
# Set up the interface to the ipcluster.
c = Client()
view = c[:]
view.push({"lnPost": lnPost})
# Modules necessary in posterior calculation should be called here
view.execute("import numpy as np")
view.execute("from hod_sim import HODsimulator")
# Setting up the Sampler
sampler = emcee.EnsembleSampler(Nwalkers,
Ndim,
lnPost,
kwargs=hod_kwargs,
pool=view)
# Setting up a file for saving the chains
chain_file = ''.join([
util.dat_dir(),
util.observable_id_flag(observables), '_Mr',
str(data_dict["Mr"]), '_theta.mcmc_chain.dat'
])
f = open(chain_file, "w")
f.close()
# Running the Sampler and writing out the chains
for result in sampler.sample(pos0,
iterations=Nchains_burn + Nchains_pro,
storechain=False):
position = result[0]
f = open(chain_file, "a")
for k in range(position.shape[0]):
output_str = '\t'.join(position[k].astype('str')) + '\n'
f.write(output_str)
f.close()
# --- Local ---
import data as Data
from hod_sim import HODsim
from group_richness import richness
Nwalkers = 10
Nchains_burn = 1
Nchains_pro = 1
# data observables
fake_obs = [] # list of observables
for obv in observables:
if obv == 'nbar':
data_nbar, data_nbar_var = Data.data_nbar(**data_dict)
fake_obs.append(data_nbar)
if obv == 'gmf':
data_gmf, data_gmf_sigma = Data.data_gmf(**data_dict)
fake_obs.append(data_gmf)
if obv == 'xi':
# import xir and full covariance matrix of xir
data_xi, data_xi_cov = Data.data_xi_full_cov(**data_dict)
# take the inverse of the covariance matrix
data_xi_invcov = solve(np.eye(len(data_xi)) , data_xi_cov)
fake_obs.append(data_xi)
# 'True' HOD parameters
data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr'])
data_hod = np.array([
data_hod_dict['logM0'], # log M0
from emcee.utils import MPIPool
# --- Local ---
import data as Data
from hod_sim import HODsim
from group_richness import richness
Nwalkers = 10
Nchains_burn = 1
Nchains_pro = 1
# data observables
fake_obs = [] # list of observables
for obv in observables:
if obv == 'nbar':
data_nbar, data_nbar_var = Data.data_nbar(**data_dict)
fake_obs.append(data_nbar)
if obv == 'gmf':
data_gmf, data_gmf_sigma = Data.data_gmf(**data_dict)
fake_obs.append(data_gmf)
if obv == 'xi':
# import xir and full covariance matrix of xir
data_xi, data_xi_cov = Data.data_xi_full_cov(**data_dict)
# take the inverse of the covariance matrix
data_xi_invcov = solve(np.eye(len(data_xi)), data_xi_cov)
fake_obs.append(data_xi)
# 'True' HOD parameters
data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr'])
data_hod = np.array([
data_hod_dict['logM0'], # log M0
