def Plk_halo(mneut, nreal, nzbin, zspace=False):
''' Return the powerspectrum multipoles for halo catalog with
total neutrino mass `mneut`, realization `nreal`, and redshift
bin `nzbin` in either real/redshift space.
'''
if mneut == 0.1: dir = ''.join([UT.dat_dir(), '0.10eV/'])
else: dir = ''.join([UT.dat_dir(), str(mneut), 'eV/'])
if zspace: str_space = 'z' # redhsift
else: str_space = 'r' # real
f = ''.join([
dir, 'plk.groups.',
str(mneut), 'eV.',
str(nreal), '.nzbin',
str(nzbin), '.', str_space, 'space.dat'
])
if not os.path.isfile(f): raise ValueError('file does not exist')
# read in plk
k, p0k, p2k, p4k = np.loadtxt(f,
skiprows=3,
unpack=True,
usecols=[0, 1, 2, 3])
plk = {'k': k, 'p0k': p0k, 'p2k': p2k, 'p4k': p4k}
# readin shot-noise from header
with open(f) as lines:
for i_l, line in enumerate(lines):
if i_l == 1:
str_sn = line
break
plk['shotnoise'] = float(str_sn.strip().split('P_shotnoise')[-1])
return plk
def NeutHalo_Plk(mneut, nreal, nzbin, zspace=False):
''' Calculate the powerspectrum multipoles for Paco's Neutrio
halo catalogs
notes
-----
* Takes ~20 seconds.
'''
# import Neutrino halo with mneut eV, realization # nreal, at z specified by nzbin
halos = Dat.NeutHalos(mneut, nreal, nzbin)
if zspace:
halos['RSDPosition'] = FM.RSD(halos, LOS=[0,0,1])
# calculate P_l(k)
plk = FM.Observables(halos, observable='plk', rsd=zspace, Nmesh=360)
# output file
if mneut == 0.1: dir = ''.join([UT.dat_dir(), '0.10eV/'])
else: dir = ''.join([UT.dat_dir(), str(mneut), 'eV/'])
if zspace: str_space = '.z'
else: str_space = '.r'
fout = ''.join([dir, 'plk.groups.', str(mneut), 'eV.', str(nreal), '.nzbin', str(nzbin), str_space, 'space.dat'])
# header
hdr = ''.join(['P_l(k) measurements for m neutrino = ', str(mneut), ' eV, realization ', str(nreal), ', zbin ', str(nzbin),
'\n P_shotnoise ', str(plk['shotnoise']),
'\n cols: k, P_0, P_2, P_4'])
# write to file
np.savetxt(fout, np.array([plk['k'], plk['p0k'], plk['p2k'], plk['p4k']]).T, header=hdr)
return None
def data_nbar(Mr=21, Nmock=500):
'''
Observed nbar measured from the mock 'data' and the covariance matrix
'''
nbar = np.loadtxt(''.join([util.dat_dir(), 'nbar.Mr', str(Mr), '.dat']))
nbar_cov = np.loadtxt(''.join([util.dat_dir(), 'nbar_cov.Mr', str(Mr), '.Nmock', str(Nmock), '.dat']))
return [nbar, nbar_cov]
def beutlerCov(zbin, NorS='ngc', ell=0):
''' Read in Florian's covariance matrix
'''
# read in C_ij from file
if NorS == 'ngc':
f_cov = ''.join([UT.dat_dir(), 'Beutler/public_material_RSD/',
'Beutleretal_cov_patchy_z', str(zbin), '_NGC_1_15_1_15_1_10_2045_60.dat'])
else:
f_cov = ''.join([UT.dat_dir(), 'Beutler/public_material_RSD/',
'Beutleretal_cov_patchy_z', str(zbin), '_SGC_1_15_1_15_1_10_2048_60.dat'])
i_k, j_k, k_i, k_j, C_ij = np.loadtxt(f_cov, skiprows=4, unpack=True, usecols=[0,1,2,3,4])
if ell == 0:
i_l0, i_l1 = 0, 14
elif ell == 2:
i_l0, i_l1 = 14, 28
elif ell == 4:
i_l0, i_l1 = 28, 37
elif ell == 'all':
i_l0, i_l1 = 0, 37
ki = k_i.reshape((int(np.sqrt(len(i_k))), int(np.sqrt(len(i_k)))))
kj = k_j.reshape((int(np.sqrt(len(i_k))), int(np.sqrt(len(i_k)))))
Cij = C_ij.reshape((int(np.sqrt(len(i_k))), int(np.sqrt(len(i_k)))))
#return C_ij.reshape((int(np.sqrt(len(i_k))), int(np.sqrt(len(i_k)))))
return ki[i_l0:i_l1,i_l0:i_l1], kj[i_l0:i_l1,i_l0:i_l1], Cij[i_l0:i_l1,i_l0:i_l1]
def data_gmf(Mr=21, Nmock=500):
'''
This loads the observed GMF from the mock 'data' and the covariance matrix (or sigma) corresponding to that.
Note to self. Return the inverse covariance matrix m**********r!
'''
gmf_dat_file = ''.join([util.dat_dir(), 'gmf.Mr', str(Mr), '.dat'])
gmf_sig_file = ''.join([util.dat_dir(), 'gmf_sigma.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
return [np.loadtxt(gmf_dat_file), np.loadtxt(gmf_sig_file)]
def save_jackknife_covariance(Mr , nsub):
wpcov_filename = util.dat_dir()+'wpcov_AM_Mr'+str(Mr)+'.dat'
nbarcov_filename = util.dat_dir()+'nbarcov_AM_Mr'+str(Mr)+'.dat'
nbarcov , wpcov = compute_jackknife_covariance(Mr , nsub)
np.savetxt(nbarcov_filename , nbarcov)
np.savetxt(wpcov_filename , wpcov)
return None
def save_nbar_clustering(Mr):
'''save nbar and wp measured for
the luminosity threshold Mr'''
wp_filename = util.dat_dir()+'wp_AM_Mr'+str(Mr)+'.dat'
nbar_filename = util.dat_dir()+'nbar_AM_Mr'+str(Mr)+'.dat'
nbar , wp = measure_nbar_clustering(Mr)
np.savetxt(nbar_filename , np.array([nbar]))
np.savetxt(wp_filename , wp)
return None
def mcmc_chains(tag, ichain=None):
''' Given some tag string return mcmc chain in a dictionary
'''
chain_dict = {}
if tag == 'beutler_z1':
# read in Florian's RSD MCMC chains
chain_file = ''.join([
UT.dat_dir(), 'Beutler/public_full_shape/',
'Beutler_et_al_full_shape_analysis_z1_chain',
str(ichain), '.dat'
])
chain = np.loadtxt(chain_file, skiprows=1)
labels = [
'alpha_perp', 'alpha_para', 'fsig8', 'b1sig8_NGC', 'b1sig8_SGC',
'b2sig8_NGC', 'b2sig8_SGC', 'N_NGC', 'N_SGC', 'sigmav_NGC',
'sigmav_SGC', 'chi2'
]
for i in range(len(labels)):
chain_dict[labels[i]] = chain[:, i + 1]
# read in mock evaluations for the chains
chain_model_ngc_file = ''.join([
UT.dat_dir(), 'Beutler/public_full_shape/',
'Beutler_et_al_full_shape_analysis_z1_chain',
str(ichain), '.model.ngc.dat'
])
chain_model_sgc_file = ''.join([
UT.dat_dir(), 'Beutler/public_full_shape/',
'Beutler_et_al_full_shape_analysis_z1_chain',
str(ichain), '.model.sgc.dat'
])
pk_ngc = np.loadtxt(chain_model_ngc_file)
pk_sgc = np.loadtxt(chain_model_sgc_file)
chain_dict['pk_ngc'] = pk_ngc[:, 1:]
chain_dict['pk_sgc'] = pk_sgc[:, 1:]
elif tag == 'manodeep':
# read in Manodeep's GMF MCMC chains
chain_file = ''.join([
UT.dat_dir(), 'manodeep/',
'status_file_Consuelo_so_mvir_Mr19_box_4022_and_4002_fit_wp_0_fit_gmf_1_pca_0.out'
])
chain = np.loadtxt(chain_file, skiprows=5)
labels = ['logMmin', 'sig_logM', 'logM0', 'logM1', 'alpha', 'chi2']
for i, lbl in enumerate(labels):
if lbl == 'chi2': chain_dict[lbl] = -2. * chain[:, i]
else: chain_dict[lbl] = chain[:, i]
chain_dict['gmf'] = chain[:, -8:]
else:
raise ValueError
return chain_dict
def data_xi_full_cov(Mr=21, Nmock=500):
'''
Observed xi (2PCF) from the 'data' and the diagonal elements of the xi covariance matrix
'''
xi_dat_file = ''.join([util.dat_dir(), 'xir.Mr', str(Mr), '.dat'])
xi = np.loadtxt(xi_dat_file, unpack=True)
# load covariance of xi
cov_dat_file = ''.join([util.dat_dir(), 'xir_covariance.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
cov = np.loadtxt(cov_dat_file)
#cii = np.diag(cov) # diagonal elements
return [xi, cov]
def load_wp_covariance(Mr):
''' loads the jackknife covariance matrix associated with wp
Parameters
----------
Mr : (int)
Absolute r-band magnitude threshold. Default is M_r = -21
'''
if Mr == 21.5:
data_file = ''.join([util.dat_dir(),
'wpxicov_dr72_bright0_mr21.5_z0.198_nj400'])
if Mr == 21.:
data_file = ''.join([util.dat_dir(),
'wpxicov_dr72_bright0_mr21.0_z0.159_nj400'])
if Mr == 20.5:
data_file = ''.join([util.dat_dir(),
'wpxicov_dr72_bright0_mr20.5_z0.132_nj400'])
if Mr == 20.:
data_file = ''.join([util.dat_dir(),
'wpxicov_dr72_bright0_mr20.0_z0.106_nj400'])
if Mr == 19.5:
data_file = ''.join([util.dat_dir(),
'wpxicov_dr72_bright0_mr19.5_z0.085_nj400'])
if Mr == 19.:
data_file = ''.join([util.dat_dir(),
'wpxicov_dr72_bright0_mr19.0_z0.064_nj400'])
if Mr == 18.5:
data_file = ''.join([util.dat_dir(),
'wpxicov_dr72_bright0_mr18.5_z0.053_nj400'])
if Mr == 18.:
data_file = ''.join([util.dat_dir(),
'wpxicov_dr72_bright0_mr18.0_z0.041_nj400'])
wpcov = np.loadtxt(data_file)[:12 , :12]
return wpcov
def load_wp(Mr):
''' loads wp
Parameters
----------
Mr : (int)
Absolute r-band magnitude threshold. Default is M_r = -21
'''
if Mr == 21.5:
data_file = ''.join([util.dat_dir(),
'wpxi_dr72_bright0_mr21.5_z0.198_nj400'])
if Mr == 21.:
data_file = ''.join([util.dat_dir(),
'wpxi_dr72_bright0_mr21.0_z0.159_nj400'])
if Mr == 20.5:
data_file = ''.join([util.dat_dir(),
'wpxi_dr72_bright0_mr20.5_z0.132_nj400'])
if Mr == 20.:
data_file = ''.join([util.dat_dir(),
'wpxi_dr72_bright0_mr20.0_z0.106_nj400'])
if Mr == 19.5:
data_file = ''.join([util.dat_dir(),
'wpxi_dr72_bright0_mr19.5_z0.085_nj400'])
if Mr == 19.:
data_file = ''.join([util.dat_dir(),
'wpxi_dr72_bright0_mr19.0_z0.064_nj400'])
if Mr == 18.5:
data_file = ''.join([util.dat_dir(),
'wpxi_dr72_bright0_mr18.5_z0.053_nj400'])
if Mr == 18.:
data_file = ''.join([util.dat_dir(),
'wpxi_dr72_bright0_mr18.0_z0.041_nj400'])
wp = np.loadtxt(data_file)[:,1]
return wp
def CenQue_File(type, cenque='default', downsampled=None):
''' Function for getting the file name of CenQue Files when cenque and type are
specified.
'''
if cenque == 'default':
tf = 7
abcrun = 'RHOssfrfq_TinkerFq_Std'
prior = 'updated'
elif cenque == 'nosmfevol':
tf = 8
abcrun = 'RHOssfrfq_TinkerFq_NOSMFevol'
prior = 'updated'
else:
raise NotImplementedError
# cenque files
if type == 'sfms':
galpop_str = 'sfms'
elif type == 'quenched':
galpop_str = 'quenched'
else:
raise ValueError
if downsampled is None:
down_str = ''
else:
down_str = ''.join(['.down', str(downsampled), 'x'])
file = ''.join([UT.dat_dir(), 'cenque/',
galpop_str, '.centrals.',
'tf', str(tf),
'.abc_', abcrun,
'.prior_', prior,
down_str,
'.hdf5'])
return file
def load_data():
'''load data
'''
data_file = ''.join([util.dat_dir(), 'f160w_25_457_557_457_557_pixels.fits'])
data = pf.open(data_file)[0].data
return data
def measure_nbar_clustering(Mr):
'''measure wp for the galaxy catalog
with the luminosity threshold Mr'''
#Corrfunc settings for wp measurements:
boxsize = 250
nthreads = 4
pimax = 40.0
binfile = path.join(path.dirname(path.abspath(__file__)),
"../", "bin")
autocorr = 1
filename = util.dat_dir()+'AM_Mr'+str(Mr)+'dat'
cat = np.loadtxt(filename)
pos = cat[:,1:4]
x, y, z = pos[:,0], pos[:,1], pos[:,2]
vel = cat[:,4:7]
vx, vy, vz = vel[:,0], vel[:,1], vel[:,2]
#applying RSD:
pos = return_xyz_formatted_array(x, y, z, velocity = vz, velocity_distortion_dimension = 'z')
#enforcing periodic boundary conditions:
pos = enforce_periodicity_of_box(pos, boxsize)
pos = pos.astype(np.float32)
x, y, z = pos[:,0] , pos[:,1] , pos[:,2]
wp_result = _countpairs.countpairs_wp(boxsize, pimax,
nthreads, binfile,
x, y, z)
nbar = 1.*len(pos)/boxsize**3.
wp = np.array(wp_result)[:,3]
return nbar , wp
def build_full_inv_covars(Mr=21, Nmock=50):
'''
Calculate the inverse covariance of full data vectors
for MCMC inference
'''
full_cov = data_full_cov(Mr=Mr, Nmock=Nmock)
N_bins = int(np.sqrt(full_cov.size))
f_unbias = np.float(Nmock - 2. - N_bins)/np.float(Nmock - 1.)
inv_c = solve(np.eye(N_bins) , full_cov) * f_unbias
outfn = ''.join([util.dat_dir(),
'nbar_gmf_xir_inv_cov.Mr', str(Mr),
'.Nmock', str(Nmock), '.dat'])
np.savetxt(outfn, inv_c)
nbgmf_cov = data_nbar_gmf_cov(Mr=Mr, Nmock=Nmock)
N_bins = int(np.sqrt(nbgmf_cov.size))
f_unbias = np.float(Nmock - 2. - N_bins)/np.float(Nmock - 1.)
inv_c = solve(np.eye(N_bins) , nbgmf_cov) * f_unbias
outfn = ''.join([util.dat_dir(),
'nbar_gmf_inv_cov.Mr', str(Mr),
'.Nmock', str(Nmock), '.dat'])
np.savetxt(outfn, inv_c)
nbxi_cov = data_nbar_xi_cov(Mr=Mr, Nmock=Nmock)
N_bins = int(np.sqrt(nbxi_cov.size))
f_unbias = np.float(Nmock - 2. - N_bins)/np.float(Nmock - 1.)
inv_c = solve(np.eye(N_bins) , nbxi_cov) * f_unbias
outfn = ''.join([util.dat_dir(),
'nbar_xi_inv_cov.Mr', str(Mr),
'.Nmock', str(Nmock), '.dat'])
np.savetxt(outfn, inv_c)
gmfxi_cov = data_gmf_xi_cov(Mr=Mr, Nmock=Nmock)
N_bins = int(np.sqrt(gmfxi_cov.size))
f_unbias = np.float(Nmock - 2. - N_bins)/np.float(Nmock - 1.)
inv_c = solve(np.eye(N_bins) , gmfxi_cov) * f_unbias
outfn = ''.join([util.dat_dir(),
'gmf_xi_inv_cov.Mr', str(Mr),
'.Nmock', str(Nmock), '.dat'])
np.savetxt(outfn, inv_c)
return None
def read_catalog():
'''read Age-matching catalog
'''
filename = util.dat_dir()+'Mr19_AM.dat'
cat = np.loadtxt(filename)
return cat
def lnPost(zbin=1):
''' *** TESTED ***
Likelihood and posterior functions can reproduce the chi-squared from
Florian's MCMC chain.
Test that the ln(Posterior) reproduces the chi-squared values of
Florian's MCMC chains
'''
# read in Florian's chains
chain_file = ''.join([
UT.dat_dir(), 'Beutler/public_full_shape/',
'Beutler_et_al_full_shape_analysis_z',
str(zbin), '_chain',
str(0), '.dat'
])
sample = np.loadtxt(chain_file, skiprows=1)
chi2s = sample[:, -1]
sample = sample[:, 1:-1]
# read in BOSS P(k) NGC + SGC
pkay = Dat.Pk()
k0, p0k_ngc = pkay.Observation(0, zbin, 'ngc')
k2, p2k_ngc = pkay.Observation(2, zbin, 'ngc')
k4, p4k_ngc = pkay.Observation(4, zbin, 'ngc')
k0, p0k_sgc = pkay.Observation(0, zbin, 'sgc')
k2, p2k_sgc = pkay.Observation(2, zbin, 'sgc')
k4, p4k_sgc = pkay.Observation(4, zbin, 'sgc')
k_list = [k0, k2, k4]
pk_ngc_list = [p0k_ngc, p2k_ngc, p4k_ngc]
pk_sgc_list = [p0k_sgc, p2k_sgc, p4k_sgc]
# read in Covariance matrix
# currently for testing purposes,
# implemented to read in Florian's covariance matrix
_, _, C_pk_ngc = Dat.beutlerCov(zbin, NorS='ngc', ell='all')
_, _, C_pk_sgc = Dat.beutlerCov(zbin, NorS='sgc', ell='all')
# calculate precision matrices (including the hartlap factor)
n_mocks_ngc = 2045
n_mocks_sgc = 2048
f_hartlap_ngc = (float(n_mocks_ngc) - float(
len(np.concatenate(pk_ngc_list))) - 2.) / (float(n_mocks_ngc) - 1.)
f_hartlap_sgc = (float(n_mocks_sgc) - float(
len(np.concatenate(pk_sgc_list))) - 2.) / (float(n_mocks_sgc) - 1.)
Cinv_ngc = np.linalg.inv(C_pk_ngc)
Cinv_sgc = np.linalg.inv(C_pk_sgc)
Cinv_ngc *= f_hartlap_ngc
Cinv_sgc *= f_hartlap_sgc
lnpost_args = (k_list, pk_ngc_list, pk_sgc_list, Cinv_ngc, Cinv_sgc)
for i in range(10):
print('Like', -2. * Inf.lnLike(sample[i, :], *lnpost_args))
print('Post', -2. * Inf.lnPost(sample[i, :], *lnpost_args))
print('chi2', chi2s[i])
return None
def File(self):
'''
'''
# Write Star forming and Quenching catalog
evol_file = ''.join([UT.dat_dir(), 'galpop/'
'sfms.centrals',
self._Spec_str(),
'.hdf5'])
return evol_file
def data_full_cov(Mr=21, Nmock=500):
full_cov_fn = ''.join([util.dat_dir(),
'nbar_gmf_xir_cov.Mr', str(Mr),
'.Nmock', str(Nmock), '.dat'])
fullcov = np.loadtxt(full_cov_fn)
return fullcov
def data_nbar_xi_cov(Mr=21, Nmock=500):
nbxi_cov_fn = ''.join([util.dat_dir(),
'nbar_xi_cov.Mr', str(Mr),
'.Nmock', str(Nmock), '.dat'])
nbxi_cov = np.loadtxt(nbxi_cov_fn)
return nbxi_cov
def build_xi_bins(Mr=21):
''' hardcoded r bins for xi.
'''
model = PrebuiltHodModelFactory('zheng07', threshold = -1.0*np.float(Mr))
model.populate_mock() # population mock realization
r_bin = model.mock.compute_galaxy_clustering(rbins=hardcoded_xi_bins())[0]
output_file = ''.join([util.dat_dir(), 'xir_rbin.Mr', str(Mr), '.dat'])
np.savetxt(output_file, r_bin)
return None
def save_luminosity_threshold(Mr):
'''save a galaxy catalog with the imposed
luminosity threshold'''
reduced_cat = impose_luminosity_threshold(Mr)
filename = util.dat_dir()+'AM_Mr'+str(Mr)+'dat'
np.savetxt(filename , reduced_cat)
return None
def data_xi_cov(Mr=21, Nmock=500):
'''
Observed xi covariance. The entire covariance matrix
'''
# load covariance of xi
cov_dat_file = ''.join([util.dat_dir(), 'xir_covariance.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
cov = np.loadtxt(cov_dat_file)
return cov
def data_gmf_xi_inv_cov(Mr=21, Nmock=500):
inv_cov_fn = ''.join([util.dat_dir(),
'gmf_xi_inv_cov.Mr', str(Mr),
'.Nmock', str(Nmock), '.dat'])
inv_cov = np.loadtxt(inv_cov_fn)
return inv_cov
def NeutHalo_pre3PCF(mneut, nreal, nzbin, zspace=False, clobber=False):
''' Pre-process halo catalogs for 3PCF run. Read in halo catalog
and output to input file format for Daniel Eisenstein's code.
parameters
----------
zspace : bool, optional
if True, calculates redshift space positions. Otherwise real space.
'''
# output file
if mneut == 0.1: dir = ''.join([UT.dat_dir(), '0.10eV/', str(nreal)])
else: dir = ''.join([UT.dat_dir(), str(mneut), 'eV/', str(nreal)])
if zspace: str_space = '.z'
else: str_space = '.r'
fout = ''.join([dir, '/groups.nzbin', str(nzbin), str_space, 'space.dat'])
if os.path.isfile(fout) and not clobber:
print('--- already written to ---\n %s' % (fout))
return None
# import Neutrino halo with mneut eV, realization # nreal, at z specified by nzbin
halos = Dat.NeutHalos(mneut, nreal, nzbin)
if zspace:
halos['RSDPosition'] = FM.RSD(halos, LOS=[0,0,1])
# halo positions
xyz = np.array(halos['Position'])
x = xyz[:,0]
y = xyz[:,1]
z = xyz[:,2]
# weights (all ones!)
w = np.ones(len(x))
# header
hdr = ''.join(['m neutrino = ', str(mneut), ' eV, realization ', str(nreal), ', zbin ', str(nzbin)])
outarr = np.array([x, y, z, w]).T
# write to file
np.savetxt(fout, outarr, header=hdr)
print('--- halo written to ---\n %s' % (fout))
return None
def load_gmf(Mr):
''' loads wp
Parameters
----------
Mr : (int)
Absolute r-band magnitude threshold. Default is M_r = -21
'''
if Mr == 20.:
data_file = ''.join([util.dat_dir(),
'gmf_mr20.0.dat'])
if Mr == 19.:
data_file = ''.join([util.dat_dir(),
'gmf_mr19.0.dat'])
if Mr == 18.:
data_file = ''.join([util.dat_dir(),
'gmf_mr18.0.dat'])
gmf = np.loadtxt(data_file)[:,2]
return gmf
def Observation(self):
''' Read in gmf(N) measurements of SDSS Mr < -19 from Manodeep
'''
f = ''.join([UT.dat_dir(), 'manodeep/sdss_Mr19_fib0_gmf_northonly.dat'])
nbin_low, nbin_high, gmf = np.loadtxt(f, unpack=True, usecols=[0,1,2])
nbins = np.concatenate([nbin_low, [nbin_high[-1]]])
self.nbin_low_obv = nbin_low
self.nbin_high_obv = nbin_high
self.nbin_obv = nbins
self.gmf_obv = gmf
return nbins, gmf
def load_nbar_variance(Mr, style):
'''load the variance of the number density of the data
'''
if style == "SHAM":
style = "SHAM"
elif style == "AM":
style = "AM"
data_file = ''.join([util.dat_dir(),
"nbarcov_"+str(style)+"_Mr"+str(Mr)+".dat"])
nbarcov = np.loadtxt(data_file)
return nbarcov
def build_xi_nbar_gmf(Mr=21):
'''
Build "data" xi, nbar, GMF values and write to file
'''
model = PrebuiltHodModelFactory('zheng07', threshold = -1.0*np.float(Mr))
model.populate_mock(halocat = halocat , enforce_PBC = False) # population mock realization
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# write xi
rbins = hardcoded_xi_bins()
rmax = rbins.max()
approx_cell1_size = [rmax, rmax, rmax]
approx_cell2_size = approx_cell1_size
approx_cellran_size = [rmax, rmax, rmax]
period = np.array([Lbox , Lbox , Lbox])
data_xir = tpcf(
sample1, rbins, sample2 = sample2,
randoms=randoms, period = period,
max_sample_size=int(1e4), estimator='Landy-Szalay',
approx_cell1_size=approx_cell1_size,
approx_cellran_size=approx_cellran_size,
RR_precomputed = RR,
NR_precomputed = NR1)
output_file = ''.join([util.dat_dir(), 'xir.Mr', str(Mr), '.dat'])
np.savetxt(output_file, data_xir)
# write nbar values
nbar = model.mock.number_density
output_file = ''.join([util.dat_dir(), 'nbar.Mr', str(Mr), '.dat'])
np.savetxt(output_file, [nbar])
# write GMF
rich = richness(model.mock.compute_fof_group_ids())
gmf = GMF(rich) # GMF
output_file = ''.join([util.dat_dir(), 'gmf.Mr', str(Mr), '.dat'])
np.savetxt(output_file, gmf)
return None
def load_gmf_covariance(Mr , pois = True):
''' loads the jackknife covariance matrix associated with gmf
Parameters
----------
Mr : (int)
Absolute r-band magnitude threshold. Default is M_r = -21
pois : True r false, whether we have poisson error in likelihood or not
'''
if Mr == 20.:
data_file = ''.join([util.dat_dir(),
'gmf_mr20.0.dat'])
if Mr == 19.:
data_file = ''.join([util.dat_dir(),
'gmf_mr19.0.dat'])
if Mr == 18.:
data_file = ''.join([util.dat_dir(),
'gmf_mr18.0.dat'])
if pois == True:
gmf_err = (np.loadtxt(data_file)[:,3]**2. + np.loadtxt(data_file)[:,4]**2.)**.5
if pois == False:
gmf_err = np.loadtxt(data_file)[:,3]
return gmf_err ** 2.
def Random(catalog, NorS='ngc'):
''' name of the original catalog files. These will be fed into the FFT
'''
if catalog == 'patchy':
return ''.join([
UT.dat_dir(), 'patchy/Patchy-Mocks-Randoms-DR12',
NorS.upper(), '-COMPSAM_V6C_x50.dat'
])
elif catalog == 'boss':
if NorS == 'ngc':
return ''.join([
UT.catalog_dir('boss'), 'random1_DR12v5_CMASSLOWZTOT_North.dat'
])
else:
raise NotImplementedError
def data_RR(NR):
'''
loads precomputed RR
NR = number of random points
it should probably also accept the size of the box.
note to self: put the size of the box in there later after
merging the sample variance with the rest of the thing
'''
rr = ''.join([util.dat_dir(),
'RR_NR' , str(NR) , '.dat'])
rr = np.loadtxt(rr)
return rr
def data_xi_inv_cov(Mr=21, Nmock=500, unbias_str=True):
'''
Observed inverse covariance of xi with/without the unbias estimator
factor. Default multiplies by the unbias estimator factor.
'''
if unbias_str:
unbias_str = '.unbias'
else:
unbias_str = ''
inv_cov_file = ''.join([util.dat_dir(),
'xi_inv_cov', unbias_str, '.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
inv_cov = np.loadtxt(inv_cov_file)
return inv_cov
def load_wp_covariance(Mr , style):
''' loads the jackknife covariance matrix associated with wp
Parameters
----------
Mr : (int)
Absolute r-band magnitude threshold. Default is M_r = -21
'''
if style == "SHAM":
style = "SHAM"
elif style == "AM":
style = "AM"
data_file = ''.join([util.dat_dir(),
"wpcov_"+str(style)+"_Mr"+str(Mr)+".dat"])
wpcov = np.loadtxt(data_file)
return wpcov
def load_wp(Mr, style):
''' loads wp
Parameters
----------
Mr : (int)
Absolute r-band magnitude threshold. Default is M_r = -21
'''
if style == "SHAM":
style = "SHAM"
elif style == "AM":
style = "AM"
data_file = ''.join([util.dat_dir(),
"wp_"+str(style)+"_Mr"+str(Mr)+".dat"])
wp = np.loadtxt(data_file)
return wp
def Observation(self, ell, zbin, nors):
''' Read in P(k) measurements of BOSS from Florian
'''
if ell == 0:
str_pole = 'mono'
elif ell == 2:
str_pole = 'quadru'
elif ell == 4:
str_pole = 'hexadeca'
str_pole += 'pole'
f = ''.join([UT.dat_dir(), 'Beutler/public_material_RSD/',
'Beutleretal_pk_', str_pole, '_DR12_', nors.upper(), '_z', str(zbin),
'_prerecon_120.dat'])
k_central, k_mean, pk = np.loadtxt(f, skiprows=31, unpack=True, usecols=[0,1,2])
self.k_obv = k_central
self.k_mean_obv = k_mean
self.k_central_obv = k_central
self.pk_obv = pk
return k_central, pk
def compute_jackknife_covariance(Mr , nsub):
box_size = 250.
filename = util.dat_dir()+'AM_Mr'+str(Mr)+'dat'
cat = np.loadtxt(filename)
pos = cat[:,1:4] #positions of all galaxies in the box
vel = cat[:,4:7] #velocities of all galaxies
x , y , z = pos[:,0], pos[:,1], pos[:,2]
vx, vy, vz = vel[:,0], vel[:,1], vel[:,2]
rsd_pos = return_xyz_formatted_array(x, y, z, velocity = vz, velocity_distortion_dimension = 'z')
rsd_pos = enforce_periodicity_of_box(rsd_pos, box_size)
number_of_subboxes = nsub ** 3
#Corrfunc settings for wp measurements:
subbox_size = box_size / nsub
nthreads = 4
pimax = 25.0
binfile = path.join(path.dirname(path.abspath(__file__)),
"../", "bin")
len_wp = len(np.loadtxt(binfile))
wps = np.zeros((number_of_subboxes , len_wp))
nbars = np.zeros((number_of_subboxes , 1))
for subvol_index in xrange(number_of_subboxes):
sub_rsd_pos = mask_positions(rsd_pos , subvol_index , nsub)
sub_rsd_pos = sub_rsd_pos.astype(np.float32)
sub_x, sub_y, sub_z = sub_rsd_pos[:,0] , sub_rsd_pos[:,1] , sub_rsd_pos[:,2]
wp_result = _countpairs.countpairs_wp(subbox_size, pimax,
nthreads, binfile,
sub_x, sub_y, sub_z)
nbars[subvol_index] = 1.*len(sub_rsd_pos)/subbox_size**3.
wps[subvol_index] = np.array(wp_result)[:,3]
nbar_covar = np.array([np.var(nbars)])
wp_covar = np.cov(wps.T)
return nbar_covar , wp_covar
def importance_weight_Pk(tag_like, ichain, zbin=1, ica_algorithm=None,
density_method='kde', n_comp_max=20, info_crit='bic', njobs=1):
''' save importance weights to file
'''
# read in MCMC chain
chain = Inf.mcmc_chains('beutler_z'+str(zbin), ichain=ichain)
ws = Inf.W_importance(tag_like, chain, zbin=zbin, ica_algorithm=ica_algorithm,
density_method=density_method, info_crit=info_crit, njobs=njobs)
if density_method == 'kde':
str_density = '.kde'
elif density_method == 'gmm':
str_density = ''.join(['.gmm_max', str(n_comp_max), 'comp_', info_crit])
str_ica = ''
if ica_algorithm is not None:
str_ica = ''.join(['.', ica_algorithm[:3], 'ICA'])
weight_file = ''.join([UT.dat_dir(), 'Beutler/public_full_shape/',
'Beutler_et_al_full_shape_analysis_z', str(zbin), '_chain', str(ichain),
'.', tag_like, '_weights', str_ica, str_density, '.dat'])
hdr = 'ln(P_denom), ln(P_numer), w_importance'
np.savetxt(weight_file, np.array(ws).T, header=hdr)
return None
def importance_weight_Gmf(tag_like, ica_algorithm=None,
density_method='kde', n_comp_max=20, info_crit='bic', njobs=1):
''' save importance weights for Manodeep's MCMC chain to file
'''
chain = Inf.mcmc_chains('manodeep')
ws = Inf.W_importance(tag_like, chain, ica_algorithm=ica_algorithm,
density_method=density_method, info_crit=info_crit, njobs=njobs)
if density_method == 'kde':
str_density = '.kde'
elif density_method == 'gmm':
str_density = ''.join(['.gmm_max', str(n_comp_max), 'comp_', info_crit])
str_ica = ''
if ica_algorithm is not None:
str_ica = ''.join(['.', ica_algorithm[:3], 'ICA'])
weight_file = ''.join([UT.dat_dir(), 'manodeep/',
'status_file_Consuelo_so_mvir_Mr19_box_4022_and_4002_fit_wp_0_fit_gmf_1_pca_0',
'.', tag_like, '_weights', str_ica, str_density, '.dat'])
hdr = 'ln(P_denom), ln(P_numer), w_importance'
np.savetxt(weight_file, np.array(ws).T, header=hdr)
return None
def build_xi_inv_cov(Mr=21, Nmock=500, unbias=True):
'''
Calculate the inverse covariance of xi multiplied by the unbiased
estimator factor (Nmocks - 2 - Nbins)/(Nmocks - 1).
Mainly used in MCMC inference
'''
xi_cov = data_xi_cov(Mr=Mr, Nmock=Nmock) # covariance matrix of xi
N_bins = int(np.sqrt(xi_cov.size)) # cov matrix is N_bin x N_bin
if unbias:
f_unbias = np.float(Nmock - 2. - N_bins)/np.float(Nmock - 1.)
unbias_str = '.unbias'
else:
f_unbias = 1.0
unbias_str = ''
inv_c = solve(np.eye(N_bins) , xi_cov) * f_unbias
output_file = ''.join([util.dat_dir(),
'xi_inv_cov', unbias_str, '.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
np.savetxt(output_file, inv_c)
return None
def div_K(div_func='kl'):
''' compare the KL or Renyi divergence for the following with their using different K values
- D( gauss(C_X) || gauss(C_X) )
- D( mock X || gauss(C_X))
- D( mock X || p(X) KDE)
- D( mock X || p(X) GMM)
- D( mock X || PI p(X^i_ICA) KDE)
- D( mock X || PI p(X^i_ICA) GMM)
'''
lbls = [r'$D( P(k) \parallel \mathcal{N}({\bf C}))$',
r'$D( P(k) \parallel p_\mathrm{KDE}(P(k)))$',
r'$D( P(k) \parallel p_\mathrm{GMM}(P(k)))$',
r'$D( P(k) \parallel \prod_i p_\mathrm{KDE}(P(k)_i^\mathrm{ICA}))$',
r'$D( P(k) \parallel \prod_i p_\mathrm{GMM}(P(k)_i^\mathrm{ICA}))$']
fig = plt.figure(figsize=(20,4))
for i_obv, obv in enumerate(['pk.ngc', 'gmf']):
if obv == 'pk.ngc':
Nref = 2000
if div_func == 'kl': hranges = [[-0.5, 0.5], [-0.5, 7.], [-0.5, 0.5], [-0.5, 0.5], [-0.5, 0.5]]##7.]
else: hranges = [[-0.5, 0.5] for i in range(5)]
Ks = [5, 10, 15]
elif obv == 'gmf':
Nref = 10000
hranges = [[-0.1, 0.4], [-0.1, 0.4], [-0.1, 0.4], [-0.1, 0.4], [-0.1, 0.4]]##7.]
Ks = [10]
for K in Ks:
fs = ['pX_gauss.K'+str(K), 'pX_scottKDE.K'+str(K), 'pX_GMM.K'+str(K)+'.ncomp30',
'pXi_ICA_scottKDE.K'+str(K), 'pXi_ICA_GMM.K'+str(K)+'.ncomp30']
divs, divs_ref = [], []
for f in fs:
f_div = ''.join([UT.dat_dir(), 'diverg.', obv, '.', f, '.Nref', str(Nref), '.',
div_func, '.dat'])
try:
div = np.loadtxt(f_div)
except IOError:
print f_div
continue
divs.append(div)
nbins = 50
bkgd = fig.add_subplot(2,1,i_obv+1, frameon=False)
for i_div, div, lbl in zip(range(len(fs)), divs, lbls):
sub = fig.add_subplot(2,5,len(fs)*i_obv+i_div+1)
y_max = 0.
hh = np.histogram(div, normed=True, range=hranges[i_div], bins=nbins)
bp = UT.bar_plot(*hh)
sub.fill_between(bp[0], np.zeros(len(bp[0])), bp[1], edgecolor='none')
y_max = max(y_max, bp[1].max())
sub.set_xlim(hranges[i_div])
sub.set_ylim([0., y_max*1.4])
if i_obv == 0:
sub.set_title(lbl)
if div_func == 'kl':
bkgd.set_xlabel(r'KL divergence', fontsize=20, labelpad=20)
elif div_func == 'renyi0.5':
bkgd.set_xlabel(r'R\'enyi-$\alpha$ divergence', fontsize=20, labelpad=20)
bkgd.set_xticklabels([])
bkgd.set_yticklabels([])
bkgd.tick_params(labelcolor='none', top='off', bottom='off', left='off', right='off')
fig.subplots_adjust(wspace=.15, hspace=0.3)
f_fig = ''.join([UT.fig_dir(), 'tests/Ktest_kNNdiverg.', div_func, '.png'])
fig.savefig(f_fig, bbox_inches='tight')
return None
def div_ICA(obv='pk.ngc', K=10, div_func='kl'):
''' compare the KL or Renyi divergence for different ICA decomposition algorithms
FastICA deflation, FastICA parallel, Infomax ICA
- D( mock X || PI p(X^i_ICA) KDE)
- D( mock X || PI p(X^i_ICA) GMM)
'''
if obv == 'pk.ngc': str_obv = 'P(k)'
elif obv == 'gmf': str_obv = '\zeta(N)'
lbls = [r'$D( '+str_obv+' \parallel \prod_i p_\mathrm{KDE}(P(k)_i^\mathrm{ICA}))$',
r'$D( '+str_obv+' \parallel \prod_i p_\mathrm{GMM}(P(k)_i^\mathrm{ICA}))$']
icas = ['ICA', 'parICA']
if obv == 'pk.ngc':
Nref = 2000
hrange = [-0.5, 0.5]
elif obv == 'gmf':
Nref = 10000
hranges = [-0.1, 0.4]
fig = plt.figure(figsize=(10,4))
bkgd = fig.add_subplot(111, frameon=False)
for i_div, str_div in enumerate(['scottKDE.K'+str(K), 'GMM.K'+str(K)+'.ncomp30']):
divs = []
for ica in icas:
f_div = ''.join([UT.dat_dir(), 'diverg.', obv,
'.pXi_', ica, '_', str_div, '.Nref', str(Nref), '.', div_func, '.dat'])
try:
div = np.loadtxt(f_div)
except IOError:
print f_div
continue
divs.append(div)
nbins = 50
sub = fig.add_subplot(1,2,i_div+1)
y_max = 0.
for div, ica in zip(divs, icas):
print np.mean(div)
hh = np.histogram(div, normed=True, range=hrange, bins=nbins)
bp = UT.bar_plot(*hh)
sub.fill_between(bp[0], np.zeros(len(bp[0])), bp[1], edgecolor='none', label=ica)
y_max = max(y_max, bp[1].max())
if i_div == 0: sub.legend(loc='upper left', prop={'size': 20})
sub.set_xlim(hrange)
sub.set_ylim([0., y_max*1.4])
sub.set_title(lbls[i_div])
if div_func == 'kl':
bkgd.set_xlabel(r'KL divergence', fontsize=20, labelpad=20)
elif div_func == 'renyi0.5':
bkgd.set_xlabel(r'R\'enyi-$\alpha$ divergence', fontsize=20, labelpad=20)
bkgd.set_xticklabels([])
bkgd.set_yticklabels([])
bkgd.tick_params(labelcolor='none', top='off', bottom='off', left='off', right='off')
fig.subplots_adjust(wspace=.15, hspace=0.3)
f_fig = ''.join([UT.fig_dir(), 'tests/',
'ICA_kNNdiverg.', obv, '.K', str(K), '.', div_func, '.png'])
fig.savefig(f_fig, bbox_inches='tight')
return None
def data_xi_bins(Mr=21):
'''
r bins for xi(r)
'''
rbin_file = ''.join([util.dat_dir(), 'xir_rbin.Mr', str(Mr), '.dat'])
return np.loadtxt(rbin_file)
def data_gmf_cov(Mr=21, Nmock=500):
'''
Returns the GMF covariance matrix
'''
cov_file = ''.join([util.dat_dir(), 'gmf_cov.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
return np.loadtxt(cov_file)
def build_xi_nbar_gmf_cov(Mr=21, Nmock=500):
'''
Build covariance matrix for xi, variance for nbar, and a bunch of stuff for gmf
...
using Nmock realizations of halotool mocks
'''
xir = []
model = PrebuiltHodModelFactory('zheng07', threshold = -1.0*np.float(Mr))
#nbars = []
#gmfs = []
#gmf_counts = []
for i in xrange(Nmock):
print 'mock#', i
model.populate_mock()
# xi(r)
xir.append(model.mock.compute_galaxy_clustering(rbins=hardcoded_xi_bins())[1])
# nbar
#nbars.append(model.mock.number_density)
# gmf
#rich = richness(model.mock.compute_fof_group_ids())
#gmfs.append(GMF(rich)) # GMF
#gmf_counts.append(GMF(rich, counts=True)) # Group counts
# save xi covariance
xi_covar = np.cov(np.array(xir).T)
output_file = ''.join([util.dat_dir(), 'xir_covariance.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
np.savetxt(output_file, xi_covar)
# save nbar values
#nbar_cov = np.var(nbars, axis=0)
#output_file = ''.join([util.dat_dir(), 'nbar_cov.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(output_file, [nbar_cov])
# write GMF covariance
#gmf_cov = np.cov(np.array(gmfs).T)
#output_file = ''.join([util.dat_dir(), 'gmf_cov.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(output_file, gmf_cov)
# write GMF Poisson
#gmf_counts_mean = np.mean(gmf_counts, axis=0)
#poisson_gmf = np.sqrt(gmf_counts_mean) / 250.**3 # poisson errors
#output_file = ''.join([util.dat_dir(), 'gmf_sigma_poisson.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(output_file, poisson_gmf)
# write GMF standard dev
#sigma_gmf = np.std(gmfs, axis=0) # sample variance
#output_file = ''.join([util.dat_dir(), 'gmf_sigma_stddev.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(output_file, sigma_gmf)
# write GMF total noise
#sigma_tot = (sigma_gmf**2 + poisson_gmf**2)**0.5 # total sigma
#output_file = ''.join([util.dat_dir(), 'gmf_sigma.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(output_file, sigma_tot)
# write full covariance matrix of various combinations of the data
# covariance for all three
#fulldatarr = np.hstack(np.array(nbars).reshape(nbars.shape[0], 1),
# np.array(gmfs), np.array(xir))
#fullcov = np.cov(fulldatarr.T)
#outfn = ''.join([util.dat_dir(), 'nbar_gmf_xir_cov.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(outfn, fullcov)
# covariance for nbar and gmf
##nbgmf_arr = np.hstack(np.array(nbars).reshape(nbars.shape[0], 1),
# np.array(gmfs))
#nbgmf_cov = np.cov(nbgmf_arr.T)
#outfn = ''.join([util.dat_dir(), 'nbar_gmf_cov.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(outfn, nbgmf_cov)
# covariance for nbar and xi
#nbxi_arr = np.hstack(np.array(nbars).reshape(nbars.shape[0], 1),
# np.array(xir))
#nbxi_cov = np.cov(nbxi_arr.T)
#outfn = ''.join([util.dat_dir(), 'nbar_xi_cov.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(outfn, nbxi_cov)
# covariance for gmf and xi
#gmfxi_arr = np.hstack(np.array(gmfs), np.array(xir))
#gmfxi_cov = np.cov(gmfxi_arr.T)
#outfn = ''.join([util.dat_dir(), 'gmf_xi_cov.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(outfn, gmfxi_cov)
return None
def ABCpmc_HOD(T,
eps_val,
N_part=1000,
prior_name='first_try',
observables=['nbar', 'xi'],
data_dict={'Mr': 21},
output_dir=None):
'''
ABC-PMC implementation.
Parameters
----------
- T : Number of iterations
- eps_val :
- N_part : Number of particles
- observables : list of observables. Options are 'nbar', 'gmf', 'xi'
- data_dict : dictionary that specifies the observation keywords
'''
if output_dir is None:
output_dir = util.dat_dir()
else:
pass
#Initializing the vector of observables and inverse covariance matrix
if observables == ['xi']:
fake_obs = Data.data_xi(**data_dict)
fake_obs_cov = Data.data_cov(**data_dict)[1:16, 1:16]
xi_Cii = np.diag(fake_obs_cov)
elif observables == ['nbar', 'xi']:
fake_obs = np.hstack(
[Data.data_nbar(**data_dict),
Data.data_xi(**data_dict)])
fake_obs_cov = Data.data_cov(**data_dict)[:16, :16]
Cii = np.diag(fake_obs_cov)
xi_Cii = Cii[1:]
nbar_Cii = Cii[0]
elif observables == ['nbar', 'gmf']:
fake_obs = np.hstack(
[Data.data_nbar(**data_dict),
Data.data_gmf(**data_dict)])
fake_obs_cov = Data.data_cov('nbar_gmf', **data_dict)
Cii = np.diag(fake_obs_cov)
gmf_Cii = Cii[1:]
nbar_Cii = Cii[0]
# True HOD parameters
data_hod_dict = Data.data_hod_param(Mr=data_dict['Mr'])
data_hod = np.array([
data_hod_dict['logM0'], # log M0
np.log(data_hod_dict['sigma_logM']), # log(sigma)
data_hod_dict['logMmin'], # log Mmin
data_hod_dict['alpha'], # alpha
data_hod_dict['logM1'] # log M1
])
# Priors
prior_min, prior_max = PriorRange(prior_name)
prior = abcpmc.TophatPrior(prior_min, prior_max)
prior_range = np.zeros((len(prior_min), 2))
prior_range[:, 0] = prior_min
prior_range[:, 1] = prior_max
# simulator
our_model = HODsim(Mr=data_dict['Mr']) # initialize model
kwargs = {'prior_range': prior_range, 'observables': observables}
def simz(tt):
sim = our_model.sum_stat(tt, **kwargs)
if sim is None:
pickle.dump(tt, open("simz_crash_theta.p", 'wb'))
pickle.dump(kwargs, open('simz_crash_kwargs.p', 'wb'))
raise ValueError('Simulator is giving NonetType')
return sim
def multivariate_rho(datum, model):
#print datum , model
dists = []
if observables == ['nbar', 'xi']:
dist_nbar = (datum[0] - model[0])**2. / nbar_Cii
dist_xi = np.sum((datum[1:] - model[1:])**2. / xi_Cii)
dists = [dist_nbar, dist_xi]
elif observables == ['nbar', 'gmf']:
dist_nbar = (datum[0] - model[0])**2. / nbar_Cii
dist_gmf = np.sum((datum[1:] - model[1:])**2. / gmf_Cii)
dists = [dist_nbar, dist_gmf]
elif observables == ['xi']:
dist_xi = np.sum((datum - model)**2. / xi_Cii)
dists = [dist_xi]
#print np.array(dists)
return np.array(dists)
mpi_pool = mpi_util.MpiPool()
abcpmc_sampler = abcpmc.Sampler(
N=N_part, #N_particles
Y=fake_obs, #data
postfn=simz, #simulator
dist=multivariate_rho, #distance function
pool=mpi_pool)
abcpmc_sampler.particle_proposal_cls = abcpmc.ParticleProposal
eps = abcpmc.MultiConstEps(T, eps_val)
pools = []
f = open("abc_tolerance.dat", "w")
f.close()
eps_str = ''
for pool in abcpmc_sampler.sample(prior, eps):
#while pool.ratio > 0.01:
new_eps_str = '\t'.join(eps(pool.t).astype('str')) + '\n'
if eps_str != new_eps_str: # if eps is different, open fiel and append
f = open("abc_tolerance.dat", "a")
eps_str = new_eps_str
f.write(eps_str)
f.close()
print("T:{0},ratio: {1:>.4f}".format(pool.t, pool.ratio))
print eps(pool.t)
# plot theta
plot_thetas(pool.thetas,
pool.ws,
pool.t,
Mr=data_dict["Mr"],
truths=data_hod,
plot_range=prior_range,
observables=observables,
output_dir=output_dir)
if (pool.t < 4) and (pool.t > 2):
pool.thetas = np.loadtxt(
"/home/mj/abc/halo/dat/gold/nbar_xi_Mr21_theta_t3.mercer.dat")
pool.ws = np.loadtxt(
"/home/mj/abc/halo/dat/gold/nbar_xi_Mr21_w_t3.mercer.dat")
eps.eps = [1.12132735353, 127.215586776]
# write theta and w to file
theta_file = ''.join([
output_dir,
util.observable_id_flag(observables), '_Mr',
str(data_dict["Mr"]), '_theta_t',
str(pool.t), '.mercer.dat'
])
w_file = ''.join([
output_dir,
util.observable_id_flag(observables), '_Mr',
str(data_dict["Mr"]), '_w_t',
str(pool.t), '.mercer.dat'
])
np.savetxt(theta_file, pool.thetas)
np.savetxt(w_file, pool.ws)
if pool.t < 3:
eps.eps = np.percentile(np.atleast_2d(pool.dists), 50, axis=0)
elif (pool.t > 2) and (pool.t < 20):
eps.eps = np.percentile(np.atleast_2d(pool.dists), 75, axis=0)
abcpmc_sampler.particle_proposal_cls = abcpmc.ParticleProposal
else:
eps.eps = np.percentile(np.atleast_2d(pool.dists), 90, axis=0)
abcpmc_sampler.particle_proposal_cls = abcpmc.ParticleProposal
#if eps.eps < eps_min:
# eps.eps = eps_min
pools.append(pool)
#abcpmc_sampler.close()
return pools
