def build_ABC_cov_nbar_xi_gmf(Mr=21, b_normal=0.25):
''' Build covariance matrix used in ABC for the full nbar, xi, gmf data vector
using realisations of galaxy mocks for "data" HOD parameters in the
halos from the multidark simulation. Covariance matrices for different sets of observables
can be extracted from the full covariance matrix by slicing through
the indices.
Notes
-----
* This covariance matrix is the covariance matrix calculated from the *entire* multidark
box. So this does _not_ account for the sample variance, which the MCMC covariance does.
'''
nbars, xir, gmfs = [], [], []
thr = -1. * np.float(Mr)
model = PrebuiltHodModelFactory('zheng07', threshold=thr)
halocat = CachedHaloCatalog(simname='multidark',
redshift=0,
halo_finder='rockstar')
rbins = xi_binedges() # some setting for tpcf calculations
rmax = rbins.max()
approx_cell1_size = [rmax, rmax, rmax]
approx_cellran_size = [rmax, rmax, rmax]
# load randoms and RRs for the ENTIRE MultiDark volume
###randoms = data_random(box='md_all')
###RR = data_RR(box='md_all')
###NR = len(randoms)
for i in xrange(1, 125):
print 'mock#', i
# populate the mock subvolume
model.populate_mock(halocat)
# returning the positions of galaxies
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# calculate nbar
nbars.append(len(pos) / 1000**3.)
# calculate xi(r) for the ENTIRE MultiDark volume
# using the natural estimator DD/RR - 1
xi = tpcf(pos,
rbins,
period=model.mock.Lbox,
max_sample_size=int(3e5),
estimator='Natural',
approx_cell1_size=approx_cell1_size)
xir.append(xi)
# calculate gmf
nbar = len(pos) / 1000**3.
b = b_normal * (nbar)**(-1. / 3)
groups = pyfof.friends_of_friends(pos, b)
w = np.array([len(x) for x in groups])
gbins = gmf_bins()
gmf = np.histogram(w, gbins)[0] / (1000.**3.)
gmfs.append(gmf) # GMF
# save nbar variance
nbar_var = np.var(nbars, axis=0, ddof=1)
nbar_file = ''.join([util.obvs_dir(), 'abc_nbar_var.Mr', str(Mr), '.dat'])
np.savetxt(nbar_file, [nbar_var])
# write full covariance matrix of various combinations of the data
# and invert for the likelihood evaluations
# --- covariance for all three ---
fulldatarr = np.hstack(
(np.array(nbars).reshape(len(nbars),
1), np.array(xir), np.array(gmfs)))
fullcov = np.cov(fulldatarr.T)
fullcorr = np.corrcoef(fulldatarr.T)
# and save the covariance matrix
nopoisson_file = ''.join([
util.obvs_dir(), 'ABC.nbar_xi_gmf_cov', '.no_poisson', '.Mr',
str(Mr), '.bnorm',
str(round(b_normal, 2)), '.dat'
])
np.savetxt(nopoisson_file, fullcov)
return None
def data_gmf_bins():
''' Just for consistency, returns the bins
'''
return gmf_bins()
def build_nbar_xi_gmf(Mr=21, b_normal=0.25):
''' Build data vector [nbar, xi, gmf] and save to file
This data vector is built from the zeroth slice of the multidark
The other slices will be used for building the covariance matrix.
Parameters
----------
Mr : (int)
Absolute magnitude cut off M_r. Default M_r = -21.
b_normal : (float)
FoF Linking length
'''
thr = -1. * np.float(Mr)
model = PrebuiltHodModelFactory('zheng07', threshold=thr)
halocat = CachedHaloCatalog(simname='multidark',
redshift=0,
halo_finder='rockstar')
####model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}
####datsubvol = lambda x: util.mask_func(x, 0)
####model.populate_mock(halocat, masking_function=datsubvol, enforce_PBC=False)
model.populate_mock(halocat)
#all the things necessary for tpcf calculation
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
#masking the galaxies outside the subvolume 0
pos = util.mask_galaxy_table(pos, 0)
rbins = xi_binedges()
rmax = rbins.max()
approx_cell1_size = [rmax, rmax, rmax]
approx_cellran_size = [rmax, rmax, rmax]
#compute number density
nbar = len(pos) / 200**3.
# load MD subvolume randoms and RRs
randoms = data_random(box='md_sub')
RR = data_RR(box='md_sub')
NR = len(randoms)
#compue tpcf with Natural estimator
data_xir = tpcf(pos,
rbins,
pos,
randoms=randoms,
period=None,
max_sample_size=int(2e5),
estimator='Natural',
approx_cell1_size=approx_cell1_size,
approx_cellran_size=approx_cellran_size,
RR_precomputed=RR,
NR_precomputed=NR)
fullvec = np.append(nbar, data_xir)
#compute gmf
b = b_normal * (nbar)**(-1. / 3)
groups = pyfof.friends_of_friends(pos, b)
w = np.array([len(x) for x in groups])
gbins = gmf_bins()
gmf = np.histogram(w, gbins)[0] / (200.**3.)
fullvec = np.append(fullvec, gmf)
output_file = data_file(Mr=Mr, b_normal=b_normal)
np.savetxt(output_file, fullvec)
return None
def build_MCMC_cov_nbar_xi_gmf(Mr=21, b_normal=0.25):
''' Build covariance matrix used in MCMC for the full nbar, xi, gmf data vector
using realisations of galaxy mocks for "data" HOD parameters in the
halos from the other subvolumes (subvolume 1 to subvolume 125) of
the simulation. Covariance matrices for different sets of observables
can be extracted from the full covariance matrix by slicing through
the indices.
'''
nbars = []
xir = []
gmfs = []
thr = -1. * np.float(Mr)
model = PrebuiltHodModelFactory('zheng07', threshold=thr)
halocat = CachedHaloCatalog(simname='multidark',
redshift=0,
halo_finder='rockstar')
###model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}
#some settings for tpcf calculations
rbins = xi_binedges()
rmax = rbins.max()
approx_cell1_size = [rmax, rmax, rmax]
approx_cellran_size = [rmax, rmax, rmax]
#load randoms and RRs
randoms = data_random(box='md_sub')
RR = data_RR(box='md_sub')
NR = len(randoms)
for i in xrange(1, 125):
print 'mock#', i
# populate the mock subvolume
###mocksubvol = lambda x: util.mask_func(x, i)
###model.populate_mock(halocat,
### masking_function=mocksubvol,
### enforce_PBC=False)
model.populate_mock(halocat)
# returning the positions of galaxies in the entire volume
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# masking out the galaxies outside the subvolume i
pos = util.mask_galaxy_table(pos, i)
# calculate nbar
print "shape of pos", pos.shape
nbars.append(len(pos) / 200**3.)
# translate the positions of randoms to the new subbox
xi0, yi0, zi0 = util.random_shifter(i)
temp_randoms = randoms.copy()
temp_randoms[:, 0] += xi0
temp_randoms[:, 1] += yi0
temp_randoms[:, 2] += zi0
#calculate xi(r)
xi = tpcf(pos,
rbins,
pos,
randoms=temp_randoms,
period=None,
max_sample_size=int(3e5),
estimator='Natural',
approx_cell1_size=approx_cell1_size,
approx_cellran_size=approx_cellran_size,
RR_precomputed=RR,
NR_precomputed=NR)
xir.append(xi)
# calculate gmf
nbar = len(pos) / 200**3.
b = b_normal * (nbar)**(-1. / 3)
groups = pyfof.friends_of_friends(pos, b)
w = np.array([len(x) for x in groups])
gbins = gmf_bins()
gmf = np.histogram(w, gbins)[0] / 200.**3.
gmfs.append(gmf)
# save nbar variance
nbar_var = np.var(nbars, axis=0, ddof=1)
nbar_file = ''.join([util.obvs_dir(), 'nbar_var.Mr', str(Mr), '.dat'])
np.savetxt(nbar_file, [nbar_var])
# write full covariance matrix of various combinations of the data
# and invert for the likelihood evaluations
# --- covariance for all three ---
fulldatarr = np.hstack(
(np.array(nbars).reshape(len(nbars),
1), np.array(xir), np.array(gmfs)))
fullcov = np.cov(fulldatarr.T)
fullcorr = np.corrcoef(fulldatarr.T)
# and save the covariance matrix
nopoisson_file = ''.join([
util.obvs_dir(), 'MCMC.nbar_xi_gmf_cov', '.no_poisson', '.Mr',
str(Mr), '.bnorm',
str(round(b_normal, 2)), '.dat'
])
np.savetxt(nopoisson_file, fullcov)
return None
def build_nbar_xi_gmf_cov(Mr=21):
''' Build covariance matrix for the full nbar, xi, gmf data vector
using realisations of galaxy mocks for "data" HOD parameters in the
halos from the multidark simulation. Covariance matrices for different sets of observables
can be extracted from the full covariance matrix by slicing through
the indices.
'''
nbars = []
xir = []
gmfs = []
thr = -1. * np.float(Mr)
model = PrebuiltHodModelFactory('zheng07', threshold=thr)
halocat = CachedHaloCatalog(simname='multidark',
redshift=0,
halo_finder='rockstar')
#some settings for tpcf calculations
rbins = hardcoded_xi_bins()
for i in xrange(1, 125):
print 'mock#', i
# populate the mock subvolume
model.populate_mock(halocat)
# returning the positions of galaxies
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# calculate nbar
nbars.append(len(pos) / 1000**3.)
# translate the positions of randoms to the new subbox
#calculate xi(r)
xi = tpcf(pos,
rbins,
period=model.mock.Lbox,
max_sample_size=int(2e5),
estimator='Landy-Szalay')
xir.append(xi)
# calculate gmf
nbar = len(pos) / 1000**3.
b_normal = 0.75
b = b_normal * (nbar)**(-1. / 3)
groups = pyfof.friends_of_friends(pos, b)
w = np.array([len(x) for x in groups])
gbins = gmf_bins()
gmf = np.histogram(w, gbins)[0] / (1000.**3.)
gmfs.append(gmf) # GMF
# save nbar variance
nbar_var = np.var(nbars, axis=0, ddof=1)
nbar_file = ''.join(
[util.multidat_dir(), 'abc_nbar_var.Mr',
str(Mr), '.dat'])
np.savetxt(nbar_file, [nbar_var])
# write full covariance matrix of various combinations of the data
# and invert for the likelihood evaluations
# --- covariance for all three ---
fulldatarr = np.hstack(
(np.array(nbars).reshape(len(nbars),
1), np.array(xir), np.array(gmfs)))
fullcov = np.cov(fulldatarr.T)
fullcorr = np.corrcoef(fulldatarr.T)
# and save the covariance matrix
nopoisson_file = ''.join([
util.multidat_dir(), 'abc_nbar_xi_gmf_cov.no_poisson.Mr',
str(Mr), '.dat'
])
np.savetxt(nopoisson_file, fullcov)
# and a correlation matrix
full_corr_file = ''.join(
[util.multidat_dir(), 'abc_nbar_xi_gmf_corr.Mr',
str(Mr), '.dat'])
np.savetxt(full_corr_file, fullcorr)
return None
def build_nbar_xi_gmf_cov(Mr=21):
''' Build covariance matrix for the full nbar, xi, gmf data vector
using realisations of galaxy mocks for "data" HOD parameters in the
halos from the multidark simulation. Covariance matrices for different sets of observables
can be extracted from the full covariance matrix by slicing through
the indices.
'''
nbars = []
xir = []
gmfs = []
thr = -1. * np.float(Mr)
model = PrebuiltHodModelFactory('zheng07', threshold=thr)
halocat = CachedHaloCatalog(simname = 'multidark', redshift = 0, halo_finder = 'rockstar')
#some settings for tpcf calculations
rbins = hardcoded_xi_bins()
for i in xrange(1,125):
print 'mock#', i
# populate the mock subvolume
model.populate_mock(halocat)
# returning the positions of galaxies
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# calculate nbar
nbars.append(len(pos) / 1000**3.)
# translate the positions of randoms to the new subbox
#calculate xi(r)
xi = tpcf(pos, rbins, period = model.mock.Lbox,
max_sample_size=int(2e5), estimator='Landy-Szalay')
xir.append(xi)
# calculate gmf
nbar = len(pos) / 1000**3.
b_normal = 0.75
b = b_normal * (nbar)**(-1./3)
groups = pyfof.friends_of_friends(pos , b)
w = np.array([len(x) for x in groups])
gbins = gmf_bins()
gmf = np.histogram(w , gbins)[0] / (1000.**3.)
gmfs.append(gmf) # GMF
# save nbar variance
nbar_var = np.var(nbars, axis=0, ddof=1)
nbar_file = ''.join([util.multidat_dir(), 'abc_nbar_var.Mr', str(Mr), '.dat'])
np.savetxt(nbar_file, [nbar_var])
# write full covariance matrix of various combinations of the data
# and invert for the likelihood evaluations
# --- covariance for all three ---
fulldatarr = np.hstack((np.array(nbars).reshape(len(nbars), 1),
np.array(xir),
np.array(gmfs)))
fullcov = np.cov(fulldatarr.T)
fullcorr = np.corrcoef(fulldatarr.T)
# and save the covariance matrix
nopoisson_file = ''.join([util.multidat_dir(), 'abc_nbar_xi_gmf_cov.no_poisson.Mr', str(Mr), '.dat'])
np.savetxt(nopoisson_file, fullcov)
# and a correlation matrix
full_corr_file = ''.join([util.multidat_dir(), 'abc_nbar_xi_gmf_corr.Mr', str(Mr), '.dat'])
np.savetxt(full_corr_file, fullcorr)
return None
def build_ABC_cov_nbar_xi_gmf(Mr=21, b_normal=0.25):
''' Build covariance matrix used in ABC for the full nbar, xi, gmf data vector
using realisations of galaxy mocks for "data" HOD parameters in the
halos from the multidark simulation. Covariance matrices for different sets of observables
can be extracted from the full covariance matrix by slicing through
the indices.
Notes
-----
* This covariance matrix is the covariance matrix calculated from the *entire* multidark
box. So this does _not_ account for the sample variance, which the MCMC covariance does.
'''
nbars, xir, gmfs = [], [], []
thr = -1. * np.float(Mr)
model = PrebuiltHodModelFactory('zheng07', threshold=thr)
halocat = CachedHaloCatalog(simname = 'multidark', redshift = 0, halo_finder = 'rockstar')
rbins = xi_binedges() # some setting for tpcf calculations
rmax = rbins.max()
approx_cell1_size = [rmax , rmax , rmax]
approx_cellran_size = [rmax , rmax , rmax]
# load randoms and RRs for the ENTIRE MultiDark volume
###randoms = data_random(box='md_all')
###RR = data_RR(box='md_all')
###NR = len(randoms)
for i in xrange(1,125):
print 'mock#', i
# populate the mock subvolume
model.populate_mock(halocat)
# returning the positions of galaxies
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# calculate nbar
nbars.append(len(pos) / 1000**3.)
# calculate xi(r) for the ENTIRE MultiDark volume
# using the natural estimator DD/RR - 1
xi = tpcf(
pos, rbins, period=model.mock.Lbox,
max_sample_size=int(3e5), estimator='Natural',
approx_cell1_size=approx_cell1_size)
xir.append(xi)
# calculate gmf
nbar = len(pos) / 1000**3.
b = b_normal * (nbar)**(-1./3)
groups = pyfof.friends_of_friends(pos , b)
w = np.array([len(x) for x in groups])
gbins = gmf_bins()
gmf = np.histogram(w , gbins)[0] / (1000.**3.)
gmfs.append(gmf) # GMF
# save nbar variance
nbar_var = np.var(nbars, axis=0, ddof=1)
nbar_file = ''.join([util.obvs_dir(), 'abc_nbar_var.Mr', str(Mr), '.dat'])
np.savetxt(nbar_file, [nbar_var])
# write full covariance matrix of various combinations of the data
# and invert for the likelihood evaluations
# --- covariance for all three ---
fulldatarr = np.hstack((np.array(nbars).reshape(len(nbars), 1),
np.array(xir),
np.array(gmfs)))
fullcov = np.cov(fulldatarr.T)
fullcorr = np.corrcoef(fulldatarr.T)
# and save the covariance matrix
nopoisson_file = ''.join([util.obvs_dir(),
'ABC.nbar_xi_gmf_cov', '.no_poisson', '.Mr', str(Mr), '.bnorm', str(round(b_normal, 2)), '.dat'])
np.savetxt(nopoisson_file, fullcov)
return None
def build_MCMC_cov_nbar_xi_gmf(Mr=21, b_normal=0.25):
''' Build covariance matrix used in MCMC for the full nbar, xi, gmf data vector
using realisations of galaxy mocks for "data" HOD parameters in the
halos from the other subvolumes (subvolume 1 to subvolume 125) of
the simulation. Covariance matrices for different sets of observables
can be extracted from the full covariance matrix by slicing through
the indices.
'''
nbars = []
xir = []
gmfs = []
thr = -1. * np.float(Mr)
model = PrebuiltHodModelFactory('zheng07', threshold=thr)
halocat = CachedHaloCatalog(simname = 'multidark', redshift = 0, halo_finder = 'rockstar')
###model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}
#some settings for tpcf calculations
rbins = xi_binedges()
rmax = rbins.max()
approx_cell1_size = [rmax , rmax , rmax]
approx_cellran_size = [rmax , rmax , rmax]
#load randoms and RRs
randoms = data_random(box='md_sub')
RR = data_RR(box='md_sub')
NR = len(randoms)
for i in xrange(1,125):
print 'mock#', i
# populate the mock subvolume
###mocksubvol = lambda x: util.mask_func(x, i)
###model.populate_mock(halocat,
### masking_function=mocksubvol,
### enforce_PBC=False)
model.populate_mock(halocat)
# returning the positions of galaxies in the entire volume
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# masking out the galaxies outside the subvolume i
pos = util.mask_galaxy_table(pos , i)
# calculate nbar
print "shape of pos" , pos.shape
nbars.append(len(pos) / 200**3.)
# translate the positions of randoms to the new subbox
xi0 , yi0 , zi0 = util.random_shifter(i)
temp_randoms = randoms.copy()
temp_randoms[:,0] += xi0
temp_randoms[:,1] += yi0
temp_randoms[:,2] += zi0
#calculate xi(r)
xi=tpcf(
pos, rbins, pos,
randoms=temp_randoms, period=None,
max_sample_size=int(3e5), estimator='Natural',
approx_cell1_size=approx_cell1_size,
approx_cellran_size=approx_cellran_size,
RR_precomputed = RR,
NR_precomputed = NR)
xir.append(xi)
# calculate gmf
nbar = len(pos) / 200**3.
b = b_normal * (nbar)**(-1./3)
groups = pyfof.friends_of_friends(pos , b)
w = np.array([len(x) for x in groups])
gbins = gmf_bins()
gmf = np.histogram(w , gbins)[0] / 200.**3.
gmfs.append(gmf)
# save nbar variance
nbar_var = np.var(nbars, axis=0, ddof=1)
nbar_file = ''.join([util.obvs_dir(), 'nbar_var.Mr', str(Mr), '.dat'])
np.savetxt(nbar_file, [nbar_var])
# write full covariance matrix of various combinations of the data
# and invert for the likelihood evaluations
# --- covariance for all three ---
fulldatarr = np.hstack((np.array(nbars).reshape(len(nbars), 1),
np.array(xir),
np.array(gmfs)))
fullcov = np.cov(fulldatarr.T)
fullcorr = np.corrcoef(fulldatarr.T)
# and save the covariance matrix
nopoisson_file = ''.join([util.obvs_dir(),
'MCMC.nbar_xi_gmf_cov', '.no_poisson', '.Mr', str(Mr), '.bnorm', str(round(b_normal,2)), '.dat'])
np.savetxt(nopoisson_file, fullcov)
return None
def build_nbar_xi_gmf(Mr=21, b_normal=0.25):
''' Build data vector [nbar, xi, gmf] and save to file
This data vector is built from the zeroth slice of the multidark
The other slices will be used for building the covariance matrix.
Parameters
----------
Mr : (int)
Absolute magnitude cut off M_r. Default M_r = -21.
b_normal : (float)
FoF Linking length
'''
thr = -1. * np.float(Mr)
model = PrebuiltHodModelFactory('zheng07', threshold=thr)
halocat = CachedHaloCatalog(simname = 'multidark', redshift = 0, halo_finder = 'rockstar')
####model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}
####datsubvol = lambda x: util.mask_func(x, 0)
####model.populate_mock(halocat, masking_function=datsubvol, enforce_PBC=False)
model.populate_mock(halocat)
#all the things necessary for tpcf calculation
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
#masking the galaxies outside the subvolume 0
pos = util.mask_galaxy_table(pos , 0)
rbins = xi_binedges()
rmax = rbins.max()
approx_cell1_size = [rmax , rmax , rmax]
approx_cellran_size = [rmax , rmax , rmax]
#compute number density
nbar = len(pos) / 200**3.
# load MD subvolume randoms and RRs
randoms = data_random(box='md_sub')
RR = data_RR(box='md_sub')
NR = len(randoms)
#compue tpcf with Natural estimator
data_xir = tpcf(
pos, rbins, pos,
randoms=randoms, period=None,
max_sample_size=int(2e5), estimator='Natural',
approx_cell1_size=approx_cell1_size,
approx_cellran_size=approx_cellran_size,
RR_precomputed=RR,
NR_precomputed=NR)
fullvec = np.append(nbar, data_xir)
#compute gmf
b = b_normal * (nbar)**(-1./3)
groups = pyfof.friends_of_friends(pos , b)
w = np.array([len(x) for x in groups])
gbins = gmf_bins()
gmf = np.histogram(w , gbins)[0] / (200.**3.)
fullvec = np.append(fullvec, gmf)
output_file = data_file(Mr=Mr, b_normal=b_normal)
np.savetxt(output_file, fullvec)
return None
