def build_nbar_xi_gmf_cov(Mr=21):
''' Build covariance matrix for the full nbar, xi, gmf data vector
using realisations of galaxy mocks for "data" HOD parameters in the
halos from the multidark simulation. Covariance matrices for different sets of observables
can be extracted from the full covariance matrix by slicing through
the indices.
'''
nbars = []
xir = []
gmfs = []
thr = -1. * np.float(Mr)
model = PrebuiltHodModelFactory('zheng07', threshold=thr)
halocat = CachedHaloCatalog(simname='multidark',
redshift=0,
halo_finder='rockstar')
#some settings for tpcf calculations
rbins = hardcoded_xi_bins()
for i in xrange(1, 125):
print 'mock#', i
# populate the mock subvolume
model.populate_mock(halocat)
# returning the positions of galaxies
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# calculate nbar
nbars.append(len(pos) / 1000**3.)
# translate the positions of randoms to the new subbox
#calculate xi(r)
xi = tpcf(pos,
rbins,
period=model.mock.Lbox,
max_sample_size=int(2e5),
estimator='Landy-Szalay')
xir.append(xi)
# calculate gmf
nbar = len(pos) / 1000**3.
b_normal = 0.75
b = b_normal * (nbar)**(-1. / 3)
groups = pyfof.friends_of_friends(pos, b)
w = np.array([len(x) for x in groups])
gbins = gmf_bins()
gmf = np.histogram(w, gbins)[0] / (1000.**3.)
gmfs.append(gmf) # GMF
# save nbar variance
nbar_var = np.var(nbars, axis=0, ddof=1)
nbar_file = ''.join(
[util.multidat_dir(), 'abc_nbar_var.Mr',
str(Mr), '.dat'])
np.savetxt(nbar_file, [nbar_var])
# write full covariance matrix of various combinations of the data
# and invert for the likelihood evaluations
# --- covariance for all three ---
fulldatarr = np.hstack(
(np.array(nbars).reshape(len(nbars),
1), np.array(xir), np.array(gmfs)))
fullcov = np.cov(fulldatarr.T)
fullcorr = np.corrcoef(fulldatarr.T)
# and save the covariance matrix
nopoisson_file = ''.join([
util.multidat_dir(), 'abc_nbar_xi_gmf_cov.no_poisson.Mr',
str(Mr), '.dat'
])
np.savetxt(nopoisson_file, fullcov)
# and a correlation matrix
full_corr_file = ''.join(
[util.multidat_dir(), 'abc_nbar_xi_gmf_corr.Mr',
str(Mr), '.dat'])
np.savetxt(full_corr_file, fullcorr)
return None
def build_nbar_xi_gmf_cov(Mr=21):
''' Build covariance matrix for the full nbar, xi, gmf data vector
using realisations of galaxy mocks for "data" HOD parameters in the
halos from the multidark simulation. Covariance matrices for different sets of observables
can be extracted from the full covariance matrix by slicing through
the indices.
'''
nbars = []
xir = []
gmfs = []
thr = -1. * np.float(Mr)
model = PrebuiltHodModelFactory('zheng07', threshold=thr)
halocat = CachedHaloCatalog(simname = 'multidark', redshift = 0, halo_finder = 'rockstar')
#some settings for tpcf calculations
rbins = hardcoded_xi_bins()
for i in xrange(1,125):
print 'mock#', i
# populate the mock subvolume
model.populate_mock(halocat)
# returning the positions of galaxies
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# calculate nbar
nbars.append(len(pos) / 1000**3.)
# translate the positions of randoms to the new subbox
#calculate xi(r)
xi = tpcf(pos, rbins, period = model.mock.Lbox,
max_sample_size=int(2e5), estimator='Landy-Szalay')
xir.append(xi)
# calculate gmf
nbar = len(pos) / 1000**3.
b_normal = 0.75
b = b_normal * (nbar)**(-1./3)
groups = pyfof.friends_of_friends(pos , b)
w = np.array([len(x) for x in groups])
gbins = gmf_bins()
gmf = np.histogram(w , gbins)[0] / (1000.**3.)
gmfs.append(gmf) # GMF
# save nbar variance
nbar_var = np.var(nbars, axis=0, ddof=1)
nbar_file = ''.join([util.multidat_dir(), 'abc_nbar_var.Mr', str(Mr), '.dat'])
np.savetxt(nbar_file, [nbar_var])
# write full covariance matrix of various combinations of the data
# and invert for the likelihood evaluations
# --- covariance for all three ---
fulldatarr = np.hstack((np.array(nbars).reshape(len(nbars), 1),
np.array(xir),
np.array(gmfs)))
fullcov = np.cov(fulldatarr.T)
fullcorr = np.corrcoef(fulldatarr.T)
# and save the covariance matrix
nopoisson_file = ''.join([util.multidat_dir(), 'abc_nbar_xi_gmf_cov.no_poisson.Mr', str(Mr), '.dat'])
np.savetxt(nopoisson_file, fullcov)
# and a correlation matrix
full_corr_file = ''.join([util.multidat_dir(), 'abc_nbar_xi_gmf_corr.Mr', str(Mr), '.dat'])
np.savetxt(full_corr_file, fullcorr)
return None
def plot_covars(data_dict={'Mr':20}, harmattan=True):
'''
Plot the data and the covariance matrices generated for the analysis.
'''
# first get them dataz
covfn = ''.join([util.multidat_dir(),
'nbar_xir_gmf_corr.Mr', str(data_dict['Mr']),
'.dat'])
fullcovar = np.loadtxt(covfn)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_axis_off()
caxr = ax.imshow(fullcovar, cmap=plt.cm.viridis)
divr = make_axes_locatable(plt.gca())
cbarr = divr.append_axes('right', '5%', pad='3%')
fig.colorbar(caxr, cax=cbarr)
picname = ''.join(['full_data_covar'
'.Mr', str(data_dict['Mr']),
'.pdf'])
plotaddress = ''.join([util.fig_dir(), picname])
fig.savefig(plotaddress)
if harmattan:
os.system("scp {0} broiler:~/public_html/tinker/".format(plotaddress))
os.system("ssh broiler chmod 644 /home/kilian/public_html/tinker/{0}"
.format(picname))
# --- and for the sub matrices ---
# the nbar-xi correlation matrix
covfn = ''.join([util.multidat_dir(),
'nbar_xi_corr.Mr', str(data_dict['Mr']),
'.dat'])
fullcovar = np.loadtxt(covfn)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_axis_off()
caxr = ax.imshow(fullcovar, cmap=plt.cm.viridis)
divr = make_axes_locatable(plt.gca())
cbarr = divr.append_axes('right', '5%', pad='3%')
fig.colorbar(caxr, cax=cbarr)
picname = ''.join(['nbar_xi_covar'
'.Mr', str(data_dict['Mr']),
'.pdf'])
plotaddress = ''.join([util.fig_dir(), picname])
fig.savefig(plotaddress)
if harmattan:
os.system("scp {0} broiler:~/public_html/tinker/".format(plotaddress))
os.system("ssh broiler chmod 644 /home/kilian/public_html/tinker/{0}"
.format(picname))
# the nbar-gmf correlation matrix
covfn = ''.join([util.multidat_dir(),
'nbar_gmf_corr.Mr', str(data_dict['Mr']),
'.dat'])
fullcovar = np.loadtxt(covfn)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_axis_off()
caxr = ax.imshow(fullcovar, cmap=plt.cm.viridis)
divr = make_axes_locatable(plt.gca())
cbarr = divr.append_axes('right', '5%', pad='3%')
fig.colorbar(caxr, cax=cbarr)
picname = ''.join(['nbar_gmf_covar'
'.Mr', str(data_dict['Mr']),
'.pdf'])
plotaddress = ''.join([util.fig_dir(), picname])
fig.savefig(plotaddress)
if harmattan:
os.system("scp {0} broiler:~/public_html/tinker/".format(plotaddress))
os.system("ssh broiler chmod 644 /home/kilian/public_html/tinker/{0}"
.format(picname))
# the xi-gmf correlation matrix
covfn = ''.join([util.multidat_dir(),
'xi_gmf_corr.Mr', str(data_dict['Mr']),
'.dat'])
fullcovar = np.loadtxt(covfn)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_axis_off()
caxr = ax.imshow(fullcovar, cmap=plt.cm.viridis)
divr = make_axes_locatable(plt.gca())
cbarr = divr.append_axes('right', '5%', pad='3%')
fig.colorbar(caxr, cax=cbarr)
picname = ''.join(['xi_gmf_covar'
'.Mr', str(data_dict['Mr']),
'.pdf'])
plotaddress = ''.join([util.fig_dir(), picname])
fig.savefig(plotaddress)
if harmattan:
os.system("scp {0} broiler:~/public_html/tinker/".format(plotaddress))
os.system("ssh broiler chmod 644 /home/kilian/public_html/tinker/{0}"
.format(picname))
# the xi correlation matrix
covfn = ''.join([util.multidat_dir(),
'xi_corr.Mr', str(data_dict['Mr']),
'.dat'])
fullcovar = np.loadtxt(covfn)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_axis_off()
caxr = ax.imshow(fullcovar, cmap=plt.cm.viridis)
divr = make_axes_locatable(plt.gca())
cbarr = divr.append_axes('right', '5%', pad='3%')
fig.colorbar(caxr, cax=cbarr)
picname = ''.join(['xi_covar'
'.Mr', str(data_dict['Mr']),
'.pdf'])
plotaddress = ''.join([util.fig_dir(), picname])
fig.savefig(plotaddress)
if harmattan:
os.system("scp {0} broiler:~/public_html/tinker/".format(plotaddress))
os.system("ssh broiler chmod 644 /home/kilian/public_html/tinker/{0}"
.format(picname))
# the gmf correlation matrix
covfn = ''.join([util.multidat_dir(),
'gmf_corr.Mr', str(data_dict['Mr']),
'.dat'])
fullcovar = np.loadtxt(covfn)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_axis_off()
caxr = ax.imshow(fullcovar, cmap=plt.cm.viridis)
divr = make_axes_locatable(plt.gca())
cbarr = divr.append_axes('right', '5%', pad='3%')
fig.colorbar(caxr, cax=cbarr)
picname = ''.join(['gmf_covar'
'.Mr', str(data_dict['Mr']),
'.pdf'])
plotaddress = ''.join([util.fig_dir(), picname])
fig.savefig(plotaddress)
if harmattan:
os.system("scp {0} broiler:~/public_html/tinker/".format(plotaddress))
os.system("ssh broiler chmod 644 /home/kilian/public_html/tinker/{0}"
.format(picname))
def plot_covars(data_dict={'Mr': 20}, harmattan=True):
'''
Plot the data and the covariance matrices generated for the analysis.
'''
# first get them dataz
covfn = ''.join([
util.multidat_dir(), 'nbar_xir_gmf_corr.Mr',
str(data_dict['Mr']), '.dat'
])
fullcovar = np.loadtxt(covfn)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_axis_off()
caxr = ax.imshow(fullcovar, cmap=plt.cm.viridis)
divr = make_axes_locatable(plt.gca())
cbarr = divr.append_axes('right', '5%', pad='3%')
fig.colorbar(caxr, cax=cbarr)
picname = ''.join(['full_data_covar' '.Mr', str(data_dict['Mr']), '.pdf'])
plotaddress = ''.join([util.fig_dir(), picname])
fig.savefig(plotaddress)
if harmattan:
os.system("scp {0} broiler:~/public_html/tinker/".format(plotaddress))
os.system(
"ssh broiler chmod 644 /home/kilian/public_html/tinker/{0}".format(
picname))
# --- and for the sub matrices ---
# the nbar-xi correlation matrix
covfn = ''.join(
[util.multidat_dir(), 'nbar_xi_corr.Mr',
str(data_dict['Mr']), '.dat'])
fullcovar = np.loadtxt(covfn)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_axis_off()
caxr = ax.imshow(fullcovar, cmap=plt.cm.viridis)
divr = make_axes_locatable(plt.gca())
cbarr = divr.append_axes('right', '5%', pad='3%')
fig.colorbar(caxr, cax=cbarr)
picname = ''.join(['nbar_xi_covar' '.Mr', str(data_dict['Mr']), '.pdf'])
plotaddress = ''.join([util.fig_dir(), picname])
fig.savefig(plotaddress)
if harmattan:
os.system("scp {0} broiler:~/public_html/tinker/".format(plotaddress))
os.system(
"ssh broiler chmod 644 /home/kilian/public_html/tinker/{0}".format(
picname))
# the nbar-gmf correlation matrix
covfn = ''.join([
util.multidat_dir(), 'nbar_gmf_corr.Mr',
str(data_dict['Mr']), '.dat'
])
fullcovar = np.loadtxt(covfn)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_axis_off()
caxr = ax.imshow(fullcovar, cmap=plt.cm.viridis)
divr = make_axes_locatable(plt.gca())
cbarr = divr.append_axes('right', '5%', pad='3%')
fig.colorbar(caxr, cax=cbarr)
picname = ''.join(['nbar_gmf_covar' '.Mr', str(data_dict['Mr']), '.pdf'])
plotaddress = ''.join([util.fig_dir(), picname])
fig.savefig(plotaddress)
if harmattan:
os.system("scp {0} broiler:~/public_html/tinker/".format(plotaddress))
os.system(
"ssh broiler chmod 644 /home/kilian/public_html/tinker/{0}".format(
picname))
# the xi-gmf correlation matrix
covfn = ''.join(
[util.multidat_dir(), 'xi_gmf_corr.Mr',
str(data_dict['Mr']), '.dat'])
fullcovar = np.loadtxt(covfn)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_axis_off()
caxr = ax.imshow(fullcovar, cmap=plt.cm.viridis)
divr = make_axes_locatable(plt.gca())
cbarr = divr.append_axes('right', '5%', pad='3%')
fig.colorbar(caxr, cax=cbarr)
picname = ''.join(['xi_gmf_covar' '.Mr', str(data_dict['Mr']), '.pdf'])
plotaddress = ''.join([util.fig_dir(), picname])
fig.savefig(plotaddress)
if harmattan:
os.system("scp {0} broiler:~/public_html/tinker/".format(plotaddress))
os.system(
"ssh broiler chmod 644 /home/kilian/public_html/tinker/{0}".format(
picname))
# the xi correlation matrix
covfn = ''.join(
[util.multidat_dir(), 'xi_corr.Mr',
str(data_dict['Mr']), '.dat'])
fullcovar = np.loadtxt(covfn)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_axis_off()
caxr = ax.imshow(fullcovar, cmap=plt.cm.viridis)
divr = make_axes_locatable(plt.gca())
cbarr = divr.append_axes('right', '5%', pad='3%')
fig.colorbar(caxr, cax=cbarr)
picname = ''.join(['xi_covar' '.Mr', str(data_dict['Mr']), '.pdf'])
plotaddress = ''.join([util.fig_dir(), picname])
fig.savefig(plotaddress)
if harmattan:
os.system("scp {0} broiler:~/public_html/tinker/".format(plotaddress))
os.system(
"ssh broiler chmod 644 /home/kilian/public_html/tinker/{0}".format(
picname))
# the gmf correlation matrix
covfn = ''.join(
[util.multidat_dir(), 'gmf_corr.Mr',
str(data_dict['Mr']), '.dat'])
fullcovar = np.loadtxt(covfn)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.set_axis_off()
caxr = ax.imshow(fullcovar, cmap=plt.cm.viridis)
divr = make_axes_locatable(plt.gca())
cbarr = divr.append_axes('right', '5%', pad='3%')
fig.colorbar(caxr, cax=cbarr)
picname = ''.join(['gmf_covar' '.Mr', str(data_dict['Mr']), '.pdf'])
plotaddress = ''.join([util.fig_dir(), picname])
fig.savefig(plotaddress)
if harmattan:
os.system("scp {0} broiler:~/public_html/tinker/".format(plotaddress))
os.system(
"ssh broiler chmod 644 /home/kilian/public_html/tinker/{0}".format(
picname))
