def bossPk(zbin):
''' calculate boss P(k) using nbody kit
'''
path_to_catalogs = UT.catalog_dir('boss')
data = FITSCatalog(
os.path.join(path_to_catalogs,
'galaxy_DR12v5_CMASSLOWZTOT_North.fits'))
print('data columns = ', data.columns)
randoms = FITSCatalog(
os.path.join(path_to_catalogs,
'random1_DR12v5_CMASSLOWZTOT_North.fits'))
print('randoms columns = ', randoms.columns)
# the fiducial BOSS DR12 cosmology
cosmo = cosmology.Cosmology(H0=67.6, Om0=0.31, flat=True)
# add Cartesian position column
data['Position'] = transform.SkyToCartesian(data['RA'],
data['DEC'],
data['Z'],
cosmo=cosmo)
randoms['Position'] = transform.SkyToCartesian(randoms['RA'],
randoms['DEC'],
randoms['Z'],
cosmo=cosmo)
randoms['WEIGHT'] = 1.0
data['WEIGHT'] = data['WEIGHT_SYSTOT'] * (data['WEIGHT_NOZ'] +
data['WEIGHT_CP'] - 1.0)
if zbin == 1:
ZMIN, ZMAX = 0.2, 0.5
elif zbin == 2:
ZMIN, ZMAX = 0.4, 0.6
randoms['Selection'] = (randoms['Z'] > ZMIN) & (randoms['Z'] < ZMAX)
data['Selection'] = (data['Z'] > ZMIN) & (data['Z'] < ZMAX)
# combine the data and randoms into a single catalog
fkp = FKPCatalog(data, randoms)
mesh = fkp.to_mesh(Nmesh=256,
nbar='NZ',
fkp_weight='WEIGHT_FKP',
comp_weight='WEIGHT',
window='tsc')
# compute the multipoles
r = ConvolvedFFTPower(mesh, poles=[0, 2, 4], dk=0.01, kmin=0.)
poles = r.poles
for ell in [0, 2, 4]:
P = poles['power_%d' % ell].real
if ell == 0: P = P - r.attrs['shotnoise']
np.savetxt(''.join(
[UT.catalog_dir('boss'), 'nbodykit.p',
str(ell), 'k.dat']), (np.vstack(np.array([poles['k'], P]))).T,
fmt=['%f', '%f'],
delimiter='\t')
return None
def Catalog(catalog, n_mock, NorS='ngc'):
''' name of the original catalog files. These will be fed into the FFT
'''
if 'patchy' in catalog:
return ''.join([
UT.catalog_dir('patchy'), 'Patchy-Mocks-DR12',
NorS.upper(), '-COMPSAM_V6C_',
str("%04d" % n_mock), '.dat'
])
elif catalog == 'boss':
if NorS == 'ngc':
return ''.join([
UT.catalog_dir('boss'), 'galaxy_DR12v5_CMASSLOWZTOT_North.dat'
])
else:
raise NotImplementedError
def Read(self, name, ireal, ibox):
''' Read in ireal-th realization and ibox-th box mock catalog
from run-th run.
'''
f = ''.join([UT.catalog_dir(name), self._file_name(name, ireal, ibox)])
gmf, nbin_low, nbin_high = np.loadtxt(f, unpack=True, usecols=[0,1,2])
self.gmf = gmf
self.nbin_low = nbin_low
self.nbin_high = nbin_high
self.nbins = np.concatenate([nbin_low, [nbin_high[-1]]])
return None
def boss_preprocess(NorS='ngc'):
''' read in and pre-process boss data
'''
if NorS == 'ngc':
str_NorS = 'North'
elif NorS == 'sgc':
str_NorS = 'South'
# read in original data in fits file format
f_orig = ''.join([
UT.catalog_dir('boss'), 'galaxy_DR12v5_CMASSLOWZTOT_', str_NorS,
'.fits'
])
data = mrdfits(f_orig)
# data columns: ra,dec,az,nbb,wsys,wnoz,wcp,comp
data_list = [
data.ra, data.dec, data.z, data.nz, data.weight_systot,
data.weight_noz, data.weight_cp, data.comp
]
data_fmt = ['%f', '%f', '%f', '%e', '%f', '%f', '%f', '%f']
header_str = "columns: ra,dec,az,nbb,wsys,wnoz,wcp,comp"
f_boss = ''.join(
[UT.catalog_dir('boss'), 'galaxy_DR12v5_CMASSLOWZTOT_North.dat'])
np.savetxt(f_boss, (np.vstack(np.array(data_list))).T,
fmt=data_fmt,
delimiter='\t',
header=header_str)
# now random catalog
f_orig = ''.join([
UT.catalog_dir('boss'), 'random1_DR12v5_CMASSLOWZTOT_', str_NorS,
'.fits.gz'
])
#data = mrdfits(f_orig)
# data columns: ra,dec,az,nbb,wsys,wnoz,wcp,comp
#data_list = [data.ra, data.dec, data.z, data.nz, data.weight_systot, data.weight_noz, data.weight_cp, data.comp]
return None
def Random(catalog, NorS='ngc'):
''' name of the original catalog files. These will be fed into the FFT
'''
if catalog == 'patchy':
return ''.join([
UT.dat_dir(), 'patchy/Patchy-Mocks-Randoms-DR12',
NorS.upper(), '-COMPSAM_V6C_x50.dat'
])
elif catalog == 'boss':
if NorS == 'ngc':
return ''.join([
UT.catalog_dir('boss'), 'random1_DR12v5_CMASSLOWZTOT_North.dat'
])
else:
raise NotImplementedError
def Read(self, name, i, ell=0, sys=None, NorS='ngc'):
''' Read in the i th mock catalog P(k)
'''
if ell not in [0, 2, 4]:
raise ValueError("ell can only be 0, 2, or 4")
#f = self._compressed_read(name, i, ell, sys)
f = ''.join([UT.catalog_dir(name), self._file_name(name, i, NorS, sys)])
if 'patchy' in name:
k, pk = np.loadtxt(f, skiprows=24, unpack=True, usecols=[0,1+ell/2]) # k, P_l(k)
else:
if ell == 0:
i_ell = 5
else:
i_ell = 1 + ell/2
k, pk, counts = np.loadtxt(f, unpack=True, usecols=[0, i_ell, -2]) # k, p0(k), and number of modes
self.counts = counts
self.k = k
self.pk = pk
return None
def Pk_NBKT_patchy(i_mock, zbin, NorS, randoms=None):
''' calculate Patchy mock P(k) using nbodykit, which uses Nick's estimator
'''
plk_name = ''.join([
UT.catalog_dir('patchy'), 'pk.patchy.',
str(i_mock), '.', NorS, '.zbin',
str(zbin), '.nbodykit.dat'
])
if os.path.isfile(plk_name):
return None
print('patchy --', i_mock)
# first read in data and random catalogs
col_data = ['RA', 'DEC', 'Z', 'DUM0', 'NZ', 'DUM1', 'VETO', 'WRED']
data = CSVCatalog(Catalog('patchy', i_mock, NorS=NorS), col_data)
if randoms is None:
col_random = ['RA', 'DEC', 'Z', 'NZ', 'BIAS', 'VETO', 'WRED']
randoms = CSVCatalog(Random('patchy', NorS=NorS), col_random)
# impose fiducial BOSS DR12 cosmology
cosmo = cosmology.Cosmology(H0=67.6, Om0=0.31, flat=True)
# add Cartesian position column
data['Position'] = transform.SkyToCartesian(data['RA'],
data['DEC'],
data['Z'],
cosmo=cosmo)
randoms['Position'] = transform.SkyToCartesian(randoms['RA'],
randoms['DEC'],
randoms['Z'],
cosmo=cosmo)
randoms['WEIGHT'] = randoms['WRED']
randoms['WEIGHT_FKP'] = 1. / (1. + randoms['NZ'] * 1e4)
data['WEIGHT'] = data['WRED']
data['WEIGHT_FKP'] = 1. / (1. + data['NZ'] * 1e4)
if zbin == 1:
ZMIN, ZMAX = 0.2, 0.5
elif zbin == 2:
ZMIN, ZMAX = 0.4, 0.6
elif zbin == 3:
ZMIN, ZMAX = 0.5, 0.75
randoms['Selection'] = (randoms['Z'] > ZMIN) & (randoms['Z'] < ZMAX) & (
randoms['VETO'] > 0)
data['Selection'] = (data['Z'] > ZMIN) & (data['Z'] < ZMAX) & (data['VETO']
> 0)
# combine the data and randoms into a single catalog
fkp = FKPCatalog(data, randoms)
mesh = fkp.to_mesh(Nmesh=360,
nbar='NZ',
fkp_weight='WEIGHT_FKP',
comp_weight='WRED',
window='tsc')
# compute the multipoles
r = ConvolvedFFTPower(mesh, poles=[0, 2, 4], dk=0.01, kmin=0.)
poles = r.poles
plk = []
for ell in [0, 2, 4]:
P = poles['power_%d' % ell].real
if ell == 0: P = P - r.attrs['shotnoise']
plk.append(P)
f = open(plk_name, 'w')
f.write("### header ### \n")
for key in r.attrs:
f.write("%s = %s \n" % (key, str(r.attrs[key])))
f.write("columns : k , P0, P2, P4 \n")
f.write('### header ### \n')
for ik in range(len(poles['k'])):
f.write("%f \t %f \t %f \t %f" %
(poles['k'][ik], plk[0][ik], plk[1][ik], plk[2][ik]))
f.write("\n")
f.close()
print('wrote:', plk_name)
return None
def Pk_NBKT_boss(zbin, NorS='ngc'):
''' calculate boss P(k) using nbodykit, which uses Nick's estimator
'''
# first read in data and random catalogs
path_to_catalogs = UT.catalog_dir('boss')
if NorS == 'ngc': str_nors = 'North'
else: str_nors = 'South'
data = FITSCatalog(
os.path.join(path_to_catalogs,
'galaxy_DR12v5_CMASSLOWZTOT_' + str_nors + '.fits'))
randoms = FITSCatalog(
os.path.join(path_to_catalogs,
'random1_DR12v5_CMASSLOWZTOT_' + str_nors + '.fits'))
# selection effects
if zbin == 1:
ZMIN, ZMAX = 0.2, 0.5
elif zbin == 2:
ZMIN, ZMAX = 0.4, 0.6
randoms['Selection'] = (randoms['Z'] > ZMIN) & (randoms['Z'] < ZMAX)
randoms = randoms[randoms['Selection']]
data['Selection'] = (data['Z'] > ZMIN) & (data['Z'] < ZMAX)
data = data[data['Selection']]
# impose fiducial BOSS DR12 cosmology
cosmo = cosmology.Cosmology(H0=67.6, Om0=0.31, flat=True)
# add Cartesian position column
data['Position'] = transform.SkyToCartesian(data['RA'],
data['DEC'],
data['Z'],
cosmo=cosmo)
randoms['Position'] = transform.SkyToCartesian(randoms['RA'],
randoms['DEC'],
randoms['Z'],
cosmo=cosmo)
randoms['WEIGHT'] = 1.0
data['WEIGHT'] = data['WEIGHT_SYSTOT'] * (data['WEIGHT_NOZ'] +
data['WEIGHT_CP'] - 1.0)
# combine the data and randoms into a single catalog
fkp = FKPCatalog(data, randoms)
mesh = fkp.to_mesh(Nmesh=360,
nbar='NZ',
fkp_weight='WEIGHT_FKP',
comp_weight='WEIGHT',
window='tsc',
selection='Selection')
# compute the multipoles
r = ConvolvedFFTPower(mesh, poles=[0, 2, 4], dk=0.01, kmin=0.)
poles = r.poles
plk = []
for ell in [0, 2, 4]:
P = poles['power_%d' % ell].real
if ell == 0: P = P - r.attrs['shotnoise']
plk.append(P)
# shot noise comparison
#print(r.attrs['shotnoise'])
#alpha = r.attrs['alpha']
#sel = mesh['randoms'][mesh.selection]
#first = mesh['randoms'][sel]
#comp_weight = first[mesh.comp_weight]
#fkp_weight = first[mesh.fkp_weight]
#S = (comp_weight**2*fkp_weight**2).sum()
#print("expected SN")
#print((1.+alpha)*alpha*S/r.attrs['randoms.norm'])
#sel = mesh['data'][mesh.selection]
#first = mesh['data'][sel]
#comp_weight = first[mesh.comp_weight]
#fkp_weight = first[mesh.fkp_weight]
#S_data = (comp_weight**2*fkp_weight**2).sum()
#print("true SN")
#print((S_data+alpha**2*S)/r.attrs['randoms.norm'])
f = open(
''.join([
UT.catalog_dir('boss'), 'pk.nbodykit.', NorS, '.zbin',
str(zbin), '.dat'
]), 'w')
f.write("### header ### \n")
for key in r.attrs:
f.write("%s = %s \n" % (key, str(r.attrs[key])))
f.write("columns : k , P0, P2, P4 \n")
f.write('### header ### \n')
for ik in range(len(poles['k'])):
f.write("%f \t %f \t %f \t %f" %
(poles['k'][ik], plk[0][ik], plk[1][ik], plk[2][ik]))
f.write("\n")
f.close()
return None
def patchyCov(zbin, NorS='ngc', ell=0, clobber=False):
''' Construct covariance matrix for patchy mocks measured using Nbodykit
'''
catalog = 'patchy.z'+str(zbin)
f_cov = ''.join([UT.catalog_dir(catalog), 'Cov_pk.', catalog, '.ell', str(ell), '.', NorS, '.NBKT.dat'])
if os.path.isfile(f_cov) and not clobber:
i_k, j_k, k_i, k_j, C_ij = np.loadtxt(f_cov, skiprows=4, unpack=True, usecols=[0,1,2,3,-1])
ki = k_i.reshape((int(np.sqrt(len(i_k))), int(np.sqrt(len(i_k)))))
kj = k_j.reshape((int(np.sqrt(len(i_k))), int(np.sqrt(len(i_k)))))
Cij = C_ij.reshape((int(np.sqrt(len(i_k))), int(np.sqrt(len(i_k)))))
return ki, kj, Cij
else:
# calculate my patchy covariance
pkay = Pk()
n_mock = pkay._n_mock(catalog)
i_mock, n_missing = 0, 0
for i in range(1,n_mock+1):
try:
pkay.Read(catalog, i, ell=ell, NorS=NorS, sys='fc')
pkay.krange([0.01,0.15])
k = pkay.k
pk = pkay.pk
n_kbin = len(k)
if i == 1:
ks = np.zeros((n_mock, n_kbin))
pks = np.zeros((n_mock, n_kbin))
ks[i_mock,:] = k
pks[i_mock,:] = pk
i_mock += 1
except IOError:
if i == 1:
raise ValueError
print('missing -- ', pkay._file_name(catalog, i, 'fc'))
n_missing += 1
print(n_missing, "P(k) files missing")
n_mock -= n_missing
if n_missing > 0:
pks = pks[:n_mock,:]
#mu_pk = np.sum(pks, axis=0)/np.float(n_mock)
#Cov_pk = np.dot((pks-mu_pk).T, pks-mu_pk)/float(n_mock-1)
Cov_pk = np.cov(pks.T)
# write to file
f = open(f_cov, 'w')
f.write('### header ### \n')
f.write('Covariance matrix for ell = '+str(ell)+' calculated from the '+str(n_mock)+' mocks \n')
f.write('5 columns: {measured power spectrum bin index i} {measured power spectrum bin index j} k_i k_j C_ij \n')
f.write('### header ### \n')
k_i, k_j = [], []
for i in range(Cov_pk.shape[0]):
for j in range(Cov_pk.shape[1]):
f.write('%i \t %i \t %f \t %f \t %e' % (i+1, j+1, k[i], k[j], Cov_pk[i,j]))
f.write('\n')
k_i.append(k[i])
k_j.append(k[j])
f.close()
k_i = np.array(k_i)
k_j = np.array(k_j)
ki = k_i.reshape(Cov_pk.shape)
kj = k_j.reshape(Cov_pk.shape)
return ki, kj, Cov_pk
def _tarfile(self, name):
''' tar ball that contains all the powerspectrum measurements.
with/without systematics, everything. P(k) files are pretty small
so who cares.
'''
return ''.join([UT.catalog_dir(name), 'power_', name, '.tar'])
