def bossPk(zbin): ''' calculate boss P(k) using nbody kit ''' path_to_catalogs = UT.catalog_dir('boss') data = FITSCatalog( os.path.join(path_to_catalogs, 'galaxy_DR12v5_CMASSLOWZTOT_North.fits')) print('data columns = ', data.columns) randoms = FITSCatalog( os.path.join(path_to_catalogs, 'random1_DR12v5_CMASSLOWZTOT_North.fits')) print('randoms columns = ', randoms.columns) # the fiducial BOSS DR12 cosmology cosmo = cosmology.Cosmology(H0=67.6, Om0=0.31, flat=True) # add Cartesian position column data['Position'] = transform.SkyToCartesian(data['RA'], data['DEC'], data['Z'], cosmo=cosmo) randoms['Position'] = transform.SkyToCartesian(randoms['RA'], randoms['DEC'], randoms['Z'], cosmo=cosmo) randoms['WEIGHT'] = 1.0 data['WEIGHT'] = data['WEIGHT_SYSTOT'] * (data['WEIGHT_NOZ'] + data['WEIGHT_CP'] - 1.0) if zbin == 1: ZMIN, ZMAX = 0.2, 0.5 elif zbin == 2: ZMIN, ZMAX = 0.4, 0.6 randoms['Selection'] = (randoms['Z'] > ZMIN) & (randoms['Z'] < ZMAX) data['Selection'] = (data['Z'] > ZMIN) & (data['Z'] < ZMAX) # combine the data and randoms into a single catalog fkp = FKPCatalog(data, randoms) mesh = fkp.to_mesh(Nmesh=256, nbar='NZ', fkp_weight='WEIGHT_FKP', comp_weight='WEIGHT', window='tsc') # compute the multipoles r = ConvolvedFFTPower(mesh, poles=[0, 2, 4], dk=0.01, kmin=0.) poles = r.poles for ell in [0, 2, 4]: P = poles['power_%d' % ell].real if ell == 0: P = P - r.attrs['shotnoise'] np.savetxt(''.join( [UT.catalog_dir('boss'), 'nbodykit.p', str(ell), 'k.dat']), (np.vstack(np.array([poles['k'], P]))).T, fmt=['%f', '%f'], delimiter='\t') return None
def Catalog(catalog, n_mock, NorS='ngc'): ''' name of the original catalog files. These will be fed into the FFT ''' if 'patchy' in catalog: return ''.join([ UT.catalog_dir('patchy'), 'Patchy-Mocks-DR12', NorS.upper(), '-COMPSAM_V6C_', str("%04d" % n_mock), '.dat' ]) elif catalog == 'boss': if NorS == 'ngc': return ''.join([ UT.catalog_dir('boss'), 'galaxy_DR12v5_CMASSLOWZTOT_North.dat' ]) else: raise NotImplementedError
def Read(self, name, ireal, ibox): ''' Read in ireal-th realization and ibox-th box mock catalog from run-th run. ''' f = ''.join([UT.catalog_dir(name), self._file_name(name, ireal, ibox)]) gmf, nbin_low, nbin_high = np.loadtxt(f, unpack=True, usecols=[0,1,2]) self.gmf = gmf self.nbin_low = nbin_low self.nbin_high = nbin_high self.nbins = np.concatenate([nbin_low, [nbin_high[-1]]]) return None
def boss_preprocess(NorS='ngc'): ''' read in and pre-process boss data ''' if NorS == 'ngc': str_NorS = 'North' elif NorS == 'sgc': str_NorS = 'South' # read in original data in fits file format f_orig = ''.join([ UT.catalog_dir('boss'), 'galaxy_DR12v5_CMASSLOWZTOT_', str_NorS, '.fits' ]) data = mrdfits(f_orig) # data columns: ra,dec,az,nbb,wsys,wnoz,wcp,comp data_list = [ data.ra, data.dec, data.z, data.nz, data.weight_systot, data.weight_noz, data.weight_cp, data.comp ] data_fmt = ['%f', '%f', '%f', '%e', '%f', '%f', '%f', '%f'] header_str = "columns: ra,dec,az,nbb,wsys,wnoz,wcp,comp" f_boss = ''.join( [UT.catalog_dir('boss'), 'galaxy_DR12v5_CMASSLOWZTOT_North.dat']) np.savetxt(f_boss, (np.vstack(np.array(data_list))).T, fmt=data_fmt, delimiter='\t', header=header_str) # now random catalog f_orig = ''.join([ UT.catalog_dir('boss'), 'random1_DR12v5_CMASSLOWZTOT_', str_NorS, '.fits.gz' ]) #data = mrdfits(f_orig) # data columns: ra,dec,az,nbb,wsys,wnoz,wcp,comp #data_list = [data.ra, data.dec, data.z, data.nz, data.weight_systot, data.weight_noz, data.weight_cp, data.comp] return None
def Random(catalog, NorS='ngc'): ''' name of the original catalog files. These will be fed into the FFT ''' if catalog == 'patchy': return ''.join([ UT.dat_dir(), 'patchy/Patchy-Mocks-Randoms-DR12', NorS.upper(), '-COMPSAM_V6C_x50.dat' ]) elif catalog == 'boss': if NorS == 'ngc': return ''.join([ UT.catalog_dir('boss'), 'random1_DR12v5_CMASSLOWZTOT_North.dat' ]) else: raise NotImplementedError
def Read(self, name, i, ell=0, sys=None, NorS='ngc'): ''' Read in the i th mock catalog P(k) ''' if ell not in [0, 2, 4]: raise ValueError("ell can only be 0, 2, or 4") #f = self._compressed_read(name, i, ell, sys) f = ''.join([UT.catalog_dir(name), self._file_name(name, i, NorS, sys)]) if 'patchy' in name: k, pk = np.loadtxt(f, skiprows=24, unpack=True, usecols=[0,1+ell/2]) # k, P_l(k) else: if ell == 0: i_ell = 5 else: i_ell = 1 + ell/2 k, pk, counts = np.loadtxt(f, unpack=True, usecols=[0, i_ell, -2]) # k, p0(k), and number of modes self.counts = counts self.k = k self.pk = pk return None
def Pk_NBKT_patchy(i_mock, zbin, NorS, randoms=None): ''' calculate Patchy mock P(k) using nbodykit, which uses Nick's estimator ''' plk_name = ''.join([ UT.catalog_dir('patchy'), 'pk.patchy.', str(i_mock), '.', NorS, '.zbin', str(zbin), '.nbodykit.dat' ]) if os.path.isfile(plk_name): return None print('patchy --', i_mock) # first read in data and random catalogs col_data = ['RA', 'DEC', 'Z', 'DUM0', 'NZ', 'DUM1', 'VETO', 'WRED'] data = CSVCatalog(Catalog('patchy', i_mock, NorS=NorS), col_data) if randoms is None: col_random = ['RA', 'DEC', 'Z', 'NZ', 'BIAS', 'VETO', 'WRED'] randoms = CSVCatalog(Random('patchy', NorS=NorS), col_random) # impose fiducial BOSS DR12 cosmology cosmo = cosmology.Cosmology(H0=67.6, Om0=0.31, flat=True) # add Cartesian position column data['Position'] = transform.SkyToCartesian(data['RA'], data['DEC'], data['Z'], cosmo=cosmo) randoms['Position'] = transform.SkyToCartesian(randoms['RA'], randoms['DEC'], randoms['Z'], cosmo=cosmo) randoms['WEIGHT'] = randoms['WRED'] randoms['WEIGHT_FKP'] = 1. / (1. + randoms['NZ'] * 1e4) data['WEIGHT'] = data['WRED'] data['WEIGHT_FKP'] = 1. / (1. + data['NZ'] * 1e4) if zbin == 1: ZMIN, ZMAX = 0.2, 0.5 elif zbin == 2: ZMIN, ZMAX = 0.4, 0.6 elif zbin == 3: ZMIN, ZMAX = 0.5, 0.75 randoms['Selection'] = (randoms['Z'] > ZMIN) & (randoms['Z'] < ZMAX) & ( randoms['VETO'] > 0) data['Selection'] = (data['Z'] > ZMIN) & (data['Z'] < ZMAX) & (data['VETO'] > 0) # combine the data and randoms into a single catalog fkp = FKPCatalog(data, randoms) mesh = fkp.to_mesh(Nmesh=360, nbar='NZ', fkp_weight='WEIGHT_FKP', comp_weight='WRED', window='tsc') # compute the multipoles r = ConvolvedFFTPower(mesh, poles=[0, 2, 4], dk=0.01, kmin=0.) poles = r.poles plk = [] for ell in [0, 2, 4]: P = poles['power_%d' % ell].real if ell == 0: P = P - r.attrs['shotnoise'] plk.append(P) f = open(plk_name, 'w') f.write("### header ### \n") for key in r.attrs: f.write("%s = %s \n" % (key, str(r.attrs[key]))) f.write("columns : k , P0, P2, P4 \n") f.write('### header ### \n') for ik in range(len(poles['k'])): f.write("%f \t %f \t %f \t %f" % (poles['k'][ik], plk[0][ik], plk[1][ik], plk[2][ik])) f.write("\n") f.close() print('wrote:', plk_name) return None
def Pk_NBKT_boss(zbin, NorS='ngc'): ''' calculate boss P(k) using nbodykit, which uses Nick's estimator ''' # first read in data and random catalogs path_to_catalogs = UT.catalog_dir('boss') if NorS == 'ngc': str_nors = 'North' else: str_nors = 'South' data = FITSCatalog( os.path.join(path_to_catalogs, 'galaxy_DR12v5_CMASSLOWZTOT_' + str_nors + '.fits')) randoms = FITSCatalog( os.path.join(path_to_catalogs, 'random1_DR12v5_CMASSLOWZTOT_' + str_nors + '.fits')) # selection effects if zbin == 1: ZMIN, ZMAX = 0.2, 0.5 elif zbin == 2: ZMIN, ZMAX = 0.4, 0.6 randoms['Selection'] = (randoms['Z'] > ZMIN) & (randoms['Z'] < ZMAX) randoms = randoms[randoms['Selection']] data['Selection'] = (data['Z'] > ZMIN) & (data['Z'] < ZMAX) data = data[data['Selection']] # impose fiducial BOSS DR12 cosmology cosmo = cosmology.Cosmology(H0=67.6, Om0=0.31, flat=True) # add Cartesian position column data['Position'] = transform.SkyToCartesian(data['RA'], data['DEC'], data['Z'], cosmo=cosmo) randoms['Position'] = transform.SkyToCartesian(randoms['RA'], randoms['DEC'], randoms['Z'], cosmo=cosmo) randoms['WEIGHT'] = 1.0 data['WEIGHT'] = data['WEIGHT_SYSTOT'] * (data['WEIGHT_NOZ'] + data['WEIGHT_CP'] - 1.0) # combine the data and randoms into a single catalog fkp = FKPCatalog(data, randoms) mesh = fkp.to_mesh(Nmesh=360, nbar='NZ', fkp_weight='WEIGHT_FKP', comp_weight='WEIGHT', window='tsc', selection='Selection') # compute the multipoles r = ConvolvedFFTPower(mesh, poles=[0, 2, 4], dk=0.01, kmin=0.) poles = r.poles plk = [] for ell in [0, 2, 4]: P = poles['power_%d' % ell].real if ell == 0: P = P - r.attrs['shotnoise'] plk.append(P) # shot noise comparison #print(r.attrs['shotnoise']) #alpha = r.attrs['alpha'] #sel = mesh['randoms'][mesh.selection] #first = mesh['randoms'][sel] #comp_weight = first[mesh.comp_weight] #fkp_weight = first[mesh.fkp_weight] #S = (comp_weight**2*fkp_weight**2).sum() #print("expected SN") #print((1.+alpha)*alpha*S/r.attrs['randoms.norm']) #sel = mesh['data'][mesh.selection] #first = mesh['data'][sel] #comp_weight = first[mesh.comp_weight] #fkp_weight = first[mesh.fkp_weight] #S_data = (comp_weight**2*fkp_weight**2).sum() #print("true SN") #print((S_data+alpha**2*S)/r.attrs['randoms.norm']) f = open( ''.join([ UT.catalog_dir('boss'), 'pk.nbodykit.', NorS, '.zbin', str(zbin), '.dat' ]), 'w') f.write("### header ### \n") for key in r.attrs: f.write("%s = %s \n" % (key, str(r.attrs[key]))) f.write("columns : k , P0, P2, P4 \n") f.write('### header ### \n') for ik in range(len(poles['k'])): f.write("%f \t %f \t %f \t %f" % (poles['k'][ik], plk[0][ik], plk[1][ik], plk[2][ik])) f.write("\n") f.close() return None
def patchyCov(zbin, NorS='ngc', ell=0, clobber=False): ''' Construct covariance matrix for patchy mocks measured using Nbodykit ''' catalog = 'patchy.z'+str(zbin) f_cov = ''.join([UT.catalog_dir(catalog), 'Cov_pk.', catalog, '.ell', str(ell), '.', NorS, '.NBKT.dat']) if os.path.isfile(f_cov) and not clobber: i_k, j_k, k_i, k_j, C_ij = np.loadtxt(f_cov, skiprows=4, unpack=True, usecols=[0,1,2,3,-1]) ki = k_i.reshape((int(np.sqrt(len(i_k))), int(np.sqrt(len(i_k))))) kj = k_j.reshape((int(np.sqrt(len(i_k))), int(np.sqrt(len(i_k))))) Cij = C_ij.reshape((int(np.sqrt(len(i_k))), int(np.sqrt(len(i_k))))) return ki, kj, Cij else: # calculate my patchy covariance pkay = Pk() n_mock = pkay._n_mock(catalog) i_mock, n_missing = 0, 0 for i in range(1,n_mock+1): try: pkay.Read(catalog, i, ell=ell, NorS=NorS, sys='fc') pkay.krange([0.01,0.15]) k = pkay.k pk = pkay.pk n_kbin = len(k) if i == 1: ks = np.zeros((n_mock, n_kbin)) pks = np.zeros((n_mock, n_kbin)) ks[i_mock,:] = k pks[i_mock,:] = pk i_mock += 1 except IOError: if i == 1: raise ValueError print('missing -- ', pkay._file_name(catalog, i, 'fc')) n_missing += 1 print(n_missing, "P(k) files missing") n_mock -= n_missing if n_missing > 0: pks = pks[:n_mock,:] #mu_pk = np.sum(pks, axis=0)/np.float(n_mock) #Cov_pk = np.dot((pks-mu_pk).T, pks-mu_pk)/float(n_mock-1) Cov_pk = np.cov(pks.T) # write to file f = open(f_cov, 'w') f.write('### header ### \n') f.write('Covariance matrix for ell = '+str(ell)+' calculated from the '+str(n_mock)+' mocks \n') f.write('5 columns: {measured power spectrum bin index i} {measured power spectrum bin index j} k_i k_j C_ij \n') f.write('### header ### \n') k_i, k_j = [], [] for i in range(Cov_pk.shape[0]): for j in range(Cov_pk.shape[1]): f.write('%i \t %i \t %f \t %f \t %e' % (i+1, j+1, k[i], k[j], Cov_pk[i,j])) f.write('\n') k_i.append(k[i]) k_j.append(k[j]) f.close() k_i = np.array(k_i) k_j = np.array(k_j) ki = k_i.reshape(Cov_pk.shape) kj = k_j.reshape(Cov_pk.shape) return ki, kj, Cov_pk
def _tarfile(self, name): ''' tar ball that contains all the powerspectrum measurements. with/without systematics, everything. P(k) files are pretty small so who cares. ''' return ''.join([UT.catalog_dir(name), 'power_', name, '.tar'])