def calc_OmHI(aa, suff):
"""Sums over the halos to compute OmHI."""
# Work out the base path.
halocat = BigFileCatalog(scratchyf + sim+ '/fastpm_%0.4f//'%aa, dataset='LL-0.200')
mp = halocat.attrs['MassTable'][1]*1e10##
halocat['Mass'] = halocat['Length'].compute() * mp
cencat = BigFileCatalog(scratchcm + sim+'/fastpm_%0.4f/cencat'%aa+suff)
satcat = BigFileCatalog(scratchcm + sim+'/fastpm_%0.4f/satcat'%aa+suff)
HImodelz = HImodel(aa)
halocat['HImass'], cencat['HImass'], satcat['HImass'] = HImodelz.assignHI(halocat, cencat, satcat)
if mode == 'halos': catalogs = [halocat]
elif mode == 'galaxies': catalogs = [cencat, satcat]
elif mode == 'all': catalogs = [halocat, cencat, satcat]
catalogs = [cencat]
rankweight = sum([cat['HImass'].sum().compute() for cat in catalogs])
mHI = comm.allreduce(rankweight)
rankweight = sum([(cat['HImass']**2).sum().compute() for cat in catalogs])
mHI2 = comm.allreduce(rankweight)
# Convert to OmegaHI.
nbar = mHI**2/mHI2/bs**3
rhoHI = mHI/bs**3/aa**3
cc = Cosmology()
#OmHI = rhoHI/cc.rhoCritCom(1/aa-1)
OmHI = rhoHI/2.7754e11
# For now just print it.
if rank == 0: print("{:6.2f} {:12.4e} {:12.4e}".format(1/aa-1,OmHI,nbar))
return OmHI, nbar
def calc_pkhalo(aa, suff):
if rank ==0 : print('Read in central/satellite catalogs')
#cencat = BigFileCatalog(project+sim+'/fastpm_%0.4f/cencat'%aa)
cencat = BigFileCatalog(scratch2+sim+'/fastpm_%0.4f/cencat-16node'%aa)
cencat = BigFileCatalog(scratch1+sim+'/fastpm_%0.4f/'%aa+\
'LL-0.200','')
rsdfac = read_conversions(scratch1+sim+'/fastpm_%0.4f/'%aa)
# Compute the power spectrum
los = [0,0,1]
cpos = cencat['Position'].compute()
start = time()
cenmesh = pm.paint(cencat['Position'])
h1mesh = cenmesh
end = time()
if rank ==0 : print('Time taken to mesh = ', end - start)
start = time()
#pkh1h1 = FFTPower(h1mesh,mode='1d',kmin=0.025,dk=0.0125).power
pkh1h1 = FFTPower(h1mesh/h1mesh.cmean(),mode='1d').power
end = time()
if rank ==0 : print('Time taken to power = ', end - start)
# Extract the quantities we want and write the file.
kk = pkh1h1['k']
sn = pkh1h1.attrs['shotnoise']
pk = np.abs(pkh1h1['power'])
if rank ==0 :
fout = open("../data/L%04d/Halocat_pks_1d_%0.4f.txt"%(bs, aa),"w")
fout.write("# Subtracting SN={:15.5e}.\n".format(sn))
fout.write("# {:>8s} {:>15s}\n".format("k","Pk_0_HI"))
for i in range(kk.size):
fout.write("{:10.5f} {:15.5e}\n".format(kk[i],pk[i]-sn))
fout.close()
def calc_bias(aa,mcut,suff):
'''Compute the bias(es) for the HI'''
print('Read in DM mesh')
dm = BigFileMesh(project+sim+'/fastpm_%0.4f/'%aa+\
'/dmesh_N%04d'%nc,'1').paint()
dm /= dm.cmean()
pkmm = FFTPower(dm,mode='1d').power
k,pkmm= pkmm['k'],pkmm['power'] # Ignore shotnoise.
#
print('Read in central/satellite catalogs')
cencat = BigFileCatalog(project+sim+'/fastpm_%0.4f/cencat'%aa)
satcat = BigFileCatalog(project+sim+'/fastpm_%0.4f/satcat'%aa+suff)
#
cencat['HImass'] = HI_hod(cencat['Mass'],aa,mcut)
satcat['HImass'] = HI_hod(satcat['Mass'],aa,mcut)
totHImass = cencat['HImass'].sum().compute() +\
satcat['HImass'].sum().compute()
cencat['HImass']/= totHImass/float(nc)**3
satcat['HImass']/= totHImass/float(nc)**3
#
allcat = MultipleSpeciesCatalog(['cen','sat'],cencat,satcat)
#
h1mesh = allcat.to_mesh(BoxSize=bs,Nmesh=[nc,nc,nc],weight='HImass')
pkh1h1 = FFTPower(h1mesh,mode='1d').power
pkh1h1 = pkh1h1['power']-pkh1h1.attrs['shotnoise']
pkh1mm = FFTPower(h1mesh,second=dm,mode='1d').power['power']
# Compute the biases.
b1x = np.abs(pkh1mm/(pkmm+1e-10))
b1a = np.abs(pkh1h1/(pkmm+1e-10))**0.5
return(k,b1x,b1a,np.abs(pkmm))
def calc_pk1d(aa, suff):
'''Compute the 1D redshift-space P(k) for the HI'''
if rank == 0: print('Read in central/satellite catalogs')
cencat = BigFileCatalog(scratch2 + sim +
'/fastpm_%0.4f/cencat-16node' % aa)
satcat = BigFileCatalog(scratch2 + sim + '/fastpm_%0.4f/satcat' % aa +
suff)
rsdfac = read_conversions(scratch1 + sim + '/fastpm_%0.4f/' % aa)
# Compute the power spectrum
los = [0, 0, 1]
cencat['RSDpos'] = cencat['Position'] + cencat['Velocity'] * los * rsdfac
satcat['RSDpos'] = satcat['Position'] + satcat['Velocity'] * los * rsdfac
cencat['HImass'] = HI_hod(cencat['Mass'], aa)
satcat['HImass'] = HI_hod(satcat['Mass'], aa)
rankHImass = cencat['HImass'].sum().compute() +\
satcat['HImass'].sum().compute()
totHImass = comm.allreduce(rankHImass)
cencat['HImass'] /= totHImass / float(nc)**3
satcat['HImass'] /= totHImass / float(nc)**3
allcat = MultipleSpeciesCatalog(['cen', 'sat'], cencat, satcat)
h1mesh = allcat.to_mesh(BoxSize=bs,Nmesh=[nc,nc,nc],\
position='RSDpos',weight='HImass')
pkh1h1 = FFTPower(h1mesh, mode='1d', kmin=0.025, dk=0.0125).power
# Extract the quantities we want and write the file.
kk = pkh1h1['k']
sn = pkh1h1.attrs['shotnoise']
pk = np.abs(pkh1h1['power'])
if rank == 0:
fout = open("HI_pks_1d_{:6.4f}.txt".format(aa), "w")
fout.write("# Subtracting SN={:15.5e}.\n".format(sn))
fout.write("# {:>8s} {:>15s}\n".format("k", "Pk_0_HI"))
for i in range(kk.size):
fout.write("{:10.5f} {:15.5e}\n".format(kk[i], pk[i] - sn))
fout.close()
def compute_power(output, Nmesh=4096):
"""Compute the compensated power spectrum from a catalogue."""
catcdm = BigFileCatalog(output, dataset='1/', header='Header')
assert 1 - catcdm.attrs['Time'] < 0.01
catcdm.to_mesh(Nmesh=Nmesh,
resampler='cic',
compensated=True,
interlaced=True)
pkcdm = FFTPower(catcdm, mode='1d', Nmesh=Nmesh)
return pkcdm.power
def readincatalog(aa, matter=False):
if matter: dmcat = BigFileCatalog(scratch + sim + '/fastpm_%0.4f/'%aa, dataset='1')
halocat = BigFileCatalog(scratch + sim + '/fastpm_%0.4f/'%aa, dataset='LL-0.200')
mp = halocat.attrs['MassTable'][1]*1e10
print('Mass of particle is = %0.2e'%mp)
halocat['Mass'] = halocat['Length'] * mp
print(halocat['Mass'][:5].compute()/1e10)
halocat['Position'] = halocat['Position']%bs # Wrapping positions assuming periodic boundary conditions
if matter:
return dmcat, halocat
else: return halocat
def readcat(path, subsample=False):
# Only used to compute concentration etc from halo mass
# thus no need to be accurate.
CP = cosmology.Cosmology(Omega0_cdm=0.3, h=0.677, Omega0_b=.05)
cat = BigFileCatalog(path, dataset='LL-0.200')
if subsample:
cat = cat[:subsample]
M0 = cat.attrs['M0'] * 1e10
if cat.comm.rank == 0:
cat.logger.info("mass of a particle %e" % M0)
cat['Mass'] = cat['Length'] * M0
if 'Aemit' in cat.columns:
redshift = 1 / cat['Aemit'] - 1
else:
redshift = 1 / cat.attrs['Time'] - 1
cat['conc'] = transform.HaloConcentration(
cat['Mass'], CP, redshift).compute(scheduler="single-threaded")
# proper to comoving
cat['rvir'] = transform.HaloRadius(cat['Mass'], CP,
redshift).compute() * (1 + redshift)
cat['vdisp'] = transform.HaloVelocityDispersion(cat['Mass'], CP,
redshift).compute()
if 'Aemit' not in cat.columns:
cat['Aemit'] = cat.attrs['Time'][0]
mean, std = stat(cat['Aemit'].compute(), cat.comm)
if cat.comm.rank == 0:
cat.logger.info("Aemit mean = %g std = %g" % (mean, std))
mean, std = stat(cat['Mass'].compute(), cat.comm)
if cat.comm.rank == 0:
cat.logger.info("mass mean, std = %g, %g" % (mean, std))
mean, std = stat(cat['conc'].compute(), cat.comm)
if cat.comm.rank == 0:
cat.logger.info("conc mean, std = %g, %g" % (mean, std))
mean, std = stat(cat['rvir'].compute(), cat.comm)
if cat.comm.rank == 0:
cat.logger.info("rvir mean, std = %g, %g" % (mean, std))
mean, std = stat(cat['vdisp'].compute(), cat.comm)
if cat.comm.rank == 0:
cat.logger.info("vdisp mean, std = %g, %g" % (mean, std))
cat.attrs['BoxSize'] = numpy.ones(3) * cat.attrs['BoxSize'][0]
return cat
def compute_power(output, Nmesh=1024, species=2, spec2=None):
"""Compute the compensated power spectrum from a catalogue."""
catnu = BigFileCatalog(output, dataset=str(species) + '/', header='Header')
sp = sptostr(species)
sp2 = sptostr(spec2)
outfile = path.join(
output, "../power-" + sp + sp2 + "-%.4f.txt" % catnu.attrs["Time"][0])
if path.isfile(outfile):
return
catnu.to_mesh(Nmesh=Nmesh, window='cic', compensated=True, interlaced=True)
if spec2 is not None:
catcdm = BigFileCatalog(output,
dataset=str(spec2) + '/',
header='Header')
catcdm.to_mesh(Nmesh=Nmesh,
window='cic',
compensated=True,
interlaced=True)
pkcross = FFTPower(catnu, mode='1d', Nmesh=1024, second=catcdm)
power = pkcross.power
else:
pknu = FFTPower(catnu, mode='1d', Nmesh=1024)
power = pknu.power
numpy.savetxt(
outfile,
numpy.array([power['k'], power['power'].real, power['modes']]).T)
return power
def read_cat_nsel(ns, nsel, nmin0, nmin1):
cat = BigFileCatalog(ns.catalog, header='Header', dataset=ns.dataset)
volume = cat.attrs['BoxSize'][0]**3
if 'Length' in cat.columns:
while nmin1 - nmin0 > 1:
nminc = (nmin1 + nmin0) / 2
sel = True
sel = sel & (cat['Length'] >= nminc)
nsel1 = cat.comm.allreduce(sel.sum().compute())
if nsel1 < nsel: # too few
nmin1 = nminc
else:
nmin0 = nminc
if cat.comm.rank == 0:
print('found nmin', nmin1, nmin0, 'nsel is', nsel1, 'target is',
nsel)
cat['Selection'] = sel
cat['RSDPosition'] = cat[
'Position'] + cat.attrs['RSDFactor'] * cat['Velocity'] * [0, 0, 1]
return cat
def plotH1mass(fname, fsize=15):
'''plot cdf of H1'''
fig, ax = plt.subplots(1, 2, figsize=(9, 4))
for i, aa in enumerate(aafiles):
zz = zzfiles[i]
print(zz)
halos = BigFileCatalog(project + sim + '/fastpm_%0.4f/halocat/' % aa)
hmass, h1mass = halos["Mass"].compute(), halos['H1mass'].compute()
ax[0].plot(hmass, h1mass, label=zz, lw=2)
# plt.plot(hmass, np.cumsum(h1mass)/h1mass.sum(), label=zz, lw=2)
ax[1].plot(hmass[::-1],
np.cumsum(h1mass[::-1]) / h1mass.sum(),
label=zz,
lw=2)
ax[0].loglog()
ax[1].legend(ncol=2, fontsize=13)
ax[1].set_xscale('log')
ax[0].set_ylabel('$M_{H1}$', fontsize=fsize)
ax[1].set_ylabel('$M_{H1}$ CDF', fontsize=fsize)
for axis in ax:
axis.set_xlabel('$M_h$', fontsize=fsize)
#axis.grid(which='both', lw=0.5, color='gray')
fig.tight_layout()
fig.savefig(figpath + fname)
def check_ic_power_spectra(genicfileout, camb_zstr, outdir=".", accuracy=0.07, m_nu=0):
"""Generate the power spectrum for each particle type from the generated simulation files
and check that it matches the input. This is a consistency test on each simulation output."""
#Generate power spectra
output = os.path.join(outdir, genicfileout)
#Now check that they match what we put into the simulation, from CAMB
#Reload the CAMB files from disc, just in case something went wrong writing them.
matterpow = os.path.join(outdir,"camb_linear/ics_matterpow_"+camb_zstr+".dat")
transfer = os.path.join(outdir, "camb_linear/ics_transfer_"+camb_zstr+".dat")
#Load DM catalog and try for a baryon catalog
cats = {1: BigFileCatalog(output, dataset='1/', header='Header')}
try:
cats[0] = BigFileCatalog(output, dataset='0/', header='Header')
except:
pass
omegab = cats[1].attrs['OmegaBaryon']
omega0 = cats[1].attrs['Omega0']
hubble = cats[1].attrs['HubbleParam']
npart = int(np.round(np.cbrt(cats[1].attrs['TotNumPart'][1])))
assert npart > 0
cambpow = CLASSPowerSpectrum(matterpow, transfer,omega0=omega0, omegab=omegab, omeganu=m_nu/93.14/hubble**2)
for sp in cats.keys():
#GenPK output is at PK-[nu,by,DM]-basename(genicfileout)
cats[sp].to_mesh(Nmesh=npart*2, window='cic', compensated=True, interlaced=True)
pk = FFTPower(cats[sp], mode='1d', Nmesh=npart*2, dk=5.0e-6)
#GenPK output is at PK-[nu,by,DM]-basename(genicfileout)
#Load the power spectra
#Convert units from kpc/h to Mpc/h
kk_ic = pk.power['k'][1:]*1e3
Pk_ic = pk.power['power'][1:].real/1e9
modes_ic = pk.power['modes'][1:]
ii = np.isfinite(kk_ic)
kk_ic = kk_ic[ii]
Pk_ic = Pk_ic[ii]
#Load the power spectrum. Note that DM may be total.
ccsp = sp
if len(cats) == 1:
ccsp = -1
if m_nu > 0:
ccsp = 3
Pk_camb = cambpow.get_class_power(species=ccsp)
(kk_ic, Pk_ic) = modecount_rebin(kk_ic, Pk_ic, modes_ic[ii], Pk_camb, ndesired=npart//2)
error = plot_ic_power(kk_ic, Pk_ic, Pk_camb(kk_ic), sp=sp, npart=npart, outdir=outdir)
#Don't worry too much about one failing mode.
if np.size(np.where(error > accuracy)) > 3:
raise RuntimeError("Pk accuracy check failed for "+str(sp)+". Max error: "+str(np.max(error)))
def make_galaxy_pk(scale_factor,nc,seed,bs=1536,T=40,B=2,simpath=sc_simpath,outpath=sc_outpath,Rsm=0):
aa = scale_factor # since particle IDs are ordered only need to load at one redshift
zz = 1/aa-1
dgrow = cosmo.scale_independent_growth_factor(zz)
fname = get_filename(nc,seed,T=T,B=B)
spath = simpath + fname
opath = outpath + fname
galpath = opath + '/hod/'
try: os.makedirs(opath+'spectra/')
except : pass
zz = 1/aa-1
pm = ParticleMesh(BoxSize=bs, Nmesh=[nc, nc, nc], dtype='f8')
rank = pm.comm.rank
dyn = BigFileCatalog(spath + '/fastpm_%0.4f/1'%aa)
# Load Matter Mesh
fpos = dyn['Position'].compute()
mlay = pm.decompose(fpos)
mmesh = pm.paint(fpos, layout=mlay)
mmesh /= mmesh.cmean()
# Load halo mesh: HOD parameters fixed for now...
mmin = 10**12.5; m1fac = 20
cendir ='cencat-aa-%.04f-Mmin-%.1f-M1f-%.1f-alpha-0p8-subvol'%(aa,np.log10(mmin), m1fac)
satdir ='satcat-aa-%.04f-Mmin-%.1f-M1f-%.1f-alpha-0p8-subvol'%(aa,np.log10(mmin), m1fac)
cencat = BigFileCatalog(galpath + cendir)
satcat = BigFileCatalog(galpath + satdir)
hpos = np.concatenate((cencat['Position'],satcat['Position']))
hlay = pm.decompose(hpos)
hmesh = pm.paint(hpos, layout=hlay)
hmesh /= hmesh.cmean()
phm = FFTPower(mmesh, second=hmesh, mode='1d').power
k, phm = phm['k'], phm['power'].real
phh = FFTPower(hmesh, mode='1d').power
k, phh = phh['k'], phh['power'].real
np.savetxt(opath+'spectra/pks-%04d-%04d-%04d-gal.txt'%(aa*10000, bs, nc), np.vstack([k, phh, phm]).T.real, header='k, phh, phm/ ', fmt='%0.4e')
def read_cat1(ns, nmin=None):
if ns.mesh:
mesh = BigFileMesh(ns.catalog, dataset=ns.dataset)
else:
cat = BigFileCatalog(ns.catalog, header='Header', dataset=ns.dataset)
volume = cat.attrs['BoxSize'][0] ** 3
if nmin is not None and nmin != 0:
sel = True
sel = sel & (cat['Length'] >= nmin)
cat['Selection'] = sel
cat['RSDPosition'] = cat['Position'] + cat.attrs['RSDFactor'] * cat['Velocity'] * [0, 0, 1]
mesh = cat.to_mesh(interlaced=True, compensated=True, window='tsc', Nmesh=ns.nmesh)
return mesh
def calc_pkmu(aa,suff):
'''Compute the redshift-space P(k) for the HI in mu bins'''
if rank ==0 : print('Read in central/satellite catalogs')
cencat = BigFileCatalog(scratch2+sim+'/fastpm_%0.4f/cencat-16node'%aa)
satcat = BigFileCatalog(scratch2+sim+'/fastpm_%0.4f/satcat'%aa+suff)
rsdfac = read_conversions(scratch1+sim+'/fastpm_%0.4f/'%aa)
# Compute P(k,mu).
los = [0,0,1]
cencat['RSDpos'] = cencat['Position']+cencat['Velocity']*los * rsdfac
satcat['RSDpos'] = satcat['Position']+satcat['Velocity']*los * rsdfac
cencat['HImass'] = HI_hod(cencat['Mass'],aa)
satcat['HImass'] = HI_hod(satcat['Mass'],aa)
totHImass = cencat['HImass'].sum().compute() +\
satcat['HImass'].sum().compute()
cencat['HImass']/= totHImass/float(nc)**3
satcat['HImass']/= totHImass/float(nc)**3
allcat = MultipleSpeciesCatalog(['cen','sat'],cencat,satcat)
h1mesh = allcat.to_mesh(BoxSize=bs,Nmesh=[nc,nc,nc],\
position='RSDpos',weight='HImass')
#pkh1h1 = FFTPower(h1mesh,mode='2d',Nmu=4,los=los).power
cenmesh = pm.paint(cencat['RSDpos'], mass=cencat['HImass'])
satmesh = pm.paint(satcat['RSDpos'], mass=satcat['HImass'])
h1mesh = cenmesh + satmesh
pkh1h1 = FFTPower(h1mesh,mode='2d',Nmu=4,los=los).power
# Extract what we want.
kk = pkh1h1.coords['k']
sn = pkh1h1.attrs['shotnoise']
pk = pkh1h1['power']
# Write the results to a file.
if rank ==0 :
fout = open("../data/HI_pks_mu_{:06.4f}.txt".format(aa),"w")
fout.write("# Redshift space power spectrum in mu bins.\n")
fout.write("# Subtracting SN={:15.5e}.\n".format(sn))
ss = "# {:>8s}".format(r'k\mu')
for i in range(pkh1h1.shape[1]):
ss += " {:15.5f}".format(pkh1h1.coords['mu'][i])
fout.write(ss+"\n")
for i in range(1,pk.shape[0]):
ss = "{:10.5f}".format(kk[i])
for j in range(pk.shape[1]):
ss += " {:15.5e}".format(np.abs(pk[i,j]-sn))
fout.write(ss+"\n")
fout.close()
def calc_bias(aa, mcut, suff):
'''Compute the bias(es) for the HI'''
if rank == 0:
print("Processing a={:.4f}...".format(aa))
print('Reading DM mesh...')
dm = BigFileMesh(scratch1+sim+'/fastpm_%0.4f/'%aa+\
'/1-mesh/N%04d'%nc,'').paint()
dm /= dm.cmean()
if rank == 0: print('Computing DM P(k)...')
pkmm = FFTPower(dm, mode='1d').power
k, pkmm = pkmm['k'], pkmm['power'] # Ignore shotnoise.
if rank == 0: print('Done.')
#
if rank == 0: print('Reading central/satellite catalogs...')
cencat = BigFileCatalog(scratch2 + sim +
'/fastpm_%0.4f/cencat-16node' % aa)
satcat = BigFileCatalog(scratch2 + sim + '/fastpm_%0.4f/satcat' % aa +
suff)
if rank == 0: print('Catalogs read.')
#
if rank == 0: print('Computing HI masses...')
cencat['HImass'] = HI_hod(cencat['Mass'], aa, mcut)
satcat['HImass'] = HI_hod(satcat['Mass'], aa, mcut)
rankHImass = cencat['HImass'].sum().compute() +\
satcat['HImass'].sum().compute()
rankHImass = np.array([rankHImass])
totHImass = np.zeros(1, dtype='float')
comm.Allreduce(rankHImass, totHImass, MPI.SUM)
totHImass = totHImass[0]
cencat['HImass'] /= totHImass / float(nc)**3
satcat['HImass'] /= totHImass / float(nc)**3
if rank == 0: print('HI masses done.')
#
if rank == 0: print('Combining catalogs and computing P(k)...')
allcat = MultipleSpeciesCatalog(['cen', 'sat'], cencat, satcat)
h1mesh = allcat.to_mesh(BoxSize=bs, Nmesh=[nc, nc, nc], weight='HImass')
pkh1h1 = FFTPower(h1mesh, mode='1d').power
pkh1h1 = pkh1h1['power'] - pkh1h1.attrs['shotnoise']
pkh1mm = FFTPower(h1mesh, second=dm, mode='1d').power['power']
if rank == 0: print('Done.')
# Compute the biases.
b1x = np.abs(pkh1mm / (pkmm + 1e-10))
b1a = np.abs(pkh1h1 / (pkmm + 1e-10))**0.5
if rank == 0: print("Finishing processing a={:.4f}.".format(aa))
return (k, b1x, b1a, np.abs(pkmm))
def calc_bias(aa,mcut,suff):
'''Compute the bias(es) for the HI'''
if rank==0:
print("Processing a={:.4f}...".format(aa))
print('Reading DM mesh...')
dm = BigFileMesh(scratch1+sim+'/fastpm_%0.4f/'%aa+\
'/1-mesh/N%04d'%nc,'').paint()
dm /= dm.cmean()
if rank==0: print('Computing DM P(k)...')
pkmm = FFTPower(dm,mode='1d').power
k,pkmm= pkmm['k'],pkmm['power'] # Ignore shotnoise.
if rank==0: print('Done.')
#
if rank==0: print('Reading central/satellite catalogs...')
cencat = BigFileCatalog(scratch2+sim+'/fastpm_%0.4f/cencat-16node'%aa)
# satcat = BigFileCatalog(scratch2+sim+'/fastpm_%0.4f/satcat'%aa+suff)
# if rank==0: print('Catalogs read.')
# #
# if rank==0: print('Computing HI masses...')
# cencat['HImass'] = HI_hod(cencat['Mass'],aa,mcut)
# satcat['HImass'] = HI_hod(satcat['Mass'],aa,mcut)
# totHImass = cencat['HImass'].sum().compute() +\
# satcat['HImass'].sum().compute()
# cencat['HImass']/= totHImass/float(nc)**3
# satcat['HImass']/= totHImass/float(nc)**3
# if rank==0: print('HI masses done.')
# #
# if rank==0: print('Combining catalogs and computing P(k)...')
# allcat = MultipleSpeciesCatalog(['cen','sat'],cencat,satcat)
# #h1mesh = allcat.to_mesh(BoxSize=bs,Nmesh=[nc,nc,nc],weight='HImass')
cenmesh = pm.paint(cencat['Position'])
#satmesh = pm.paint(satcat['Position'], mass=satcat['HImass'])
h1mesh = cenmesh
pkh1h1 = FFTPower(h1mesh,mode='1d').power
pkh1h1 = pkh1h1['power']-pkh1h1.attrs['shotnoise']
pkh1mm = FFTPower(h1mesh,second=dm,mode='1d').power['power']
if rank==0: print('Done.')
# Compute the biases.
b1x = np.abs(pkh1mm/(pkmm+1e-10))
b1a = np.abs(pkh1h1/(pkmm+1e-10))**0.5
if rank==0: print("Finishing processing a={:.4f}.".format(aa))
if rank==0:
fout = open("../data/L%04d/HI_bias_halo_%0.4f.txt"%(bs, aa),"w")
fout.write("# Mcut={:12.4e}Msun/h.\n".format(mcut))
fout.write("# {:>8s} {:>10s} {:>10s} {:>15s}\n".\
format("k","b1_x","b1_a","Pkmm"))
for i in range(1,kk.size):
fout.write("{:10.5f} {:10.5f} {:10.5f} {:15.5e}\n".\
format(kk[i],b1x[i],b1a[i],pkmm[i]))
fout.close()
bavg = np.mean(b1x[1:10])
flog.write("{:6.2f} {:12.4e} {:6.3f}\n".format(1/aa-1,mcut,bavg))
flog.flush()
if rank==0: flog.close()
#
return(k,b1x,b1a,np.abs(pkmm))
def plot_image(snapshot, dataset=1, colorbar=True, Nmesh=None):
"""Make a pretty picture of the mass distribution."""
cat = BigFileCatalog(snapshot, dataset=str(dataset) + '/', header='Header')
if Nmesh is None:
Nmesh = 2 * int(np.round(np.cbrt(cat.attrs['TotNumPart'][dataset])))
box = cat.attrs['BoxSize'] / 1000
mesh = cat.to_mesh(Nmesh=Nmesh)
plt.clf()
plt.imshow(np.log10(mesh.preview(axes=(0, 1)) / Nmesh),
extent=(0, box, 0, box))
if colorbar:
plt.colorbar()
plt.xlabel("x (Mpc/h)")
plt.ylabel("y (Mpc/h)")
plt.tight_layout()
snap = os.path.basename(os.path.normpath(snapshot))
plt.savefig("dens-plt-type" + str(dataset) + snap + ".pdf")
plt.clf()
def calc_pkll(aa,suff):
'''Compute the redshift-space P_ell(k) for the HI'''
if rank ==0 : print('Read in central/satellite catalogs')
cencat = BigFileCatalog(scratch2+sim+'/fastpm_%0.4f/cencat-16node'%aa)
satcat = BigFileCatalog(scratch2+sim+'/fastpm_%0.4f/satcat'%aa+suff)
rsdfac = read_conversions(scratch1+sim+'/fastpm_%0.4f/'%aa)
#
los = [0,0,1]
cencat['RSDpos'] = cencat['Position']+cencat['Velocity']*los * rsdfac
satcat['RSDpos'] = satcat['Position']+satcat['Velocity']*los * rsdfac
cencat['HImass'] = HI_hod(cencat['Mass'],aa)
satcat['HImass'] = HI_hod(satcat['Mass'],aa)
totHImass = cencat['HImass'].sum().compute() +\
satcat['HImass'].sum().compute()
cencat['HImass']/= totHImass/float(nc)**3
satcat['HImass']/= totHImass/float(nc)**3
allcat = MultipleSpeciesCatalog(['cen','sat'],cencat,satcat)
h1mesh = allcat.to_mesh(BoxSize=bs,Nmesh=[nc,nc,nc],\
position='RSDpos',weight='HImass')
#pkh1h1 = FFTPower(h1mesh,mode='2d',Nmu=8,los=los,poles=[0,2,4]).poles
cenmesh = pm.paint(cencat['RSDpos'], mass=cencat['HImass'])
satmesh = pm.paint(satcat['RSDpos'], mass=satcat['HImass'])
h1mesh = cenmesh + satmesh
pkh1h1 = FFTPower(h1mesh,mode='2d',Nmu=8,los=los,poles=[0,2,4]).poles
# Extract the quantities of interest.
kk = pkh1h1.coords['k']
sn = pkh1h1.attrs['shotnoise']
P0 = pkh1h1['power_0'].real - sn
P2 = pkh1h1['power_2'].real
P4 = pkh1h1['power_4'].real
# Write the results to a file.
if rank ==0 :
fout = open("../data/HI_pks_ll_{:06.4f}.txt".format(aa),"w")
fout.write("# Redshift space power spectrum multipoles.\n")
fout.write("# Subtracting SN={:15.5e} from monopole.\n".format(sn))
fout.write("# {:>8s} {:>15s} {:>15s} {:>15s}\n".format("k","P0","P2","P4"))
for i in range(1,kk.size):
fout.write("{:10.5f} {:15.5e} {:15.5e} {:15.5e}\n".\
format(kk[i],P0[i],P2[i],P4[i]))
fout.close()
def savecatalogmesh(bs, nc, aa):
dmcat = BigFileCatalog(scratchyf + sim + '/fastpm_%0.4f/' % aa,
dataset='1')
pm = ParticleMesh(BoxSize=bs, Nmesh=[nc, nc, nc])
pos = dmcat['Position'].compute()
layout = pm.decompose(pos)
mesh = pm.paint(pos, layout=layout)
mesh = FieldMesh(mesh)
path = project + sim + '/fastpm_%0.4f/' % aa + '/dmesh_N%04d' % nc
mesh.save(path, dataset='1', mode='real')
def main():
ap = ArgumentParser()
ap.add_argument('fpm', help='e.g. /scratch/fpm_0.1000/')
ap.add_argument('ll', type=float, help='e.g. 0.2 or 0.168')
ap.add_argument('--with-peak', help='Find Peaks KDDensity estimation (slow)', default=True)
ap.add_argument('fof', help='e.g. /scratch/fpm_0.1000/fof . Will write to {fof}/{ll}')
ap.add_argument('--nmin', type=int, default=20, help='min number of particles to be in the catalogue')
ns = ap.parse_args()
cat = BigFileCatalog(ns.fpm, header='Header', dataset='1/')
cat.attrs['BoxSize'] = numpy.ones(3) * cat.attrs['BoxSize'][0]
cat.attrs['Nmesh'] = numpy.ones(3) * cat.attrs['NC'][0]
cosmo = Planck15.match(Omega0_m=cat.attrs['OmegaM'])
M0 = cat.attrs['OmegaM'][0] * 27.75 * 1e10 * cat.attrs['BoxSize'].prod() / cat.csize
if cat.comm.rank == 0:
print('BoxSize', cat.attrs['BoxSize'])
print('Nmesh', cat.attrs['Nmesh'])
print('Mass of a particle', M0)
print('OmegaM', cosmo.Om0)
if ns.with_peak:
cat['Density'] = KDDensity(cat).density
fof = FOF(cat, linking_length=ns.ll, nmin=ns.nmin)
if ns.with_peak:
features = fof.find_features(peakcolumn='Density')
else:
features = fof.find_features(peakcolumn=None)
features['Mass'] = M0 * features['Length']
if fof.comm.rank == 0:
print('Total number of features found', features.csize)
print('Saving columns', features.columns)
features.save(ns.fof + '/%0.3f' % ns.ll, features.columns)
def read_cat2(ns, nmin=None):
cat = BigFileCatalog(ns.catalog, header='Header', dataset=ns.dataset)
volume = cat.attrs['BoxSize'][0] ** 3
if nmin is not None and nmin != 0:
sel = True
sel = sel & (cat['Length'] >= nmin)
cat['Selection'] = sel
cat['RSDPosition'] = cat['Position'] + cat.attrs['RSDFactor'] * cat['Velocity'] * [0, 0, 1]
return cat
def plot_image(sim, snap, dataset=1, colorbar=False):
"""Make a pretty picture of the mass distribution."""
pp = os.path.join(os.path.join(datadir, sim), "output/PART_00" + str(snap))
cat = BigFileCatalog(pp, dataset=str(dataset), header='Header')
mesh = cat.to_mesh(Nmesh=512)
plt.clf()
plt.imshow(np.log10(mesh.preview(axes=(0, 1)) / 512),
extent=(0, 3, 0, 3),
vmin=-0.2,
vmax=0.2)
if colorbar:
plt.colorbar()
plt.xlabel("x (100 Mpc/h)")
plt.ylabel("y (100 Mpc/h)")
plt.tight_layout()
plt.savefig(
os.path.join(
savedir,
"dens-plt-" + munge_scale(sim) + "t" + str(dataset) + ".pdf"))
plt.clf()
def find_haloes_epoch(self, filename, z, N_min=None, with_peak=True):
if N_min is not None: self.N_min = N_min
cat = BigFileCatalog(filename, header='Header', dataset='1/')
self.cosmo = self.cosmo.match(Omega0_m=cat.attrs['Om0'])
if self.n_p is None: self.n_p = cat.attrs['nc'][0]
cat.attrs['boxsize'] = np.ones(3) * cat.attrs['boxsize'][0]
cat.attrs['Nmesh'] = np.ones(3) * cat.attrs['nc'][0]
# M0 = cat.attrs['Om0'][0] * 27.75 * 1e10 * cat.attrs['boxsize'].prod() / cat.csize
M0 = cat.attrs['Om0'][0] * critical_density0(
cosmo=self.cosmo,
hunits=True).value * cat.attrs['boxsize'].prod() / cat.csize
self.M_p = M0
cat.attrs['BoxSize'] = cat.attrs['boxsize']
print('BoxSize:', cat.attrs['boxsize'], 'Mpc')
print('Nmesh:', cat.attrs['Nmesh'])
print('Mass of a particle:', np.round(M0 / 1e8, decimals=3),
'x 1e8 solar mass')
print('OmegaM:', np.round(self.cosmo.Om0, decimals=3))
print('Finding haloes...')
cat['Density'] = KDDensity(cat).density
fof = FOF(cat, linking_length=self.b, nmin=self.N_min)
features = fof.find_features(
peakcolumn='Density') if with_peak else fof.find_features(
peakcolumn=None)
halo_catalog = fof.to_halos(M0, self.cosmo, z)
# print(halo_catalog)
features['Mass'] = M0 * features['Length']
print('...done')
print('Total number of haloes found:', halo_catalog.csize)
# print('Saving columns', features.columns)
# print('halos', halo_catalog.columns)
return halo_catalog
def fiddlebiasgal(aa, suff, nc=nc, mcfv=[1.], saveb=False, bname='h1bias', ofolder=None):
'''Fiddle bias for galaxies'''
if ofolder is None: ofolder = project + '/%s/fastpm_%0.4f/'%(sim, aa)
pm = ParticleMesh(BoxSize = bs, Nmesh = [nc, nc, nc])
print('Read in catalogs')
cencat = BigFileCatalog(project + sim + '/fastpm_%0.4f/cencat'%aa)
satcat = BigFileCatalog(project + sim + '/fastpm_%0.4f/satcat'%aa+suff)
cpos, spos = cencat['Position'], satcat['Position']
cmass, smass = cencat['Mass'], satcat['Mass']
pos = np.concatenate((cpos, spos), axis=0)
dm = BigFileMesh(project + sim + '/fastpm_%0.4f/'%aa + '/dmesh_N%04d'%nc, '1').paint()
pkm = FFTPower(dm/dm.cmean(), mode='1d').power
k, pkm = pkm['k'], pkm['power']
b1, b1sq = np.zeros((k.size, len(mcfv))), np.zeros((k.size, len(mcfv)))
for imc, mcf in enumerate(mcfv):
print(mcf)
ch1mass = HI_masscutfiddle(cmass, aa, mcutf=mcf)
sh1mass = HI_masscutfiddle(smass, aa, mcutf=mcf)
h1mass = np.concatenate((ch1mass, sh1mass), axis=0)
#
h1mesh = pm.paint(pos, mass=h1mass)
pkh1 = FFTPower(h1mesh/h1mesh.cmean(), mode='1d').power['power']
pkh1m = FFTPower(h1mesh/h1mesh.cmean(), second=dm/dm.cmean(), mode='1d').power['power']
#Bias
b1[:, imc] = pkh1m/pkm
b1sq[:, imc] = pkh1/pkm
np.savetxt(ofolder+bname+'auto'+suff+'.txt', np.concatenate((k.reshape(-1, 1), b1sq**0.5), axis=1),
header='mcut factors = %s\nk, pkh1xm/pkm, pkh1/pkm^0.5'%mcfv)
np.savetxt(ofolder+bname+'cross'+suff+'.txt', np.concatenate((k.reshape(-1, 1), b1), axis=1),
header='mcut factors = %s\nk, pkh1xm/pkm, pkh1mx/pkm'%mcfv)
return k, b1, b1sq
def read_cat(ns, nmin=None):
cat = BigFileCatalog(ns.catalog, header='Header', dataset=ns.dataset)
volume = cat.attrs['BoxSize'][0]**3
if nmin is not None:
sel = True
sel = sel & (cat['Length'] >= nmin)
cat['Selection'] = sel
# cat = cat[sel]
return cat
def readinhalos(aafiles, sim, dpath=project, HI_hod=HI_hod):
aafiles = np.array(aafiles)
zzfiles = np.array([round(atoz(aa), 2) for aa in aafiles])
hpos, hmass, hid, h1mass = {}, {}, {}, {}
size = {}
for i, aa in enumerate(aafiles):
zz = zzfiles[i]
halos = BigFileCatalog(dpath + sim + '/fastpm_%0.4f/halocat/' % aa)
hmass[zz] = halos["Mass"].compute()
if HI_hod is not None: h1mass[zz] = HI_hod(halos['Mass'].compute(), aa)
hpos[zz] = halos['Position'].compute()
hid[zz] = np.arange(hmass[zz].size).astype(int)
size[zz] = halos.csize
return hpos, hmass, hid, h1mass, size
def readinsatellites(aafiles, suff, sim, dpath=myscratch, HI_hod=HI_hod):
aafiles = np.array(aafiles)
zzfiles = np.array([round(atoz(aa), 2) for aa in aafiles])
spos, smass, sh1mass, shid = {}, {}, {}, {}
size = {}
for i, aa in enumerate(aafiles):
zz = zzfiles[i]
sat = BigFileCatalog(dpath + sim + '/fastpm_%0.4f/satcat' % aa + suff)
smass[zz] = sat["Mass"].compute()
if HI_hod is not None: sh1mass[zz] = HI_hod(sat['Mass'].compute(), aa)
spos[zz] = sat['Position'].compute()
shid[zz] = sat['GlobalID'].compute()
size[zz] = sat.csize
return spos, smass, shid, sh1mass, size
def readincentrals(aafiles, suff, sim, dpath=myscratch, HI_hod=HI_hod):
aafiles = np.array(aafiles)
zzfiles = np.array([round(atoz(aa), 2) for aa in aafiles])
cpos, cmass, ch1mass, chid = {}, {}, {}, {}
size = {}
for i, aa in enumerate(aafiles):
zz = zzfiles[i]
cen = BigFileCatalog(dpath + sim + '/fastpm_%0.4f/cencat' % aa + suff)
cmass[zz] = cen["Mass"].compute()
if HI_hod is not None: ch1mass[zz] = HI_hod(cen['Mass'].compute(), aa)
cpos[zz] = cen['Position'].compute()
chid[zz] = cen['GlobalID'].compute()
size[zz] = cen.csize
return cpos, cmass, chid, ch1mass, size
def compute_fast_power(output,
ICS,
vthresh=850,
Nmesh=1024,
species=2,
spec2=None):
"""Compute the compensated power spectrum from a catalogue."""
sp = sptostr(species)
sp2 = sptostr(spec2)
catnu = BigFileCatalog(output, dataset=str(species) + '/', header='Header')
outfile = path.join(
output,
"../power-fast-" + sp + sp2 + "-%.4f.txt" % catnu.attrs["Time"][0])
if path.isfile(outfile):
return
catics = BigFileCatalog(ICS, dataset=str(species) + '/', header='Header')
fast = (catics['Velocity']**
2).sum(axis=1) < vthresh**2 / catics.attrs["Time"]**3
fastids = catics["ID"][fast].compute()
fastids.sort()
#Note: map_blocks runs elementwise over blocks.
#So we need to pre-compute fastids: if we do not we will
#end up checking whether elements in a block of catnu["ID"]
#are in the equivalent block in fastids.
select = catnu["ID"].map_blocks(wrapper,
fastids,
dtype=numpy.bool,
chunks=catnu["ID"].chunks)
catnu[select].to_mesh(Nmesh=Nmesh,
window='cic',
compensated=True,
interlaced=True)
if spec2 is not None:
catcdm = BigFileCatalog(output,
dataset=str(spec2) + '/',
header='Header')
catcdm.to_mesh(Nmesh=Nmesh,
window='cic',
compensated=True,
interlaced=True)
pkcross = FFTPower(catnu[select],
mode='1d',
Nmesh=Nmesh,
second=catcdm,
dk=5.0e-6)
power = pkcross.power
else:
pknu = FFTPower(catnu[select], mode='1d', Nmesh=Nmesh, dk=5.0e-6)
power = pknu.power
numpy.savetxt(
outfile,
numpy.array([power['k'], power['power'].real, power['modes']]).T)
return power
def read_cat(ns):
cat = BigFileCatalog(ns.catalog, header='Header', dataset=ns.dataset)
volume = cat.attrs['BoxSize'][0] ** 3
sel = True
if ns.abundance is not None:
if cat.comm.rank == 0:
print('Abundance cut %g / %d halos ' % (volume * ns.abundance, cat.csize))
sel = sel & (cat.Index < volume * ns.abundance)
if ns.nmin is not None:
sel = sel & (cat['Length'] >= ns.nmin)
if ns.nmax is not None:
sel = sel & (cat['Length'] <= ns.nmax)
cat['VelocityOffset'] = cat['Velocity'] * cat.attrs['RSDFactor']
cat['Selection'] = sel
if ns.with_rsd:
cat['Position'] += cat['VelocityOffset'] * [0, 0, 1.]
return cat
