def calc_bias(aa, h1mesh, suff):
'''Compute the bias(es) for the HI'''
if rank == 0: print('Calculating bias')
if rank == 0:
print("Processing a={:.4f}...".format(aa))
print('Reading DM mesh...')
dm = BigFileMesh(args['matterfile'] % (aa), 'N1024').paint()
dm /= dm.cmean()
if rank == 0: print('Computing DM P(k)...')
pkmm = FFTPower(dm, mode='1d').power
k, pkmm = pkmm['k'], pkmm['power'] # Ignore shotnoise.
if rank == 0: print('Done.')
#
pkh1h1 = FFTPower(h1mesh, mode='1d').power
kk = pkh1h1.coords['k']
pkh1h1 = pkh1h1['power'] - pkh1h1.attrs['shotnoise']
pkh1mm = FFTPower(h1mesh, second=dm, mode='1d').power['power']
if rank == 0: print('Done.')
# Compute the biases.
b1x = np.abs(pkh1mm / (pkmm + 1e-10))
b1a = np.abs(pkh1h1 / (pkmm + 1e-10))**0.5
if rank == 0: print("Finishing processing a={:.4f}.".format(aa))
#
if rank == 0:
fout = open(outfolder + "HI_bias_{:6.4f}.txt".format(aa), "w")
fout.write("# {:>8s} {:>10s} {:>10s} {:>15s}\n".\
format("k","b1_x","b1_a","Pkmm"))
for i in range(1, kk.size):
fout.write("{:10.5f} {:10.5f} {:10.5f} {:15.5e}\n".\
format(kk[i],b1x[i],b1a[i],pkmm[i].real))
fout.close()
def make_lagfields(nc,
seed,
bs=1536,
T=40,
B=2,
simpath=sc_simpath,
outpath=sc_outpath,
Rsm=0):
fname = get_filename(nc, seed, T=T, B=B)
spath = simpath + fname
opath = outpath + fname
try:
os.makedirs(opath)
except:
pass
pm = ParticleMesh(BoxSize=bs, Nmesh=[nc, nc, nc], dtype='f4')
rank = pm.comm.rank
lin = BigFileMesh(spath + '/linear', 'LinearDensityK').paint()
lin -= lin.cmean()
print(rank, 'lin field read')
header = '1,b1,b2,bg,bk'
names = header.split(',')
lag_fields = tools.getlagfields(
pm, lin, R=Rsm) # use the linear field at the desired redshift
print(rank, 'lag field created')
for i, ff in enumerate(lag_fields):
x = FieldMesh(ff)
x.save(opath + 'lag', dataset=names[i], mode='real')
def calc_bias(aa,mcut,suff):
'''Compute the bias(es) for the HI'''
print('Read in DM mesh')
dm = BigFileMesh(project+sim+'/fastpm_%0.4f/'%aa+\
'/dmesh_N%04d'%nc,'1').paint()
dm /= dm.cmean()
pkmm = FFTPower(dm,mode='1d').power
k,pkmm= pkmm['k'],pkmm['power'] # Ignore shotnoise.
#
print('Read in central/satellite catalogs')
cencat = BigFileCatalog(project+sim+'/fastpm_%0.4f/cencat'%aa)
satcat = BigFileCatalog(project+sim+'/fastpm_%0.4f/satcat'%aa+suff)
#
cencat['HImass'] = HI_hod(cencat['Mass'],aa,mcut)
satcat['HImass'] = HI_hod(satcat['Mass'],aa,mcut)
totHImass = cencat['HImass'].sum().compute() +\
satcat['HImass'].sum().compute()
cencat['HImass']/= totHImass/float(nc)**3
satcat['HImass']/= totHImass/float(nc)**3
#
allcat = MultipleSpeciesCatalog(['cen','sat'],cencat,satcat)
#
h1mesh = allcat.to_mesh(BoxSize=bs,Nmesh=[nc,nc,nc],weight='HImass')
pkh1h1 = FFTPower(h1mesh,mode='1d').power
pkh1h1 = pkh1h1['power']-pkh1h1.attrs['shotnoise']
pkh1mm = FFTPower(h1mesh,second=dm,mode='1d').power['power']
# Compute the biases.
b1x = np.abs(pkh1mm/(pkmm+1e-10))
b1a = np.abs(pkh1h1/(pkmm+1e-10))**0.5
return(k,b1x,b1a,np.abs(pkmm))
def fiddlebias(aa, saveb=False, bname='h1bias.txt', ofolder=None):
#Fiddling bias for halos. Deprecated.
print('Read in catalogs')
halocat = readincatalog(aa, matter=False)
hpos = halocat['Position']
hmass = halocat['Mass']
h1mass = dohod.HI_mass(hmass, aa)
dm = BigFileMesh(project + sim + '/fastpm_%0.4f/'%aa + '/dmesh_N%04d'%nc, '1').paint()
if ofolder is None: ofolder = project + '/%s/fastpm_%0.4f/'%(sim, aa)
#measure power
pm = ParticleMesh(BoxSize = bs, Nmesh = [nc, nc, nc])
pkm = FFTPower(dm/dm.cmean(), mode='1d').power
k, pkm = pkm['k'], pkm['power']
##H1
print("H1")
h1mesh = pm.paint(hpos, mass=h1mass)
pkh1 = FFTPower(h1mesh/h1mesh.cmean(), mode='1d').power['power']
pkh1m = FFTPower(h1mesh/h1mesh.cmean(), second=dm/dm.cmean(), mode='1d').power['power']
#Bias
b1h1 = pkh1m/pkm
b1h1sq = pkh1/pkm
if saveb:
np.savetxt(ofolder+bname, np.stack((k, b1h1, b1h1sq**0.5), axis=1),
header='k, pkh1xm/pkm, pkh1/pkm^0.5')
return k, b1h1, b1h1sq
def savenoisep2d(Nmu=4):
for ia, aa in enumerate([0.3333, 0.2000, 0.1429]):
zz = 1/aa-1
for iw, wopt in enumerate(['opt', 'pess']):
#for iw, wopt in enumerate(['opt']):
for it, thopt in enumerate(['opt', 'pess', 'reas']):
#for it, thopt in enumerate([ 'reas']):
if rank == 0: print(aa, wopt, thopt)
dpath = '/global/cscratch1/sd/chmodi/m3127/21cm_cleaning/recon/fastpm_%0.4f/wedge_kmin%0.2f_%s/'%(aa, 0.03, wopt)
dpath += 'L%04d-N%04d-R//thermal-%s-hex/ZA/opt_s999_h1massA_fourier_rsdpos/'%(bs, nc, thopt)
ofolder = '../../data/ZArecon-rsd/kmin-003_wedge-%s/thermal-%s-hex/Nmu%d/'%(wopt, thopt, Nmu)
try: os.makedirs(ofolder)
except: pass
fname = ofolder + 'pnoise-L%04d_%0.4f.txt'%(bs, aa)
if args.pp : fname = fname[:-4] + '-up.txt'
try:
rep = np.loadtxt(fname).T
rpfit = [{'k':rep[0], 'power':rep[i+1]} for i in range(3)]
except:
pass
if args.pp:
ivar = BigFileMesh(dpath+'/ivarmesh_up', 'ivar').paint()
else:
ivar = BigFileMesh(dpath+'/ivarmesh', 'ivar').paint()
svar = (ivar.r2c()**-0.5).c2r()
p0 = FFTPower(svar, mode='2d', Nmu=Nmu).power
if rank == 0: np.savetxt(fname, p0['power'].real)
def calc_bias(aa, h1mesh, suff):
'''Compute the bias(es) for the HI'''
if rank == 0:
print("Processing a={:.4f}...".format(aa))
print('Reading DM mesh...')
if ncsim == 10240:
dm = BigFileMesh(scratchyf+sim+'/fastpm_%0.4f/'%aa+\
'/1-mesh/N%04d'%nc,'').paint()
else:
dm = BigFileMesh(project+sim+'/fastpm_%0.4f/'%aa+\
'/dmesh_N%04d/1/'%nc,'').paint()
dm /= dm.cmean()
if rank == 0: print('Computing DM P(k)...')
pkmm = FFTPower(dm, mode='1d').power
k, pkmm = pkmm['k'], pkmm['power'] # Ignore shotnoise.
if rank == 0: print('Done.')
#
pkh1h1 = FFTPower(h1mesh, mode='1d').power
pkh1h1 = pkh1h1['power'] - pkh1h1.attrs['shotnoise']
pkh1mm = FFTPower(h1mesh, second=dm, mode='1d').power['power']
if rank == 0: print('Done.')
# Compute the biases.
b1x = np.abs(pkh1mm / (pkmm + 1e-10))
b1a = np.abs(pkh1h1 / (pkmm + 1e-10))**0.5
if rank == 0: print("Finishing processing a={:.4f}.".format(aa))
return (k, b1x, b1a, np.abs(pkmm))
def savestd():
#for ia, aa in enumerate([0.3333, 0.2000, 0.1429]):
for ia, aa in enumerate([0.1429]):
#for ia, aa in enumerate([0.3333, 0.2000, 0.1429]):
zz = 1 / aa - 1
for iw, wopt in enumerate(['opt', 'pess']):
#for iw, wopt in enumerate(['opt']):
for it, thopt in enumerate(['opt', 'pess', 'reas']):
#for it, thopt in enumerate([ 'pess']):
if rank == 0: print(aa, wopt, thopt)
dpath = '/global/cscratch1/sd/chmodi/m3127/21cm_cleaning/recon/fastpm_%0.4f/wedge_kmin%0.2f_%s/' % (
aa, 0.03, wopt)
dpath += 'L%04d-N%04d-R//thermal-%s-hex/ZA/opt_s999_h1massA_fourier_rsdpos/' % (
bs, nc, thopt)
ofolder = '../../data/ZArecon-rsd/kmin-003_wedge-%s/thermal-%s-hex/' % (
wopt, thopt)
fname = ofolder + 'std-L%04d_%0.4f.txt' % (bs, aa)
fxname = ofolder + 'xstd-L%04d_%0.4f.txt' % (bs, aa)
if args.pp: fname = fname[:-4] + '-up.txt'
if args.pp: fxname = fxname[:-4] + '-up.txt'
try:
rep = np.loadtxt(fname + 's').T
except:
try:
if args.pp:
std = BigFileMesh(dpath + '/stdrecon_up-noise',
'std').paint()
ss = BigFileMesh(dpath + '/datap_up', 's').paint()
else:
std = BigFileMesh(dpath + '/stdrecon-noise',
'std').paint()
ss = BigFileMesh(dpath + '/datap', 's').paint()
p0 = FFTPower(std, mode='1d').power
px = FFTPower(std, second=ss, mode='1d').power
if rank == 0:
np.savetxt(fname,
np.stack([p0['k']] +
[p0['power'].real]).T,
header='k, p0')
if rank == 0:
np.savetxt(fxname,
np.stack([px['k']] +
[px['power'].real]).T,
header='k, px')
except Exception as e:
print(e)
def calc_bias(aa, mcut, suff):
'''Compute the bias(es) for the HI'''
if rank == 0:
print("Processing a={:.4f}...".format(aa))
print('Reading DM mesh...')
dm = BigFileMesh(scratch1+sim+'/fastpm_%0.4f/'%aa+\
'/1-mesh/N%04d'%nc,'').paint()
dm /= dm.cmean()
if rank == 0: print('Computing DM P(k)...')
pkmm = FFTPower(dm, mode='1d').power
k, pkmm = pkmm['k'], pkmm['power'] # Ignore shotnoise.
if rank == 0: print('Done.')
#
if rank == 0: print('Reading central/satellite catalogs...')
cencat = BigFileCatalog(scratch2 + sim +
'/fastpm_%0.4f/cencat-16node' % aa)
satcat = BigFileCatalog(scratch2 + sim + '/fastpm_%0.4f/satcat' % aa +
suff)
if rank == 0: print('Catalogs read.')
#
if rank == 0: print('Computing HI masses...')
cencat['HImass'] = HI_hod(cencat['Mass'], aa, mcut)
satcat['HImass'] = HI_hod(satcat['Mass'], aa, mcut)
rankHImass = cencat['HImass'].sum().compute() +\
satcat['HImass'].sum().compute()
rankHImass = np.array([rankHImass])
totHImass = np.zeros(1, dtype='float')
comm.Allreduce(rankHImass, totHImass, MPI.SUM)
totHImass = totHImass[0]
cencat['HImass'] /= totHImass / float(nc)**3
satcat['HImass'] /= totHImass / float(nc)**3
if rank == 0: print('HI masses done.')
#
if rank == 0: print('Combining catalogs and computing P(k)...')
allcat = MultipleSpeciesCatalog(['cen', 'sat'], cencat, satcat)
h1mesh = allcat.to_mesh(BoxSize=bs, Nmesh=[nc, nc, nc], weight='HImass')
pkh1h1 = FFTPower(h1mesh, mode='1d').power
pkh1h1 = pkh1h1['power'] - pkh1h1.attrs['shotnoise']
pkh1mm = FFTPower(h1mesh, second=dm, mode='1d').power['power']
if rank == 0: print('Done.')
# Compute the biases.
b1x = np.abs(pkh1mm / (pkmm + 1e-10))
b1a = np.abs(pkh1h1 / (pkmm + 1e-10))**0.5
if rank == 0: print("Finishing processing a={:.4f}.".format(aa))
return (k, b1x, b1a, np.abs(pkmm))
def calc_bias(aa,h1mesh, outfolder, fname):
'''Compute the bias(es) for the HI'''
if rank==0: print('Calculating bias')
if rank==0:
print("Processing a={:.4f}...".format(aa))
print('Reading DM mesh...')
if ncsim == 10240:
dm = BigFileMesh(scratchyf+sim+'/fastpm_%0.4f/'%aa+\
'/1-mesh/N%04d'%nc,'').paint()
else:
dm = BigFileMesh(project+sim+'/fastpm_%0.4f/'%aa+\
'/dmesh_N%04d/1/'%nc,'').paint()
dm /= dm.cmean()
if rank==0: print('Computing DM P(k)...')
pkmm = FFTPower(dm,mode='1d').power
k,pkmm= pkmm['k'],pkmm['power'] # Ignore shotnoise.
if rank==0: print('Done.')
#
pkh1h1 = FFTPower(h1mesh,mode='1d').power
kk = pkh1h1.coords['k']
pkh1h1 = pkh1h1['power']-pkh1h1.attrs['shotnoise']
pkh1mm = FFTPower(h1mesh,second=dm,mode='1d').power['power']
if rank==0: print('Done.')
# Compute the biases.
b1x = np.abs(pkh1mm/(pkmm+1e-10))
b1a = np.abs(pkh1h1/(pkmm+1e-10))**0.5
if rank==0: print("Finishing processing a={:.4f}.".format(aa))
#
if rank==0:
fout = open(outfolder + "{}_bias_{:6.4f}.txt".format(fname, aa),"w")
fout.write("# {:>8s} {:>10s} {:>10s} {:>15s}\n".\
format("k","b1_x","b1_a","Pkmm"))
for i in range(1,kk.size):
fout.write("{:10.5f} {:10.5f} {:10.5f} {:15.5e}\n".\
format(kk[i],b1x[i],b1a[i],pkmm[i].real))
fout.close()
def fiddlebiasgal(aa, suff, nc=nc, mcfv=[1.], saveb=False, bname='h1bias', ofolder=None):
'''Fiddle bias for galaxies'''
if ofolder is None: ofolder = project + '/%s/fastpm_%0.4f/'%(sim, aa)
pm = ParticleMesh(BoxSize = bs, Nmesh = [nc, nc, nc])
print('Read in catalogs')
cencat = BigFileCatalog(project + sim + '/fastpm_%0.4f/cencat'%aa)
satcat = BigFileCatalog(project + sim + '/fastpm_%0.4f/satcat'%aa+suff)
cpos, spos = cencat['Position'], satcat['Position']
cmass, smass = cencat['Mass'], satcat['Mass']
pos = np.concatenate((cpos, spos), axis=0)
dm = BigFileMesh(project + sim + '/fastpm_%0.4f/'%aa + '/dmesh_N%04d'%nc, '1').paint()
pkm = FFTPower(dm/dm.cmean(), mode='1d').power
k, pkm = pkm['k'], pkm['power']
b1, b1sq = np.zeros((k.size, len(mcfv))), np.zeros((k.size, len(mcfv)))
for imc, mcf in enumerate(mcfv):
print(mcf)
ch1mass = HI_masscutfiddle(cmass, aa, mcutf=mcf)
sh1mass = HI_masscutfiddle(smass, aa, mcutf=mcf)
h1mass = np.concatenate((ch1mass, sh1mass), axis=0)
#
h1mesh = pm.paint(pos, mass=h1mass)
pkh1 = FFTPower(h1mesh/h1mesh.cmean(), mode='1d').power['power']
pkh1m = FFTPower(h1mesh/h1mesh.cmean(), second=dm/dm.cmean(), mode='1d').power['power']
#Bias
b1[:, imc] = pkh1m/pkm
b1sq[:, imc] = pkh1/pkm
np.savetxt(ofolder+bname+'auto'+suff+'.txt', np.concatenate((k.reshape(-1, 1), b1sq**0.5), axis=1),
header='mcut factors = %s\nk, pkh1xm/pkm, pkh1/pkm^0.5'%mcfv)
np.savetxt(ofolder+bname+'cross'+suff+'.txt', np.concatenate((k.reshape(-1, 1), b1), axis=1),
header='mcut factors = %s\nk, pkh1xm/pkm, pkh1mx/pkm'%mcfv)
return k, b1, b1sq
def read_cat1(ns, nmin=None):
if ns.mesh:
mesh = BigFileMesh(ns.catalog, dataset=ns.dataset)
else:
cat = BigFileCatalog(ns.catalog, header='Header', dataset=ns.dataset)
volume = cat.attrs['BoxSize'][0] ** 3
if nmin is not None and nmin != 0:
sel = True
sel = sel & (cat['Length'] >= nmin)
cat['Selection'] = sel
cat['RSDPosition'] = cat['Position'] + cat.attrs['RSDFactor'] * cat['Velocity'] * [0, 0, 1]
mesh = cat.to_mesh(interlaced=True, compensated=True, window='tsc', Nmesh=ns.nmesh)
return mesh
with open(pp) as ff:
for line in ff.readlines():
if 'RSDFactor' in line: rsdfaccat = float(line.split()[-2])
else: rsdfaccat = 0.
rsdfac = rsdfaccat * 100. / aa ##Add hoc factor due to incorrect velocity dimensions in nbody.py
lsstposition = lsstcat['Position'] + lsstcat['Velocity'] * np.array(
[0, 0, 1]) * rsdfaccat
llayout = truth_pm.decompose(lsstposition)
lmesh = truth_pm.paint(lsstposition, mass=lsstmasswt, layout=llayout)
lmesh /= lmesh.cmean()
lmesh -= 1
if rsdpos:
if ray:
hmesh = BigFileMesh(
proj + '/HV%d-R/fastpm_%0.4f/HImeshz-N%04d/' % (bs * 10, aa, nc),
'ModelD').paint()
else:
hmesh = BigFileMesh(
proj + '/HV%d-F/fastpm_%0.4f/HImeshz-N%04d/' % (bs * 10, aa, nc),
'ModelD').paint()
else:
if ray:
hmesh = BigFileMesh(
proj + '/HV%d-R/fastpm_%0.4f/HImesh-N%04d/' % (bs * 10, aa, nc),
'ModelD').paint()
else:
hmesh = BigFileMesh(
proj + '/HV%d-F/fastpm_%0.4f/HImesh-N%04d/' % (bs * 10, aa, nc),
'ModelD').paint()
truth_pm.comm.Barrier()
## Note: horizontal and vertical flipping of 2D slices.
train_gen = ImageDataGenerator(featurewise_center=False,\
rotation_range=0,\
width_shift_range=0.,\
height_shift_range=0.,\
horizontal_flip=True,\
vertical_flip=True,\
rescale=1.,\
preprocessing_function=pprocess,\
validation_split=valid_frac) ## Last Validation split.
## Fit whitening params.
## train_gen.fit(X)
<<<<<<< HEAD
mesh = BigFileMesh(fpath, dataset='1/Field', mode='real', header='Header').preview()
for sslice in np.arange(nslice):
## Split 3D sim into _nslice_ 2D (x, z) slices.
X[iid + nsplit * sslice, :, :, 0] = mesh[:, sslice, :]
X_train = X[:, :, :, :]
if train:
## Note: horizontal and vertical flipping of 2D slices.
train_gen = ImageDataGenerator(featurewise_center=False,\
rotation_range=0,\
width_shift_range=0.,\
height_shift_range=0.,\
horizontal_flip=True,\
vertical_flip=True,\
conf = Config(bs=bs, nc=nc, seed=seed)
pm = conf.pm
assert conf['stages'].size == nsteps
grid = pm.generate_uniform_particle_grid(shift=0).astype(np.float32)
kvec = tools.fftk((nc, nc, nc), bs, dtype=np.float32, symmetric=False)
solver = Solver(pm, conf['cosmology'])
conf['kvec'] = kvec
conf['grid'] = grid
#PM
#whitec = pm.generate_whitenoise(seed, mode='complex', unitary=False)
#lineark = whitec.apply(lambda k, v:Planck15.get_pklin(sum(ki ** 2 for ki in k)**0.5, 0) ** 0.5 * v / v.BoxSize.prod() ** 0.5)
#linear = lineark.c2r()
linear = BigFileMesh(
'/project/projectdirs/astro250/chmodi/cosmo4d/data/z00/L0400_N0128_S0100_05step/mesh/',
's').paint()
lineark = linear.r2c()
state = solver.lpt(lineark, grid, conf['stages'][0], order=2)
solver.nbody(state, leapfrog(conf['stages']))
final = pm.paint(state.X)
if pm.comm.rank == 0:
print('X, V computed')
cat = ArrayCatalog({
'Position': state.X,
'Velocity': state.V
},
BoxSize=pm.BoxSize,
Nmesh=pm.Nmesh)
kdd = KDDensity(cat).density
cat['KDDensity'] = kdd
cat = ArrayCatalog({'ID': idd, 'InitPosition': grid}, BoxSize=pm.BoxSize, Nmesh=pm.Nmesh)
#cat.save(lpath + 'initpos', tosave)
#cat.attrs = dyn.attrs
#header = '1,b1,b2,bg,bk'
header = 'b1,b2,bg,bk'
names = header.split(',')
#cat.save(lpath + 'dynamic/1', ('ID', 'InitPosition', 'Position'))
tosave = ['ID', 'InitPosition'] + names
if rank == 0: print(tosave)
for i in range(len(names)):
ff = BigFileMesh(lpath+ '/lag', names[i]).paint()
pm = ff.pm
rank = pm.comm.rank
glay, play = pm.decompose(grid), pm.decompose(fpos)
wts = ff.readout(grid, layout = glay, resampler='nearest')
cat[names[i]] = wts
#x = FieldMesh(pm.paint(fpos, mass=wts, layout=play))
#x.save(lpath + 'eul-z%03d-R%d'%(zz*100, Rsm), dataset=names[i], mode='real')
del pm, ff, wts
if rank == 0: print(rank, ' got weights')
cat.save(lpath + 'lagweights/', tosave)
for seed in range(9200, 9210, 1):
aa = 1.0000
zz = 1/aa-1
dgrow = cosmo.scale_independent_growth_factor(zz)
Rsm = 0
dpath = '/global/cscratch1/sd/yfeng1/m3127/desi/6144-%d-40eae2464/'%(seed)
lpath = '/global/cscratch1/sd/chmodi/m3127/crowcanyon/N%d-T0/S%d/'%(nc, seed)
ofolder = lpath + '/spectra/'
try: os.makedirs(ofolder)
except : pass
pm = ParticleMesh(BoxSize=bs, Nmesh=[nc, nc, nc], dtype='f4')
rank = pm.comm.rank
#grid = pm.mesh_coordinates()*bs/nc
lin = BigFileMesh(lpath+ '/linear', 'LinearDensityK').paint()
lin -= lin.cmean()
print(rank, 'lin field read')
header = '1,b1,b2,bg,bk'
names = header.split(',')
lag_fields = tools.getlagfields(pm, lin, R=Rsm) # use the linear field at the desired redshift
print(rank, 'lag field created')
for i, ff in enumerate(lag_fields):
x = FieldMesh(ff)
x.save(lpath + 'lag', dataset=names[i], mode='real')
def make_bias_plot():
"""Does the work of making the real-space xi(r) and b(r) figure."""
hmesh = BigFileMesh(dpath+'ZA/opt_s999_h1massA_fourier/datap', 'mapp').paint()
noises = np.loadtxt('/global/u1/c/chmodi/Programs/21cm/21cm_cleaning/data/summaryHI.txt').T
for i in range(noises[0].size):
if noises[0][i] == np.round(1/aa-1, 2): noise = noises[3][i]
print(noise)
zamod = BigFileMesh(dpath+'ZA/opt_s999_h1massA_fourier/fitp/', 'mapp').paint()
pmmod = BigFileMesh(dpath+'T05-B1/opt_s999_h1massA_fourier/fitp/', 'mapp').paint()
fin = BigFileMesh(dpath+'ZA/opt_s999_h1massA_fourier/datap/', 'd').paint()
fin /= fin.cmean()
fin -= 1
finsm = ft.smooth(fin, 3, 'gauss')
grid = pm.mesh_coordinates()*bs/nc
params, finmod = getbias(pm, basemesh=finsm, hmesh=hmesh, pos=grid, grid=grid) #
models = [zamod, pmmod, finmod, fin.copy()]
lbls = ['ZA shift', 'PM shift', 'Eulerian (R=3)', 'Final Matter']
lss = ['-', '-', '-', '--']
fig, ax = plt.subplots(1, 3, figsize=(15, 4))
for ii, mod in enumerate(models):
pmod = FFTPower(mod, mode='1d').power
k, pmod = pmod['k'], pmod['power']
ph = FFTPower(hmesh, mode='1d').power['power']
pxmodh = FFTPower(hmesh, second=mod, mode='1d').power['power']
perr = FFTPower(hmesh -mod, mode='1d').power['power']
ax[0].plot(k, pxmodh/(pmod*ph)**0.5, label=lbls[ii], lw=2, ls=lss[ii])
ax[0].set_ylabel('$r_{cc}$', fontdict=font)
ax[1].plot(k,(pmod/ph)**0.5, lw=2, ls=lss[ii])
#ax[1].set_ylabel(r'$\sqrt{P_{\rm mod}/P_{hh}}$', fontdict=font)
ax[1].set_ylabel(r'$T_f$', fontdict=font)
ax[2].plot(k, perr, lw=2, ls=lss[ii])
ax[2].set_ylabel(r'$P_{\delta_{\rm mod}-\delta_h}$', fontdict=font)
ax[2].set_yscale('log')
ax[2].axhline(noise)
for axis in ax:
axis.set_xscale('log')
axis.grid(which='both')
axis.legend(prop=fontmanage)
axis.set_xlabel(r'$k\quad [h\,{\rm Mpc}^{-1}]$', fontdict=font)
axis.set_xscale('log')
# Put on some more labels.
for axis in ax:
axis.set_xscale('log')
for tick in axis.xaxis.get_major_ticks():
tick.label.set_fontproperties(fontmanage)
for tick in axis.yaxis.get_major_ticks():
tick.label.set_fontproperties(fontmanage)
##and finish
plt.tight_layout(rect=[0, 0, 1, 0.95])
plt.savefig(figpath + '/bfit_L%04d_%04d.pdf'%(bs, aa*10000))
#dynamics
stages = numpy.linspace(0.1, aa, nsteps, endpoint=True)
#dynamic_model = NBodyModel(cosmo, truth_pm, B=B, steps=stages)
dynamic_model = ZAModel(cosmo, truth_pm, B=B, steps=stages)
#noise
#Artifically low noise since the data is constructed from the model
truth_noise_model = map.NoiseModel(
truth_pm, None,
noisevar * (truth_pm.BoxSize / truth_pm.Nmesh).prod(), 1234)
mock_model = map.MockModel(dynamic_model)
#Create and save data if not found
dyn = BigFileCatalog(dfolder + 'dynamic/1')
s_truth = BigFileMesh(dfolder + '/mesh/', 's').paint()
data_p = mock_model.make_observable(s_truth)
layout = truth_pm.decompose(dyn['Position'])
data_p.mapp = truth_pm.paint(dyn['Position'], layout=layout)
data_p.save(optfolder + 'datap/')
data_n = truth_noise_model.add_noise(data_p)
data_n.save(optfolder + 'datan/')
fit_p = mock_model.make_observable(s_truth)
fit_p.save(optfolder + 'fitp/')
s_init = truth_pm.generate_whitenoise(iseed, mode='complex')\
.apply(lambda k, v: v * (pk(sum(ki **2 for ki in k) **0.5) / v.BoxSize.prod()) ** 0.5)\
.c2r()*0.001
##
kk = None
ff = open('../../data/lostmodes-N%d.txt' % nc, 'w')
ff.write('#%4s %5s %5s %6s %6s %6s\n' %
('a', 'wedge', 'noise', 'angle', 'lost-w', 'lost-n'))
for aa in [0.1429, 0.2000, 0.3333]:
#for aa in [0.1429]:
zz = 1 / aa - 1
for wed in ['opt', 'pess']:
for th in ['opt', 'pess', 'reas']:
optfolder = recfolder + '/fastpm_%0.4f/wedge_kmin0.03_%s/L1024-N0256-R/thermal-%s-hex/ZA/opt_s999_h1massA_fourier_rsdpos/' % (
aa, wed, th)
if nc == 256:
dd = BigFileMesh(optfolder + 'datap/', 'mapp').paint()
dn = BigFileMesh(optfolder + 'datan/', 'mapp').paint()
else:
dd = BigFileMesh(optfolder + 'datap_up/', 'mapp').paint()
dn = BigFileMesh(optfolder + 'datan_up/', 'mapp').paint()
dn -= dd
if kk is None: kk = dd.r2c().x
pd2 = FFTPower(dd, mode='2d', Nmu=Nmu).power
k2, modes2, pd2 = pd2['k'], pd2['modes'], pd2['power']
pn2 = FFTPower(dn, mode='2d', Nmu=Nmu).power['power']
#Lost due to wedge
wedang = mapnoise.wedge(1 / aa - 1, att=wed, angle=True)
wedmu = mapnoise.wedge(1 / aa - 1, att=wed, angle=False)
mask = getmask(kk, wedang, kmin=0.03)
lostw = 1 - mask.sum() / mask.size
dtype=np.float32,
comm=MPI.COMM_SELF)
gridsmall = pmsmall.generate_uniform_particle_grid(shift=0)
for zz in [3.5, 4.0]:
aa = 1 / (1 + zz)
cats, meshes = setupuvmesh(zz,
suff=suff,
sim=sim,
pm=pm,
profile=profile,
stellar=stellar)
cencat, satcat = cats
h1meshfid, h1mesh, lmesh, uvmesh = meshes
if ncsim == 10240:
dm = BigFileMesh(scratchyf+sim+'/fastpm_%0.4f/'%aa+\
'/1-mesh/N%04d'%nc,'').paint()
else: dm = BigFileMesh(project+sim+'/fastpm_%0.4f/'%aa+\
'/dmesh_N%04d/1/'%nc,'').paint()
meshes = [dm, h1meshfid, h1mesh, lmesh, uvmesh]
names = ['dm', 'h1fid', 'h1new', 'lum', 'uv']
if rank == 0: print('\nMeshes read\n')
bindexes = np.arange(ncube**3)
bindexsplit = np.array_split(bindexes, wsize)
maxload = max(np.array([len(i) for i in bindexsplit]))
indices = np.zeros((ncube**3, 3))
totals = np.zeros((ncube**3, len(names)))
if __name__ == "__main__":
#bs, nc = 256, 128
bs, nc = 1024, 256
model = 'ModelA'
ik = 50
ii = 50
ffile = '../../data/bparams-L%04d-N%04d-%s.txt' % (bs, nc, model)
header = 'b1, b2, bg, b0, bk \nFit bias upto 0.3\nFit tf upto 1.0'
pm = ParticleMesh(BoxSize=bs, Nmesh=[nc, nc, nc])
rank = pm.comm.rank
grid = pm.mesh_coordinates() * bs / nc
lin = BigFileMesh(
'/global/cscratch1/sd/chmodi/m3127/cm_lowres/5stepT-B1/%d-%d-9100/linear'
% (bs, nc), 'LinearDensityK').paint()
tosave = []
for aa in [0.1429, 0.2000, 0.3333]:
zz = 1 / aa - 1
print(aa)
dyn = BigFileCatalog(
'/global/cscratch1/sd/chmodi/m3127/cm_lowres/5stepT-B1/%d-%d-9100/fastpm_%0.4f/1'
% (bs, nc, aa))
hmesh = BigFileMesh(
'/global/cscratch1/sd/chmodi/m3127/H1mass/highres/%d-9100/fastpm_%0.4f/HImesh-N%04d/'
% (bs * 10, aa, nc), model).paint()
fpos = dyn['Position']
dgrow = cosmo.scale_independent_growth_factor(zz)
def measurepk(nc=nc, dpath=scratchyf):
'''plot the power spectrum of halos on 'nc' grid'''
pm = ParticleMesh(BoxSize=bs, Nmesh=[nc, nc, nc])
for i, aa in enumerate(aafiles):
zz = zzfiles[i]
if rank == 0: print('redshift = ', zz)
if ncsim == 10240:
dm = BigFileMesh(scratchyf+sim+'/fastpm_%0.4f/'%aa+\
'/1-mesh/N%04d'%nc,'').paint()
else: dm = BigFileMesh(project+sim+'/fastpm_%0.4f/'%aa+\
'/dmesh_N%04d/1/'%nc,'').paint()
#dm = BigFileMesh(project + sim + '/fastpm_%0.4f/'%aa + '/dmesh_N%04d'%nc, '1').paint()
halos = BigFileCatalog(scratchyf + sim + '/fastpm_%0.4f/' % aa,
dataset='LL-0.200')
mp = halos.attrs['MassTable'][1] * 1e10
if rank == 0: print('Mass of particle is = %0.2e' % mp)
hmass = halos["Length"].compute() * mp
hpos = halos['Position'].compute()
layout = pm.decompose(hpos)
if rank == 0: print("paint")
hpmesh = pm.paint(hpos, layout=layout)
hmesh = pm.paint(hpos, mass=hmass, layout=layout)
print(rank, dm.cmean(), hmesh.cmean(), hpmesh.cmean())
pkm = FFTPower(dm / dm.cmean(), mode='1d').power
pkh = FFTPower(hmesh / hmesh.cmean(), mode='1d').power
pkhp = FFTPower(hpmesh / hpmesh.cmean(), mode='1d').power
pkhm = FFTPower(hmesh / hmesh.cmean(),
second=dm / dm.cmean(),
mode='1d').power
pkhpm = FFTPower(hpmesh / hpmesh.cmean(),
second=dm / dm.cmean(),
mode='1d').power
def savebinned(path, binstat, header):
if halos.comm.rank == 0:
k, p, modes = binstat['k'].real, binstat[
'power'].real, binstat['modes'].real
np.savetxt(path,
np.stack((k, p, modes), axis=1),
header=header)
ofolder = "../data/outputs/halos/{}/".format(sim)
if rank == 0:
print(ofolder)
try:
os.makedirs(ofolder)
except:
pass
savebinned(ofolder + 'pkhp_%0.4f.txt' % aa,
pkhp,
header='k, P(k), Nmodes')
savebinned(ofolder + 'pkhm_%0.4f.txt' % aa,
pkh,
header='k, P(k), Nmodes')
savebinned(ofolder + 'pkd_%0.4f.txt' % aa,
pkm,
header='k, P(k), Nmodes')
savebinned(ofolder + 'pkhpxd_%0.4f.txt' % aa,
pkhpm,
header='k, P(k), Nmodes')
savebinned(ofolder + 'pkhmxd_%0.4f.txt' % aa,
pkhm,
header='k, P(k), Nmodes')
def plotz1_2D(nc=nc, hirax=False):
aa = 0.5000
lsstn = 0.0500
elgn = 0.0010
dpath = '/global/cscratch1/sd/chmodi/m3127/21cm_cleaning/reconlsstv2//fastpm_0.5000/wedge_kmin0.03_pess/L1024-N0256-R/thermal-reas-hex/ZA/opt_s777_h1massD_%s_rsdpos/'
#dpath1 = '/global/cscratch1/sd/chmodi/m3127/21cm_cleaning/reconlsstv2//fastpm_0.5000/wedge_kmin0.03_pess/L1024-N0256-R/thermal-reas-hex/ZA/opt_s777_h1massD_%s_rsdpos/'
if hirax:
dpath = '/global/cscratch1/sd/chmodi/m3127/21cm_cleaning/reconlsstv2//fastpm_0.5000/wedge_kmin0.03_pess/L1024-N0256-R/thermal-reas-hex-hirax/ZA/opt_s777_h1massD_%s_rsdpos/'
if nc == 256:
hmesh = BigFileMesh(dpath % ('lwt0-nob2') + '/datap', 'mapp').paint()
lmesh = BigFileMesh(dpath % ('lsst-nob2_ln0500') + '/datap',
'mapp2').paint()
emesh = BigFileMesh(dpath % ('elg_ln%04d' % (elgn * 1e4)) + '/datap',
'mapp2').paint()
ph = FFTPower(hmesh, mode='2d').power
k, ph = ph['k'], ph['power']
pl = FFTPower(lmesh, mode='2d').power['power']
pe = FFTPower(emesh, mode='2d').power['power']
suff = 'lwt0-nob2'
h0, l0 = BigFileMesh(dpath % suff + '256-0.00/best-fit/',
'mapp').paint(), BigFileMesh(
dpath % suff + '256-0.00/best-fit/',
'mapp2').paint()
ph0, ph0x = getps2D(h0, hmesh)
pl0, pl0x = getps2D(l0, lmesh)
suff = 'lsst-nob2_ln%04d' % (lsstn * 1e4)
h1, l1 = BigFileMesh(dpath % suff + '256-0.00/best-fit/',
'mapp').paint(), BigFileMesh(
dpath % suff + '256-0.00/best-fit/',
'mapp2').paint()
ph1, ph1x = getps2D(h1, hmesh)
pl1, pl1x = getps2D(l1, lmesh)
suff = 'elg-nob2_ln%04d' % (elgn * 1e4)
h2, l2 = BigFileMesh(dpath % suff + '256-0.00/best-fit/',
'mapp').paint(), BigFileMesh(
dpath % suff + '256-0.00/best-fit/',
'mapp2').paint()
ph2, ph2x = getps2D(h2, hmesh)
pe2, pe2x = getps2D(l2, emesh)
# suff = 'lwt0-nob2'
# h3, l3 = BigFileMesh(dpath1%suff + '256-0.00/best-fit/', 'mapp').paint(), BigFileMesh(dpath%suff + '256-0.00/best-fit/', 'mapp2').paint()
# ph3, ph3x = getps2D(h3, hmesh)
# pe3, pe3x = getps2D(l3, emesh)
#
elif nc == 512:
hmesh = BigFileMesh(dpath % ('lwt0') + '/datap_up', 'mapp').paint()
lmesh = BigFileMesh(
dpath % ('lsst_ln%04d' % (lsstn * 1e4)) + '/datap_up',
'mapp2').paint()
emesh = BigFileMesh(
dpath % ('elg_ln%04d' % (elgn * 1e4)) + '/datap_up',
'mapp2').paint()
ph = FFTPower(hmesh, mode='2d').power
k, ph = ph['k'], ph['power']
pl = FFTPower(lmesh, mode='2d').power['power']
pe = FFTPower(emesh, mode='2d').power['power']
suff = 'lwt0-nob2'
h0, l0 = BigFileMesh(dpath % suff + 'upsample2/512-0.00/best-fit/',
'mapp').paint(), BigFileMesh(
dpath % suff + 'upsample2/512-0.00/best-fit/',
'mapp2').paint()
ph0, ph0x = getps2D(h0, hmesh)
pl0, pl0x = getps2D(l0, lmesh)
suff = 'lsst-nob2_ln%04d' % (lsstn * 1e4)
h1, l1 = BigFileMesh(dpath % suff + 'upsample2/512-0.00/best-fit/',
'mapp').paint(), BigFileMesh(
dpath % suff + 'upsample2/512-0.00/best-fit/',
'mapp2').paint()
ph1, ph1x = getps2D(h1, hmesh)
pl1, pl1x = getps2D(l1, lmesh)
suff = 'elg-nob2_ln%04d' % (elgn * 1e4)
h2, l2 = BigFileMesh(dpath % suff + 'upsample2/512-0.00/best-fit/',
'mapp').paint(), BigFileMesh(
dpath % suff + 'upsample2/512-0.00/best-fit/',
'mapp2').paint()
ph2, ph2x = getps2D(h2, hmesh)
pe2, pe2x = getps2D(l2, emesh)
print(k.shape)
print(ph.shape)
fig, ax = plt.subplots(1, 2, figsize=(9, 4))
ax[0].plot(k[:, 0],
ph0x[:, 0] / (ph0[:, 0] * ph[:, 0])**0.5,
'C0',
label='HI data')
#if not hirax: ax[0].plot(k[:, 0], ph3x[:, 0]/(ph3[:, 0]*ph[:, 0])**0.5, 'C4--', lw=2, alpha=0.7)
ax[0].plot(k[:, 0],
ph1x[:, 0] / (ph1[:, 0] * ph[:, 0])**0.5,
'C1',
label='HI + LSST data')
ax[0].plot(k[:, 0],
ph2x[:, 0] / (ph2[:, 0] * ph[:, 0])**0.5,
'C2',
label='HI + DESI data')
#ax[0].plot(k[:, 0], pl0x/(pl0*pl)**0.5, 'C0--')
ax[0].plot(k[:, 0], pl1x[:, 0] / (pl1[:, 0] * pl[:, 0])**0.5, 'C1--')
ax[0].plot(k[:, 0], pe2x[:, 0] / (pe2[:, 0] * pe[:, 0])**0.5, 'C2:')
ax[-1].plot(k[:, 0],
ph0x[:, -1] / (ph0[:, -1] * ph[:, -1])**0.5,
'C0',
label='HI')
#if not hirax: ax[-1].plot(k[:, 0], ph3x[:, -1]/(ph3[:, -1]*ph[:, -1])**0.5, 'C4--', lw=2, alpha=0.7)
ax[-1].plot(k[:, 0], ph1x[:, -1] / (ph1[:, -1] * ph[:, -1])**0.5, 'C1')
ax[-1].plot(k[:, 0], ph2x[:, -1] / (ph2[:, -1] * ph[:, -1])**0.5, 'C2')
#ax[-1].plot(k[:, 0], pl0x/(pl0*pl)**0.5, 'C0--')
ax[-1].plot(k[:, 0],
pl1x[:, -1] / (pl1[:, -1] * pl[:, -1])**0.5,
'C1--',
label='LSST')
ax[-1].plot(k[:, 0],
pe2x[:, -1] / (pe2[:, -1] * pe[:, -1])**0.5,
'C2:',
label='DESI')
# ax[1].plot(k, (ph0/ph)**0.5, 'C0', label='HI')
# if not hirax: ax[1].plot(k, (ph1/ph)**0.5, 'C4--', label='HI data2', lw=2, alpha=0.5)
# ax[1].plot(k, (ph1/ph)**0.5, 'C1')
# ax[1].plot(k, (ph2/ph)**0.5, 'C2')
# #ax[1].plot(k, (pl0/pl)**0.5, 'C0--', label='LSST')
# ax[1].plot(k, (pl1/pl)**0.5, 'C1--', label='LSST')
# ax[1].plot(k, (pe2/pe)**0.5, 'C2:', label='DESI')
#
ax[0].set_ylabel('$r_{cc},\, \mu=0.1$', fontdict=font)
ax[1].set_ylabel('$r_{cc},\, \mu=0.9$', fontdict=font)
ax[0].set_ylim(0.5, 1.05)
ax[1].set_ylim(0.5, 1.05)
for axis in ax:
axis.set_xlabel(r'$k\quad [h\,{\rm Mpc}^{-1}]$', fontdict=font)
for axis in ax.flatten():
axis.set_xscale('log')
axis.grid(which='both')
axis.legend(prop=fontmanage)
# Put on some more labels.
for axis in ax.flatten():
for tick in axis.xaxis.get_major_ticks():
tick.label.set_fontproperties(fontmanage)
for tick in axis.yaxis.get_major_ticks():
tick.label.set_fontproperties(fontmanage)
plt.tight_layout(rect=[0, 0, 1, 0.95])
if rank == 0:
if hirax:
plt.savefig(figpath + '/lsst_N%04d_%04d-hirax-2D-nob2.pdf' %
(nc, aa * 10000))
else:
plt.savefig(figpath + '/lsst_N%04d_%04d-2D-nob2.pdf' %
(nc, aa * 10000))
def plotz4_2D(nc=nc):
aa = 0.2000
lsstn = 0.0350
dpath = '/global/cscratch1/sd/chmodi/m3127/21cm_cleaning/reconlsstv2//fastpm_0.2000/wedge_kmin0.03_pess/L1024-N0256-R/thermal-reas-hex/ZA/opt_s777_h1massD_%s_rsdpos/'
if nc == 256:
hmesh = BigFileMesh(dpath % ('lwt0') + '/datap', 'mapp').paint()
lmesh = BigFileMesh(dpath % ('lwt0') + '/datap', 'mapp2').paint()
ph = FFTPower(hmesh, mode='2d').power
k, ph = ph['k'], ph['power']
pl = FFTPower(lmesh, mode='2d').power['power']
suff = 'lwt0'
h0, l0 = BigFileMesh(dpath % suff + '256-0.00/best-fit/',
'mapp').paint(), BigFileMesh(
dpath % suff + '256-0.00/best-fit/',
'mapp2').paint()
ph0, ph0x = getps2D(h0, hmesh)
pl0, pl0x = getps2D(l0, lmesh)
suff = 'lsst_ln%04d' % (3.5e-3 * 1e4)
h1, l1 = BigFileMesh(dpath % suff + '256-0.00/best-fit/',
'mapp').paint(), BigFileMesh(
dpath % suff + '256-0.00/best-fit/',
'mapp2').paint()
ph1, ph1x = getps2D(h1, hmesh)
pl1, pl1x = getps2D(l1, lmesh)
elif nc == 512:
hmesh = BigFileMesh(dpath % ('lwt0') + '/datap_up', 'mapp').paint()
lmesh = BigFileMesh(dpath % ('lwt0') + '/datap_up', 'mapp2').paint()
ph = FFTPower(hmesh, mode='2d').power
k, ph = ph['k'], ph['power']
pl = FFTPower(lmesh, mode='2d').power['power']
suff = 'lwt0'
h0, l0 = BigFileMesh(dpath % suff + 'upsample2/512-0.00/best-fit/',
'mapp').paint(), BigFileMesh(
dpath % suff + 'upsample2/512-0.00/best-fit/',
'mapp2').paint()
ph0, ph0x = getps2D(h0, hmesh)
pl0, pl0x = getps2D(l0, lmesh)
suff = 'lsst_ln%04d' % (3.5e-3 * 1e4)
h1, l1 = BigFileMesh(dpath % suff + 'upsample2/512-0.00/best-fit/',
'mapp').paint(), BigFileMesh(
dpath % suff + 'upsample2/512-0.00/best-fit/',
'mapp2').paint()
ph1, ph1x = getps2D(h1, hmesh)
pl1, pl1x = getps2D(l1, lmesh)
fig, ax = plt.subplots(1, 2, figsize=(9, 4))
ax[0].plot(k[:, 0],
ph0x[:, 0] / (ph0[:, 0] * ph[:, 0])**0.5,
'C0',
label='HI data')
ax[0].plot(k[:, 0],
ph1x[:, 0] / (ph1[:, 0] * ph[:, 0])**0.5,
'C1',
label='HI + LSST data')
ax[0].plot(k[:, 0], pl0x[:, 0] / (pl0[:, 0] * pl[:, 0])**0.5, 'C0--')
ax[0].plot(k[:, 0], pl1x[:, 0] / (pl1[:, 0] * pl[:, 0])**0.5, 'C1--')
ax[-1].plot(k[:, 0],
ph0x[:, -1] / (ph0[:, -1] * ph[:, -1])**0.5,
'C0',
label='HI')
ax[-1].plot(k[:, 0], ph1x[:, -1] / (ph1[:, -1] * ph[:, -1])**0.5, 'C1')
ax[-1].plot(k[:, 0],
pl0x[:, -1] / (pl0[:, -1] * pl[:, -1])**0.5,
'C0--',
label='LSST')
ax[-1].plot(k[:, 0], pl1x[:, -1] / (pl1[:, -1] * pl[:, -1])**0.5, 'C1--')
ax[0].set_ylabel('$r_{cc},\, \mu=0.1$', fontdict=font)
ax[1].set_ylabel('$r_{cc},\, \mu=0.9$', fontdict=font)
ax[0].set_ylim(0.5, 1.05)
ax[1].set_ylim(0.5, 1.05)
for axis in ax:
axis.set_xlabel(r'$k\quad [h\,{\rm Mpc}^{-1}]$', fontdict=font)
for axis in ax.flatten():
axis.set_xscale('log')
axis.grid(which='both')
axis.legend(prop=fontmanage)
# Put on some more labels.
for axis in ax.flatten():
axis.axhline(1, color='k', ls="--")
for tick in axis.xaxis.get_major_ticks():
tick.label.set_fontproperties(fontmanage)
for tick in axis.yaxis.get_major_ticks():
tick.label.set_fontproperties(fontmanage)
plt.tight_layout(rect=[0, 0, 1, 0.95])
if rank == 0:
plt.savefig(figpath + '/lsst_N%04d_%04d-2D.pdf' % (nc, aa * 10000))
optfolder = optfolder[:-1] + 'iter200/'
if truth_pm.comm.rank == 0: print('Output Folder is %s' % optfolder)
for folder in [ofolder, optfolder]:
try:
os.makedirs(folder)
except:
pass
####################################
#initiate
if rsdpos:
if ray:
hmesh = BigFileMesh(
'/global/cscratch1/sd/chmodi/m3127/H1mass/highres/%d-9100/fastpm_%0.4f/HImeshz-N%04d/'
% (bs * 10, aa, nc), 'ModelA').paint()
else:
hmesh = BigFileMesh(
'/global/cscratch1/sd/chmodi/m3127/H1mass/highres/%d-9100-fixed/fastpm_%0.4f/HImeshz-N%04d/'
% (bs * 10, aa, nc), 'ModelA').paint()
if rsdpos:
if ray:
pp = '/global/cscratch1/sd/chmodi/m3127/H1mass/highres/10240-9100/fastpm_%0.4f/Header/attr-v2' % aa
else:
pp = '/global/cscratch1/sd/chmodi/m3127/H1mass/highres/10240-9100-fixed/fastpm_%0.4f/Header/attr-v2' % aa
with open(pp) as ff:
for line in ff.readlines():
if 'RSDFactor' in line: rsdfac = float(line.split()[-2])
else:
if ray:
if rank == 0: print(dynamic_model)
#noise
#Artifically low noise since the data is constructed from the model
truth_noise_model = map.NoiseModel(
truth_pm, None,
noisevar * (truth_pm.BoxSize / truth_pm.Nmesh).prod(), 1234)
truth_noise_model = None
#Create and save data if not found
s_truth = truth_pm.generate_whitenoise(seed, mode='complex')\
.apply(lambda k, v: v * (ipk(sum(ki **2 for ki in k) **0.5) / v.BoxSize.prod()) ** 0.5)\
.c2r()
s_truth = BigFileMesh(dfolder + 'mesh', 's').paint()
d_truth = BigFileMesh(dfolder + 'mesh', 'd').paint()
try:
params = numpy.loadtxt(optfolder + '/params.txt')
kerror, perror = numpy.loadtxt(optfolder + '/error_ps.txt', unpack=True)
except Exception as e:
mock_model_setup = map.MockModel(dynamic_model,
rsdpos=rsdpos,
rsdfac=rsdfac)
fpos, linear, linearsq, shear = mock_model_setup.get_code().compute(
['xp', 'linear', 'linearsq', 'shear'], init={'parameters': s_truth})
grid = truth_pm.generate_uniform_particle_grid(shift=0.0, dtype='f4')
params, bmod = getbias(truth_pm, hmesh, [linear, linearsq, shear], fpos,
grid)
title = ['%0.3f' % i for i in params]
def load_TNG_map(TNG_basepath, snapNum, field, pm):
assert field in ['dm', 'Mstar', 'ne', 'T', 'nHI', 'neVz', 'MstarVz']
#Try directly loading map. If it does not exist, paint the map and save it for future use.
address = TNG_basepath + '/snapdir_' + str(snapNum).zfill(
3) + '/' + field + 'map_Nmesh' + str(pm.Nmesh[0])
try:
TNGmap = BigFileMesh(address, dataset='Field').to_real_field()
except:
if field == 'dm':
pos = []
for mdi in range(3):
pos.append(
load_TNG_data(TNG_basepath=TNG_basepath,
snapNum=snapNum,
partType='dm',
field='Coordinates',
mdi=mdi))
pos = np.array(pos).T
mass = 1.0 * pm.Nmesh.prod() / comm.allreduce(len(pos), op=MPI.SUM)
layout = pm.decompose(pos)
pos1 = layout.exchange(pos)
del pos
TNGmap = pm.create(type="real")
TNGmap.paint(pos1, mass=mass, layout=None, hold=False)
del pos1
elif field == 'Mstar':
mass = load_TNG_data(TNG_basepath=TNG_basepath,
snapNum=snapNum,
partType='stars',
field='Masses')
pos = np.zeros((len(mass), 3), dtype=np.float32)
for mdi in range(3):
pos[:, mdi] = load_TNG_data(TNG_basepath=TNG_basepath,
snapNum=snapNum,
partType='stars',
field='Coordinates',
mdi=mdi)
layout = pm.decompose(pos)
pos1 = layout.exchange(pos)
del pos
mass1 = layout.exchange(mass)
del mass
TNGmap = pm.create(type="real")
TNGmap.paint(pos1, mass=mass1, layout=None, hold=False)
del pos1, mass1
elif field == 'ne':
raise NotImplementedError
elif field == 'T':
raise NotImplementedError
elif field == 'nHI':
def nHI(mass, XHI):
h = 0.6774
XH = 0.76
mp = 1.6726219e-27
Msun10 = 1.989e40
BoxSize = 205.
Mpc_cm = 3.085677581e24
return Msun10 / h / (BoxSize / args.Nmesh * Mpc_cm /
h)**3 * mass * XH / mp * XHI
gasmass = load_TNG_data(TNG_basepath=TNG_basepath,
snapNum=snapNum,
partType='gas',
field='Masses')
XHI = load_TNG_data(TNG_basepath=TNG_basepath,
snapNum=snapNum,
partType='gas',
field='NeutralHydrogenAbundance')
mass = nHI(gasmass, XHI)
pos = np.zeros((len(mass), 3), dtype=np.float32)
for mdi in range(3):
pos[:, mdi] = load_TNG_data(TNG_basepath=TNG_basepath,
snapNum=snapNum,
partType='gas',
field='Coordinates',
mdi=mdi)
layout = pm.decompose(pos)
pos1 = layout.exchange(pos)
del pos
mass1 = layout.exchange(mass)
del mass
TNGmap = pm.create(type="real")
TNGmap.paint(pos1, mass=mass1, layout=None, hold=False)
del pos1, mass1
elif field == 'ne_vz':
raise NotImplementedError
elif field == 'Mstar_vz':
raise NotImplementedError
FieldMesh(TNGmap).save(address)
return TNGmap
def make_bsum_plot():
"""bias rcc, tf, error, bk"""
ik = args.ik
grid = pm.mesh_coordinates()*bs/nc
noises = np.loadtxt('/global/u1/c/chmodi/Programs/21cm/21cm_cleaning/data/summaryHI.txt').T
for i in range(noises[0].size):
if noises[0][i] == np.round(1/aa-1, 2): noise = noises[3][i]
print(noise)
if args.ray:
hmesh = BigFileMesh('/global/cscratch1/sd/chmodi/m3127/H1mass/highres/%d-9100/fastpm_%0.4f/HImesh-N%04d/'%(bs*10, aa, nc), 'ModelA').paint()
lin = BigFileMesh('/global/cscratch1/sd/chmodi/m3127/cm_lowres/5stepT-B1/%d-%d-9100/linear'%(bs, nc), 'LinearDensityK').paint()
dyn = BigFileCatalog('/global/cscratch1/sd/chmodi/m3127/cm_lowres/5stepT-B1/%d-%d-9100/fastpm_%0.4f/1'%(bs, nc, aa), comm=pm.comm, header='Header')
else:
hmesh = BigFileMesh('/global/cscratch1/sd/chmodi/m3127/H1mass/highres/%d-9100-fixed/fastpm_%0.4f/HImesh-N%04d/'%(bs*10, aa, nc), 'ModelA').paint()
lin = BigFileMesh('/global/cscratch1/sd/chmodi/m3127/cm_lowres/5stepT-B1/%d-%d-9100-fixed/linear'%(bs, nc), 'LinearDensityK').paint()
dyn = BigFileCatalog('/global/cscratch1/sd/chmodi/m3127/cm_lowres/5stepT-B1/%d-%d-9100-fixed/fastpm_%0.4f/1'%(bs, nc, aa), comm=pm.comm, header='Header')
print(pm.comm.rank, dyn.Index.compute())
fpos = dyn['Position']
flay = pm.decompose(fpos)
#
dgrow = cosmo.scale_independent_growth_factor(zz)
zapos = za.doza(lin.r2c(), grid, z=zz, dgrow=dgrow)
print(zapos.shape, fpos.shape)
paramsza, zamod = getbias(pm, basemesh=lin, hmesh=hmesh, pos=zapos, grid=grid, ik=ik)
_, pmmod = getbias(pm, basemesh=lin, hmesh=hmesh, pos=fpos, grid=grid, ik=ik)
kk, paramszak, _ = getbiask(pm, basemesh=lin, hmesh=hmesh, pos=zapos, grid=grid)
#
fin = pm.paint(fpos, layout=flay)
fin /= fin.cmean()
fin -= 1
finsm = ft.smooth(fin, 3, 'gauss')
_, finmod = getbias(pm, basemesh=finsm, hmesh=hmesh, pos=grid, grid=grid, ik=ik)
models = [zamod, pmmod, finmod, finsm.copy()]
lbls = ['ZA shift', 'PM shift', 'Eulerian (R=3)', 'Eulerian (R=3), $b_1^E$']
lss = ['-', '-', '-', '--']
print('Setup done')
#####
fig, ax = plt.subplots(2, 2, figsize=(8, 6), sharex=True)
for ii, mod in enumerate(models):
pmod = FFTPower(mod, mode='1d').power
k, pmod = pmod['k'], pmod['power']
ph = FFTPower(hmesh, mode='1d').power['power']
if ii == 3:
mod *= (ph[1]/pmod[1]).real**0.5
pmod *= (ph[1]/pmod[1]).real
pxmodh = FFTPower(hmesh, second=mod, mode='1d').power['power']
perr = FFTPower(hmesh -mod, mode='1d').power['power']
ax[0, 0].plot(k, pxmodh/(pmod*ph)**0.5, label=lbls[ii], lw=2, ls=lss[ii])
ax[0, 0].set_ylabel('$r_{cc}$', fontdict=font)
#if ii == 3: pmod *= ph[1]/pmod[1]
ax[0, 1].plot(k,(pmod/ph)**0.5, lw=2, ls=lss[ii])
ax[0, 1].set_ylabel(r'$T_f$', fontdict=font)
#ax[1].set_ylabel(r'$\sqrt{P_{\rm mod}/P_{hh}}$', fontdict=font)
ax[1, 0].plot(k, perr, lw=2, ls=lss[ii])
ax[1, 0].set_ylabel(r'$P_{\rm err}$', fontdict=font)
ax[1, 0].set_yscale('log')
ax[1, 0].axhline(noise, color='k', ls="--")
ax[1, 0].set_ylim(2, 5e2)
lbls = ['$b_1$', '$b_2$', '$b_s$']
for ii in range(3):
ax[1, 1].plot(kk, paramszak[ii], lw=2, color='C%d'%ii, label=lbls[ii])
ax[1, 1].axhline(paramsza[ii], lw=1, color='C%d'%ii, ls="--")
ax[1, 1].set_ylabel(r'$b(k)$', fontdict=font)
for axis in ax.flatten():
axis.set_xscale('log')
axis.grid(which='both', alpha=0.2, color='gray', lw=0.2)
axis.legend(prop=fontmanage)
axis.set_xlabel(r'$k\quad [h\,{\rm Mpc}^{-1}]$', fontdict=font)
axis.set_xscale('log')
axis.axvline(kk[ik], color='k', ls="--")
# Put on some more labels.
for axis in ax.flatten():
axis.set_xscale('log')
for tick in axis.xaxis.get_major_ticks():
tick.label.set_fontproperties(fontmanage)
for tick in axis.yaxis.get_major_ticks():
tick.label.set_fontproperties(fontmanage)
##and finish
plt.tight_layout(rect=[0, 0, 1, 0.95])
if args.ray:
plt.savefig(figpath + '/bsum_L%04d_N%04d_%04d_ik%d.pdf'%(bs, nc, aa*10000, ik))
else:
plt.savefig(figpath + '/bsumfix_L%04d_N%04d_%04d_ik%d.pdf'%(bs, nc, aa*10000, ik))
optfolder = ofolder + 'opt_%s/' % fname
if truth_pm.comm.rank == 0: print('Output Folder is %s' % optfolder)
for folder in [ofolder, optfolder]:
try:
os.makedirs(folder)
except:
pass
####################################
#initiate
#
if ray:
hmeshreal = BigFileMesh(
'/global/cscratch1/sd/chmodi/m3127/H1mass/highres/%d-9100/fastpm_%0.4f/HImesh-N%04d/'
% (bs * 10, aa, ncd), 'ModelA').paint()
else:
hmeshreal = BigFileMesh(
'/global/cscratch1/sd/chmodi/m3127/H1mass/highres/%d-9100-fixed/fastpm_%0.4f/HImesh-N%04d/'
% (bs * 10, aa, ncd), 'ModelA').paint()
hmeshreal /= hmeshreal.cmean()
hmeshreal -= 1
if rsdpos:
if ray:
hmesh = BigFileMesh(
'/global/cscratch1/sd/chmodi/m3127/H1mass/highres/%d-9100/fastpm_%0.4f/HImeshz-N%04d/'
% (bs * 10, aa, ncd), 'ModelA').paint()
else:
hmesh = BigFileMesh(
[0.1429, 0.1538, 0.1667, 0.1818, 0.2000, 0.2222, 0.2500, 0.2857, 0.3333])
#aafiles = aafiles[:1]
zzfiles = np.array([round(tools.atoz(aa), 2) for aa in aafiles])
hpos, hmass, hid, h1mass = tools.readinhalos(aafiles, sim)
print('Bin Halos')
hbins, hcount, hm, hh1 = {}, {}, {}, {}
dmesh, pkm = {}, {}
for iz, zz in enumerate(zzfiles):
print('For redshift : ', zz)
#measure power
dmesh[zz] = BigFileMesh(
dpath + sim + '/fastpm_%0.4f/' % aafiles[iz] + '/dmesh_N%04d' % nc,
'1').paint()
pk = FFTPower(dmesh[zz] / dmesh[zz].cmean(), mode='1d').power
k, pkm = pk['k'], pk['power']
hbins[zz] = np.logspace(
np.log10(hmass[zz][-1]) - 0.01,
np.log10(hmass[zz][0]) - 0.01, 100)
bias_table = []
massval = []
inb = 6
posmesh = pm.paint(hpos[zz])
massmesh = pm.paint(hpos[zz], mass=hmass[zz])