def NeutHalos(mneut, nreal, nzbin):
''' Read halo catalogs generated by Paco
parameters
----------
mneut : float,
total neutrino mass
nreal : int,
realization number
nzbin : int,
integer specifying the redshift of the snapshot.
nzbin = 0 --> z=3
nzbin = 1 --> z=2
nzbin = 2 --> z=1
nzbin = 3 --> z=0.5
nzbin = 4 --> z=0
'''
if mneut == 0.1:
dir = ''.join([UT.dat_dir(), '0.10eV/', str(nreal)])
else:
dir = ''.join([UT.dat_dir(), str(mneut), 'eV/', str(nreal)])
# read in Gadget header
header = RS.read_gadget_header(''.join(
[dir, '/snapdir_',
str(nzbin).zfill(3), '/snap_',
str(nzbin).zfill(3)]))
# get cosmology from header
Omega_b = 0.049 # fixed baryon
cosmo = NBlab.cosmology.Planck15.clone(Omega_cdm=header['Omega_m'] -
Omega_b,
h=header['h'],
Omega_b=Omega_b)
Fof = readfof.FoF_catalog(dir,
nzbin,
long_ids=False,
swap=False,
SFR=False)
group_data = {}
group_data['Length'] = Fof.GroupLen
group_data['Position'] = Fof.GroupPos / 1e3
group_data['Velocity'] = Fof.GroupVel
group_data['Mass'] = Fof.GroupMass * 1e10
# calculate velocity offset
rsd_factor = (1. + header['z']) / (100. * cosmo.efunc(header['z']))
group_data['VelocityOffset'] = group_data['Velocity'] * rsd_factor
#group_data['RSDPosition'] = group_data['Position'] + group_data['VelocityOffset'] * LOS
# save to ArryCatalog for consistency
cat = NBlab.ArrayCatalog(group_data,
BoxSize=np.array([1000., 1000., 1000.]))
cat = NBlab.HaloCatalog(cat, cosmo=cosmo, redshift=header['z'], mdef='vir')
return cat
def _check_X_HODLHD(
seed_hod,
obvs='plk',
ell=0,
Nmesh=360,
rsd=True,
karr=False, # kwargs specifying P(k)
prior='sinha2017prior_narrow',
samples=40,
method='mdu', # kwargs specifying the LHD
silent=True):
''' check whether all the X_HODLHD files are there
'''
f_dir = ''.join([
UT.dat_dir(), 'lhd/onlyHOD/', method, '_',
str(samples), '_', prior, '/'
])
# rsd flag
if rsd: rsd_str = 'z'
else: rsd_str = 'r'
# read in observable
for i in range(samples):
if i == 21: continue # 21 requires ton of memory
# directory of theta_i,LHD
fname = ''.join([
f_dir, 'HOD', method, '_seed',
str(seed_hod), '_',
str(i), '/', 'pk.menut0.0.nreal1.nzbin4.', rsd_str, 'space.',
str(Nmesh), '.nbkt.dat'
])
if not os.path.isfile(fname):
print('%s does not exist' % fname)
return None
def HOD_LHD(prior=None, samples=None, method=None, ndim=None, overwrite=False):
''' Return latin hypercubes with `samples` elements using `method` method
that spans the specified prior of the vanilla Zheng et al.(2007) HOD model.
References
-------
- Zheng Z., et al. (2007) -- arXiv:0512071
- Sinha M., et al. (2017) -- arXiv:1708.04892
'''
if ndim is None: str_ndim = ''
else: str_ndim = str(ndim) + 'D.'
fname = ''.join([
UT.dat_dir(), 'lhd/onlyHOD/', 'HOD_LHD.', str_ndim, prior, '.', method,
'.',
str(samples), 'samples.dat'
])
if os.path.isfile(fname) and not overwrite: # file exists
lhcube = np.loadtxt(fname, skiprows=4)
if ndim is None:
# skip 21st LHD parameter
indx = np.ones(lhcube.shape[0], dtype=bool)
indx[21] = False
lhcube = lhcube[indx, :]
else:
hodprior = HODprior(prior)
hod_range = hodprior.range()
if ndim is None: # Nsample x Ndim latin hypercube
lhcube = lhd.thetaLHD(hod_range,
samples=samples,
method=method,
ndim=ndim)
elif ndim == 1: # only vary alpha
lhcube = np.zeros((samples, len(hod_range[0])))
for i in range(4):
lhcube[:, i] = 0.5 * (hod_range[0] + hod_range[1])[i]
lhc = lhd.thetaLHD([[hod_range[0][4]], [hod_range[1][4]]],
samples=samples,
method=method,
ndim=ndim)
lhcube[:, 4] = lhc.flatten()
f = open(fname, 'w')
f.write('# ' + prior + '\n')
f.write('# parameters : ' + ', '.join(hodprior.labels) + '\n')
f.write('# theta min : ' + ', '.join([str(r)
for r in hod_range[0]]) + '\n')
f.write('# theta max : ' + ', '.join([str(r)
for r in hod_range[1]]) + '\n')
for i in range(lhcube.shape[0]):
f.write('%f' % (lhcube[i, 0]))
for j in range(1, lhcube.shape[1]):
f.write('\t %f' % (lhcube[i, j]))
f.write('\n')
f.close()
return lhcube
def X_HODLHD(
seed_hod,
obvs='plk',
ell=0,
Nmesh=360,
rsd=True,
karr=False, # kwargs specifying P(k)
prior='sinha2017prior_narrow',
samples=40,
method='mdu',
ndim=None, # kwargs for LHD
silent=True):
''' Read observable (e.g. P(k)) for a LHD of HOD parameters (specified by prior,
samples, and method)
'''
# directory of the catalogs
if ndim is None: str_ndim = ''
else: str_ndim = '_' + str(ndim) + 'D'
f_dir = ''.join([
UT.dat_dir(), 'lhd/onlyHOD/', method, '_',
str(samples), '_', prior, str_ndim, '/'
])
# rsd flag
if rsd: rsd_str = 'z'
else: rsd_str = 'r'
# read in observable
pks = []
for i in range(samples):
# 21st LHD HOD theta requires ton of memory
# this should be addressed more robustly but for
# the sake of my sanity at the moment, I will skip
# it for now.
if i == 21: continue
# directory of theta_i,LHD
fname = ''.join([
f_dir, 'HOD', method, '_seed',
str(seed_hod), '_',
str(i), '/', 'pk.menut0.0.nreal1.nzbin4.', rsd_str, 'space.',
str(Nmesh), '.nbkt.dat'
])
if not silent:
print('%i -- reading %s' % (i, fname))
# read in plks
k, pk = np.loadtxt(fname,
skiprows=4,
unpack=True,
usecols=[0, 1 + ell / 2])
pks.append(pk)
if karr:
return k, np.array(pks)
else:
return np.array(pks)
def NeutParticles(mneut, nreal, nzbin, clobber=False):
''' Read particle catalog generated by Paco and return NBlab.ArrayCatalog
parameters
----------
mneut : float,
total neutrino mass
nreal : int,
realization number
nzbin : int,
integer specifying the redshift of the snapshot.
nzbin = 0 --> z=3
nzbin = 1 --> z=2
nzbin = 2 --> z=1
nzbin = 3 --> z=0.5
nzbin = 4 --> z=0
clobber : bool, optional
if True, reconstructs the BigFile data
'''
dir = ''.join([
UT.dat_dir(),
str(mneut), 'eV/',
str(nreal), '/snapdir_',
str(nzbin).zfill(3), '/'
])
dir_list = [
dir + '/Matter' + sub for sub in ['', '/Position', '/Velocity', '/ID']
]
if (not np.all([os.path.isdir(dd) for dd in dir_list])) or clobber:
f = ''.join([dir, 'snap_', str(nzbin).zfill(3)])
# read in Gadget header
header = RS.read_gadget_header(f)
# read in CDM particles (parttype = 1) and create catalogue
particle_data = {}
particle_data['Position'] = RS.read_block(f, 'POS ',
parttype=1) / 1000. # Mpc/h
particle_data['Velocity'] = RS.read_block(f, 'VEL ', parttype=1)
particle_data['ID'] = RS.read_block(f, 'ID ', parttype=1)
cat = NBlab.ArrayCatalog(particle_data,
BoxSize=np.array([
header['boxsize'], header['boxsize'],
header['boxsize']
]))
#cat['KDDensity'] = KDDensity(cat).density
cat.save(dir + '/Matter', ["Position", "Velocity", "ID"])
else:
cat = NBlab.BigFileCatalog(dir + '/Matter', header='Header')
return cat
def X_testHODLHD(
seed_hod,
obvs='plk',
ell=0,
Nmesh=360,
rsd=True,
karr=False, # kwargs specifying P(k)
prior='sinha2017prior_narrow',
samples=20,
ndim=None, # kwargs specifying the LHD
silent=True):
''' Read observable (e.g. P(k)) for a LHD of HOD parameters (specified by prior,
samples, and method)
'''
# directory of the catalogs
if ndim is None: str_ndim = ''
else: str_ndim = '_' + str(ndim) + 'D'
f_dir = ''.join([
UT.dat_dir(), 'lhd/onlyHOD/test_',
str(samples), '_', prior, str_ndim, '/'
])
# rsd flag
if rsd: rsd_str = 'z'
else: rsd_str = 'r'
# read in observable
pks = []
for i in range(samples):
# directory of theta_i,LHD
fname = ''.join([
f_dir, 'testHOD_seed',
str(seed_hod), '_',
str(i), '/', 'pk.menut0.0.nreal1.nzbin4.', rsd_str, 'space.',
str(Nmesh), '.nbkt.dat'
])
if not silent:
print('%i -- reading %s' % (i, fname))
# read in plks
k, pk = np.loadtxt(fname,
skiprows=4,
unpack=True,
usecols=[0, 1 + ell / 2])
pks.append(pk)
if karr:
return k, np.array(pks)
else:
return np.array(pks)
def HODLHDpks(seed_hod,
i_lhd,
prior='sinha2017prior_narrow',
method='mdu',
samples=17,
ndim=None,
Nmesh=360,
rsd=True):
''' Measure and write P(k|theta_i,LHD) for the HOD LHD that spans the `prior`
'''
# RSD flag
if rsd: str_rsd = '.zspace'
else: str_rsd = '.rspace'
# directory
if ndim is None: str_ndim = ''
else: str_ndim = '_' + str(ndim) + 'D'
parent_dir = ''.join([
UT.dat_dir(), 'lhd/onlyHOD/', method, '_',
str(samples), '_', prior, str_ndim, '/HOD', method, '_seed',
str(seed_hod), '_',
str(i_lhd), '/'
])
fname = ''.join([
parent_dir, 'pk.menut0.0.nreal1.nzbin4', str_rsd, '.',
str(Nmesh), '.nbkt.dat'
])
if not os.path.exists(parent_dir): # make directory
raise ValueError("the directory \n %s \n does not exist!" % parent_dir)
# read from galaxy catalog
gals = NBlab.BigFileCatalog(parent_dir, header='Header')
plk = FM.Observables(gals, observable='plk', rsd=rsd, Nmesh=Nmesh)
# save to file
f = open(fname, 'w')
f.write("### header ### \n")
f.write("# shotnoise %f \n" % plk['shotnoise'])
f.write("# columns : k , P0, P2, P4 \n")
f.write('### header ### \n')
for ik in range(len(plk['k'])):
f.write("%f \t %f \t %f \t %f" %
(plk['k'][ik], plk['p0k'][ik], plk['p2k'][ik], plk['p4k'][ik]))
f.write("\n")
f.close()
return None
def _NeutHalos(mneut, nreal, nzbin, clobber=False):
''' Construct Friend-of-Friend halo catalog from Paco's neutrino
particle catalogs.
'''
dir = ''.join([
UT.dat_dir(),
str(mneut), 'eV/',
str(nreal), '/snapdir_',
str(nzbin).zfill(3), '/'
])
f = ''.join([dir, 'snap_', str(nzbin).zfill(3)])
# read in Gadget header
header = RS.read_gadget_header(f)
# get cosmology from header
Omega_b = 0.049 # fixed baryon
cosmo = NBlab.cosmology.Planck15.clone(Omega_cdm=header['Omega_m'] -
Omega_b,
h=header['h'],
Omega_b=Omega_b)
if (not os.path.isdir(dir + '/FOF')) or clobber:
# DM particle mass (parttype=1)
m_part = header['masses'][1]
cat = NeutParticles(mneut, nreal,
nzbin) # read in neutrino particles with
cat.attrs['Nmesh'] = [512, 512, 512]
# calculate friend-of-friend with only CDM particles
fof = NBlab.FOF(cat, linking_length=0.2, nmin=20)
# now make them into halos
fofcat = fof.to_halos(particle_mass=m_part,
cosmo=cosmo,
redshift=header['z'])
fofcat.save(
dir + '/FOF',
['Position', 'Velocity', 'VelocityOffset', 'Mass', 'Radius'])
else:
fofcat = NBlab.BigFileCatalog(dir + '/FOF', header='Header')
return fofcat
def testHOD_LHD(prior=None, samples=None, overwrite=False, ndim=None, seed=1):
''' Return parameters of the vanilla Zheng et al.(2007) HOD model
for testing the LHD. These values will be significantly more within
the boundaries of the parameter space
'''
if ndim is None: str_ndim = ''
else: str_ndim = str(ndim) + 'D.'
fname = ''.join([
UT.dat_dir(), 'lhd/onlyHOD/', 'HOD_LHDtest.', str_ndim, prior,
'.rseed',
str(seed), '.',
str(samples), 'samples.dat'
])
if os.path.isfile(fname) and not overwrite: # file exists
theta_test = np.loadtxt(fname, skiprows=4)
else:
# prior and its range
hodprior = HODprior(prior)
hod_range = hodprior.range()
np.random.seed(seed)
theta_test = np.zeros((samples, len(hod_range[0])))
if ndim is None:
for i in range(theta_test.shape[1]):
dHOD_range = (hod_range[1] - hod_range[0])[i]
# linearly spaced in the inner 50% of the parameter space
ti = np.linspace(hod_range[0][i] + 0.25 * dHOD_range,
hod_range[1][i] - 0.25 * dHOD_range,
theta_test.shape[0])
np.random.shuffle(ti)
theta_test[:, i] = ti
elif ndim == 1:
# only alpha varies
dHOD_range = (hod_range[1] - hod_range[0])[4]
# linearly spaced in the inner 50% of the parameter space
ti = np.linspace(hod_range[0][4] + 0.25 * dHOD_range,
hod_range[1][4] - 0.25 * dHOD_range,
theta_test.shape[0])
np.random.shuffle(ti)
theta_test[:, 4] = ti
theta_test[:, 0] = 0.5 * (hod_range[0] + hod_range[1])[0]
theta_test[:, 1] = 0.5 * (hod_range[0] + hod_range[1])[1]
theta_test[:, 2] = 0.5 * (hod_range[0] + hod_range[1])[2]
theta_test[:, 3] = 0.5 * (hod_range[0] + hod_range[1])[3]
f = open(fname, 'w')
f.write('# ' + prior + '\n')
f.write('# parameters : ' + ', '.join(hodprior.labels) + '\n')
f.write('# theta min : ' + ', '.join([str(r)
for r in hod_range[0]]) + '\n')
f.write('# theta max : ' + ', '.join([str(r)
for r in hod_range[1]]) + '\n')
for i in range(theta_test.shape[0]):
f.write('%f' % (theta_test[i, 0]))
for j in range(1, theta_test.shape[1]):
f.write('\t %f' % (theta_test[i, j]))
f.write('\n')
f.close()
return theta_test
def HODLHDcatalogs(seed_hod,
i_lhd,
prior='sinha2017prior_narrow',
method='mdu',
samples=40,
ndim=None):
''' Generate HOD catalog of the i_lhd th parameter of the HOD LHD
that spans the Sinha M., et al. (2017) HOD parameter priors.
parameters
----------
seed_hod : int,
random seed for the HOD
prior : str, optional
string specifying the HOD range. Default is 'sinha2017prior', which uses
the prior from Sinha et al. (2017)
method : str
string specifying the method of LHD. Default is nohl
samples : int, optional
sample size of the LHD. Default is 17, which is from the fixed sample size of NOHL method
'''
if i_lhd >= samples:
raise ValueError("i_lhd has to be less than samples")
# read in Neutrino halo with m_neut = 0.0 eV, realization #1, at z_bin = 4
# this is hardcoded; don't change this
halos = Dat.NeutHalos(0.0, 1, 4)
# read in the HOD LHD
lhcube = onlyHOD.HOD_LHD(prior=prior,
samples=samples,
method=method,
ndim=ndim)
print('%i of %i LHD' % (i_lhd + 1, samples))
p_hod = {}
hodkeys = ['logMmin', 'sigma_logM', 'logM0', 'logM1', 'alpha']
for ik, k in enumerate(hodkeys):
p_hod[k] = lhcube[i_lhd, ik]
print(p_hod)
# populate the halo catalogs using HOD
gals = FM.Galaxies(halos, p_hod, seed=seed_hod)
# RSD position (hardcoded in the z direction)
gals['RSDPosition'] = FM.RSD(gals, LOS=[0, 0, 1])
# directory of the catalogs
if ndim is None: str_ndim = ''
else: str_ndim = '_' + str(ndim) + 'D'
parent_dir = ''.join([
UT.dat_dir(), 'lhd/onlyHOD/', method, '_',
str(samples), '_', prior, str_ndim, '/HOD', method, '_seed',
str(seed_hod), '_',
str(i_lhd), '/'
])
print('writing to %s ---------' % parent_dir)
if not os.path.exists(parent_dir): # make directory
os.mkdir(parent_dir)
# save to file
gals.save(parent_dir, ('Position', 'Velocity', 'RSDPosition'))
return None
def Fiducial_Obvs(obvs,
nreal,
nzbin,
seed_hod,
mneut=0.0,
Nmesh=360,
rsd=True,
HODrange='sinha2017prior_narrow',
silent=False):
''' Calculate and save observables of the fiducial HOD catalogs
'''
if mneut != 0.0:
raise ValueError("Fiducial should be calculated at m_nu=0.0eV")
if rsd: str_rsd = '.zspace'
else: str_rsd = '.rspace'
folder = ''.join([
UT.dat_dir(), 'lhd/',
str(mneut), 'eV_',
str(nreal), '_z',
str(nzbin), '_fiducial/', 'HOD_seed',
str(seed_hod), '/'
])
if obvs == 'plk':
fname = ''.join([
folder, 'pk.menut',
str(mneut), '.nreal',
str(nreal), '.nzbin',
str(nzbin), str_rsd, '.',
str(Nmesh), '.nbkt.dat'
])
if os.path.isfile(fname):
if not silent: print('--- reading from --- \n %s' % fname)
# read observalbe from file
k, p0k, p2k, p4k = np.loadtxt(fname,
skiprows=4,
unpack=True,
usecols=[0, 1, 2, 3])
obvs = {'k': k, 'p0k': p0k, 'p2k': p2k, 'p4k': p4k}
# readin shot-noise from header
f = open(fname, 'r')
_ = f.readline()
str_sn = f.readline()
obvs['shotnoise'] = float(str_sn.strip().split('shotnoise')[-1])
else:
gals = Fiducial_Catalog(nreal,
nzbin,
seed_hod,
mneut=mneut,
HODrange=HODrange)
if obvs == 'plk': # power spectrum multipole
plk = FM.Observables(gals, observable='plk', rsd=rsd, Nmesh=Nmesh)
# save to file
f = open(fname, 'w')
f.write("### header ### \n")
f.write("# shotnoise %f \n" % plk['shotnoise'])
f.write("# columns : k , P0, P2, P4 \n")
f.write('### header ### \n')
for ik in range(len(plk['k'])):
f.write("%f \t %f \t %f \t %f" %
(plk['k'][ik], plk['p0k'][ik], plk['p2k'][ik],
plk['p4k'][ik]))
f.write("\n")
f.close()
obvs = plk
else:
raise NotImplementedError('only Plk implemented')
return obvs
def Fiducial_Catalog(nreal,
nzbin,
seed_hod,
mneut=0.0,
HODrange='sinha2017prior_narrow'):
''' Generate fiducial HOD catalogs from specified m_nu = 0.0eV halo catalog
parameters
----------
mneut : float,
total neutrino mass
nreal : int,
realization number
nzbin : int,
integer specifying the redshift of the snapshot.
nzbin = 0 --> z=3
nzbin = 1 --> z=2
nzbin = 2 --> z=1
nzbin = 3 --> z=0.5
nzbin = 4 --> z=0
seed_hod : int,
random seed for the HOD
'''
if mneut != 0.0:
raise ValueError("Fiducial should be calculated at m_nu=0.0eV")
folder = ''.join([
UT.dat_dir(), 'lhd/',
str(mneut), 'eV_',
str(nreal), '_z',
str(nzbin), '_fiducial/', 'HOD_seed',
str(seed_hod), '/'
])
# read in Neutrino halo with mneut eV, realization # nreal, at z specified by nzbin
halos = NeutHalos(mneut, nreal, nzbin)
if not np.all([
os.path.exists(folder + subfold + '/')
for subfold in ['Position', 'Velocity', 'RSDPosition']
]):
# fiducial HOD parameter values
if HODrange in ['sinha2017prior', 'sinha2017prior_narrow']:
keylist = ['logMmin', 'sigma_logM', 'logM0', 'logM1', 'alpha']
p_hod = {
'logMmin': 11.60,
'sigma_logM': 0.26,
'logM0': 11.49,
'logM1': 12.83,
'alpha': 1.02
}
# populate the halo catalogs using HOD
gals = FM.Galaxies(halos, p_hod, seed=seed_hod)
# RSD position (hardcoded in the z direction)
gals['RSDPosition'] = FM.RSD(gals, LOS=[0, 0, 1])
parent_dir = '/'.join(folder[:-1].split('/')[:-1]) + '/'
if not os.path.exists(parent_dir): # make directory
os.mkdir(parent_dir)
# save to file
gals.save(folder, ('Position', 'Velocity', 'RSDPosition'))
else:
# read from file
gals = NBlab.BigFileCatalog(folder, header='Header')
gals.cosmo = halos.cosmo # save cosmology
return gals