def NeutHalos(mneut, nreal, nzbin): ''' Read halo catalogs generated by Paco parameters ---------- mneut : float, total neutrino mass nreal : int, realization number nzbin : int, integer specifying the redshift of the snapshot. nzbin = 0 --> z=3 nzbin = 1 --> z=2 nzbin = 2 --> z=1 nzbin = 3 --> z=0.5 nzbin = 4 --> z=0 ''' if mneut == 0.1: dir = ''.join([UT.dat_dir(), '0.10eV/', str(nreal)]) else: dir = ''.join([UT.dat_dir(), str(mneut), 'eV/', str(nreal)]) # read in Gadget header header = RS.read_gadget_header(''.join( [dir, '/snapdir_', str(nzbin).zfill(3), '/snap_', str(nzbin).zfill(3)])) # get cosmology from header Omega_b = 0.049 # fixed baryon cosmo = NBlab.cosmology.Planck15.clone(Omega_cdm=header['Omega_m'] - Omega_b, h=header['h'], Omega_b=Omega_b) Fof = readfof.FoF_catalog(dir, nzbin, long_ids=False, swap=False, SFR=False) group_data = {} group_data['Length'] = Fof.GroupLen group_data['Position'] = Fof.GroupPos / 1e3 group_data['Velocity'] = Fof.GroupVel group_data['Mass'] = Fof.GroupMass * 1e10 # calculate velocity offset rsd_factor = (1. + header['z']) / (100. * cosmo.efunc(header['z'])) group_data['VelocityOffset'] = group_data['Velocity'] * rsd_factor #group_data['RSDPosition'] = group_data['Position'] + group_data['VelocityOffset'] * LOS # save to ArryCatalog for consistency cat = NBlab.ArrayCatalog(group_data, BoxSize=np.array([1000., 1000., 1000.])) cat = NBlab.HaloCatalog(cat, cosmo=cosmo, redshift=header['z'], mdef='vir') return cat
def _check_X_HODLHD( seed_hod, obvs='plk', ell=0, Nmesh=360, rsd=True, karr=False, # kwargs specifying P(k) prior='sinha2017prior_narrow', samples=40, method='mdu', # kwargs specifying the LHD silent=True): ''' check whether all the X_HODLHD files are there ''' f_dir = ''.join([ UT.dat_dir(), 'lhd/onlyHOD/', method, '_', str(samples), '_', prior, '/' ]) # rsd flag if rsd: rsd_str = 'z' else: rsd_str = 'r' # read in observable for i in range(samples): if i == 21: continue # 21 requires ton of memory # directory of theta_i,LHD fname = ''.join([ f_dir, 'HOD', method, '_seed', str(seed_hod), '_', str(i), '/', 'pk.menut0.0.nreal1.nzbin4.', rsd_str, 'space.', str(Nmesh), '.nbkt.dat' ]) if not os.path.isfile(fname): print('%s does not exist' % fname) return None
def HOD_LHD(prior=None, samples=None, method=None, ndim=None, overwrite=False): ''' Return latin hypercubes with `samples` elements using `method` method that spans the specified prior of the vanilla Zheng et al.(2007) HOD model. References ------- - Zheng Z., et al. (2007) -- arXiv:0512071 - Sinha M., et al. (2017) -- arXiv:1708.04892 ''' if ndim is None: str_ndim = '' else: str_ndim = str(ndim) + 'D.' fname = ''.join([ UT.dat_dir(), 'lhd/onlyHOD/', 'HOD_LHD.', str_ndim, prior, '.', method, '.', str(samples), 'samples.dat' ]) if os.path.isfile(fname) and not overwrite: # file exists lhcube = np.loadtxt(fname, skiprows=4) if ndim is None: # skip 21st LHD parameter indx = np.ones(lhcube.shape[0], dtype=bool) indx[21] = False lhcube = lhcube[indx, :] else: hodprior = HODprior(prior) hod_range = hodprior.range() if ndim is None: # Nsample x Ndim latin hypercube lhcube = lhd.thetaLHD(hod_range, samples=samples, method=method, ndim=ndim) elif ndim == 1: # only vary alpha lhcube = np.zeros((samples, len(hod_range[0]))) for i in range(4): lhcube[:, i] = 0.5 * (hod_range[0] + hod_range[1])[i] lhc = lhd.thetaLHD([[hod_range[0][4]], [hod_range[1][4]]], samples=samples, method=method, ndim=ndim) lhcube[:, 4] = lhc.flatten() f = open(fname, 'w') f.write('# ' + prior + '\n') f.write('# parameters : ' + ', '.join(hodprior.labels) + '\n') f.write('# theta min : ' + ', '.join([str(r) for r in hod_range[0]]) + '\n') f.write('# theta max : ' + ', '.join([str(r) for r in hod_range[1]]) + '\n') for i in range(lhcube.shape[0]): f.write('%f' % (lhcube[i, 0])) for j in range(1, lhcube.shape[1]): f.write('\t %f' % (lhcube[i, j])) f.write('\n') f.close() return lhcube
def X_HODLHD( seed_hod, obvs='plk', ell=0, Nmesh=360, rsd=True, karr=False, # kwargs specifying P(k) prior='sinha2017prior_narrow', samples=40, method='mdu', ndim=None, # kwargs for LHD silent=True): ''' Read observable (e.g. P(k)) for a LHD of HOD parameters (specified by prior, samples, and method) ''' # directory of the catalogs if ndim is None: str_ndim = '' else: str_ndim = '_' + str(ndim) + 'D' f_dir = ''.join([ UT.dat_dir(), 'lhd/onlyHOD/', method, '_', str(samples), '_', prior, str_ndim, '/' ]) # rsd flag if rsd: rsd_str = 'z' else: rsd_str = 'r' # read in observable pks = [] for i in range(samples): # 21st LHD HOD theta requires ton of memory # this should be addressed more robustly but for # the sake of my sanity at the moment, I will skip # it for now. if i == 21: continue # directory of theta_i,LHD fname = ''.join([ f_dir, 'HOD', method, '_seed', str(seed_hod), '_', str(i), '/', 'pk.menut0.0.nreal1.nzbin4.', rsd_str, 'space.', str(Nmesh), '.nbkt.dat' ]) if not silent: print('%i -- reading %s' % (i, fname)) # read in plks k, pk = np.loadtxt(fname, skiprows=4, unpack=True, usecols=[0, 1 + ell / 2]) pks.append(pk) if karr: return k, np.array(pks) else: return np.array(pks)
def NeutParticles(mneut, nreal, nzbin, clobber=False): ''' Read particle catalog generated by Paco and return NBlab.ArrayCatalog parameters ---------- mneut : float, total neutrino mass nreal : int, realization number nzbin : int, integer specifying the redshift of the snapshot. nzbin = 0 --> z=3 nzbin = 1 --> z=2 nzbin = 2 --> z=1 nzbin = 3 --> z=0.5 nzbin = 4 --> z=0 clobber : bool, optional if True, reconstructs the BigFile data ''' dir = ''.join([ UT.dat_dir(), str(mneut), 'eV/', str(nreal), '/snapdir_', str(nzbin).zfill(3), '/' ]) dir_list = [ dir + '/Matter' + sub for sub in ['', '/Position', '/Velocity', '/ID'] ] if (not np.all([os.path.isdir(dd) for dd in dir_list])) or clobber: f = ''.join([dir, 'snap_', str(nzbin).zfill(3)]) # read in Gadget header header = RS.read_gadget_header(f) # read in CDM particles (parttype = 1) and create catalogue particle_data = {} particle_data['Position'] = RS.read_block(f, 'POS ', parttype=1) / 1000. # Mpc/h particle_data['Velocity'] = RS.read_block(f, 'VEL ', parttype=1) particle_data['ID'] = RS.read_block(f, 'ID ', parttype=1) cat = NBlab.ArrayCatalog(particle_data, BoxSize=np.array([ header['boxsize'], header['boxsize'], header['boxsize'] ])) #cat['KDDensity'] = KDDensity(cat).density cat.save(dir + '/Matter', ["Position", "Velocity", "ID"]) else: cat = NBlab.BigFileCatalog(dir + '/Matter', header='Header') return cat
def X_testHODLHD( seed_hod, obvs='plk', ell=0, Nmesh=360, rsd=True, karr=False, # kwargs specifying P(k) prior='sinha2017prior_narrow', samples=20, ndim=None, # kwargs specifying the LHD silent=True): ''' Read observable (e.g. P(k)) for a LHD of HOD parameters (specified by prior, samples, and method) ''' # directory of the catalogs if ndim is None: str_ndim = '' else: str_ndim = '_' + str(ndim) + 'D' f_dir = ''.join([ UT.dat_dir(), 'lhd/onlyHOD/test_', str(samples), '_', prior, str_ndim, '/' ]) # rsd flag if rsd: rsd_str = 'z' else: rsd_str = 'r' # read in observable pks = [] for i in range(samples): # directory of theta_i,LHD fname = ''.join([ f_dir, 'testHOD_seed', str(seed_hod), '_', str(i), '/', 'pk.menut0.0.nreal1.nzbin4.', rsd_str, 'space.', str(Nmesh), '.nbkt.dat' ]) if not silent: print('%i -- reading %s' % (i, fname)) # read in plks k, pk = np.loadtxt(fname, skiprows=4, unpack=True, usecols=[0, 1 + ell / 2]) pks.append(pk) if karr: return k, np.array(pks) else: return np.array(pks)
def HODLHDpks(seed_hod, i_lhd, prior='sinha2017prior_narrow', method='mdu', samples=17, ndim=None, Nmesh=360, rsd=True): ''' Measure and write P(k|theta_i,LHD) for the HOD LHD that spans the `prior` ''' # RSD flag if rsd: str_rsd = '.zspace' else: str_rsd = '.rspace' # directory if ndim is None: str_ndim = '' else: str_ndim = '_' + str(ndim) + 'D' parent_dir = ''.join([ UT.dat_dir(), 'lhd/onlyHOD/', method, '_', str(samples), '_', prior, str_ndim, '/HOD', method, '_seed', str(seed_hod), '_', str(i_lhd), '/' ]) fname = ''.join([ parent_dir, 'pk.menut0.0.nreal1.nzbin4', str_rsd, '.', str(Nmesh), '.nbkt.dat' ]) if not os.path.exists(parent_dir): # make directory raise ValueError("the directory \n %s \n does not exist!" % parent_dir) # read from galaxy catalog gals = NBlab.BigFileCatalog(parent_dir, header='Header') plk = FM.Observables(gals, observable='plk', rsd=rsd, Nmesh=Nmesh) # save to file f = open(fname, 'w') f.write("### header ### \n") f.write("# shotnoise %f \n" % plk['shotnoise']) f.write("# columns : k , P0, P2, P4 \n") f.write('### header ### \n') for ik in range(len(plk['k'])): f.write("%f \t %f \t %f \t %f" % (plk['k'][ik], plk['p0k'][ik], plk['p2k'][ik], plk['p4k'][ik])) f.write("\n") f.close() return None
def _NeutHalos(mneut, nreal, nzbin, clobber=False): ''' Construct Friend-of-Friend halo catalog from Paco's neutrino particle catalogs. ''' dir = ''.join([ UT.dat_dir(), str(mneut), 'eV/', str(nreal), '/snapdir_', str(nzbin).zfill(3), '/' ]) f = ''.join([dir, 'snap_', str(nzbin).zfill(3)]) # read in Gadget header header = RS.read_gadget_header(f) # get cosmology from header Omega_b = 0.049 # fixed baryon cosmo = NBlab.cosmology.Planck15.clone(Omega_cdm=header['Omega_m'] - Omega_b, h=header['h'], Omega_b=Omega_b) if (not os.path.isdir(dir + '/FOF')) or clobber: # DM particle mass (parttype=1) m_part = header['masses'][1] cat = NeutParticles(mneut, nreal, nzbin) # read in neutrino particles with cat.attrs['Nmesh'] = [512, 512, 512] # calculate friend-of-friend with only CDM particles fof = NBlab.FOF(cat, linking_length=0.2, nmin=20) # now make them into halos fofcat = fof.to_halos(particle_mass=m_part, cosmo=cosmo, redshift=header['z']) fofcat.save( dir + '/FOF', ['Position', 'Velocity', 'VelocityOffset', 'Mass', 'Radius']) else: fofcat = NBlab.BigFileCatalog(dir + '/FOF', header='Header') return fofcat
def testHOD_LHD(prior=None, samples=None, overwrite=False, ndim=None, seed=1): ''' Return parameters of the vanilla Zheng et al.(2007) HOD model for testing the LHD. These values will be significantly more within the boundaries of the parameter space ''' if ndim is None: str_ndim = '' else: str_ndim = str(ndim) + 'D.' fname = ''.join([ UT.dat_dir(), 'lhd/onlyHOD/', 'HOD_LHDtest.', str_ndim, prior, '.rseed', str(seed), '.', str(samples), 'samples.dat' ]) if os.path.isfile(fname) and not overwrite: # file exists theta_test = np.loadtxt(fname, skiprows=4) else: # prior and its range hodprior = HODprior(prior) hod_range = hodprior.range() np.random.seed(seed) theta_test = np.zeros((samples, len(hod_range[0]))) if ndim is None: for i in range(theta_test.shape[1]): dHOD_range = (hod_range[1] - hod_range[0])[i] # linearly spaced in the inner 50% of the parameter space ti = np.linspace(hod_range[0][i] + 0.25 * dHOD_range, hod_range[1][i] - 0.25 * dHOD_range, theta_test.shape[0]) np.random.shuffle(ti) theta_test[:, i] = ti elif ndim == 1: # only alpha varies dHOD_range = (hod_range[1] - hod_range[0])[4] # linearly spaced in the inner 50% of the parameter space ti = np.linspace(hod_range[0][4] + 0.25 * dHOD_range, hod_range[1][4] - 0.25 * dHOD_range, theta_test.shape[0]) np.random.shuffle(ti) theta_test[:, 4] = ti theta_test[:, 0] = 0.5 * (hod_range[0] + hod_range[1])[0] theta_test[:, 1] = 0.5 * (hod_range[0] + hod_range[1])[1] theta_test[:, 2] = 0.5 * (hod_range[0] + hod_range[1])[2] theta_test[:, 3] = 0.5 * (hod_range[0] + hod_range[1])[3] f = open(fname, 'w') f.write('# ' + prior + '\n') f.write('# parameters : ' + ', '.join(hodprior.labels) + '\n') f.write('# theta min : ' + ', '.join([str(r) for r in hod_range[0]]) + '\n') f.write('# theta max : ' + ', '.join([str(r) for r in hod_range[1]]) + '\n') for i in range(theta_test.shape[0]): f.write('%f' % (theta_test[i, 0])) for j in range(1, theta_test.shape[1]): f.write('\t %f' % (theta_test[i, j])) f.write('\n') f.close() return theta_test
def HODLHDcatalogs(seed_hod, i_lhd, prior='sinha2017prior_narrow', method='mdu', samples=40, ndim=None): ''' Generate HOD catalog of the i_lhd th parameter of the HOD LHD that spans the Sinha M., et al. (2017) HOD parameter priors. parameters ---------- seed_hod : int, random seed for the HOD prior : str, optional string specifying the HOD range. Default is 'sinha2017prior', which uses the prior from Sinha et al. (2017) method : str string specifying the method of LHD. Default is nohl samples : int, optional sample size of the LHD. Default is 17, which is from the fixed sample size of NOHL method ''' if i_lhd >= samples: raise ValueError("i_lhd has to be less than samples") # read in Neutrino halo with m_neut = 0.0 eV, realization #1, at z_bin = 4 # this is hardcoded; don't change this halos = Dat.NeutHalos(0.0, 1, 4) # read in the HOD LHD lhcube = onlyHOD.HOD_LHD(prior=prior, samples=samples, method=method, ndim=ndim) print('%i of %i LHD' % (i_lhd + 1, samples)) p_hod = {} hodkeys = ['logMmin', 'sigma_logM', 'logM0', 'logM1', 'alpha'] for ik, k in enumerate(hodkeys): p_hod[k] = lhcube[i_lhd, ik] print(p_hod) # populate the halo catalogs using HOD gals = FM.Galaxies(halos, p_hod, seed=seed_hod) # RSD position (hardcoded in the z direction) gals['RSDPosition'] = FM.RSD(gals, LOS=[0, 0, 1]) # directory of the catalogs if ndim is None: str_ndim = '' else: str_ndim = '_' + str(ndim) + 'D' parent_dir = ''.join([ UT.dat_dir(), 'lhd/onlyHOD/', method, '_', str(samples), '_', prior, str_ndim, '/HOD', method, '_seed', str(seed_hod), '_', str(i_lhd), '/' ]) print('writing to %s ---------' % parent_dir) if not os.path.exists(parent_dir): # make directory os.mkdir(parent_dir) # save to file gals.save(parent_dir, ('Position', 'Velocity', 'RSDPosition')) return None
def Fiducial_Obvs(obvs, nreal, nzbin, seed_hod, mneut=0.0, Nmesh=360, rsd=True, HODrange='sinha2017prior_narrow', silent=False): ''' Calculate and save observables of the fiducial HOD catalogs ''' if mneut != 0.0: raise ValueError("Fiducial should be calculated at m_nu=0.0eV") if rsd: str_rsd = '.zspace' else: str_rsd = '.rspace' folder = ''.join([ UT.dat_dir(), 'lhd/', str(mneut), 'eV_', str(nreal), '_z', str(nzbin), '_fiducial/', 'HOD_seed', str(seed_hod), '/' ]) if obvs == 'plk': fname = ''.join([ folder, 'pk.menut', str(mneut), '.nreal', str(nreal), '.nzbin', str(nzbin), str_rsd, '.', str(Nmesh), '.nbkt.dat' ]) if os.path.isfile(fname): if not silent: print('--- reading from --- \n %s' % fname) # read observalbe from file k, p0k, p2k, p4k = np.loadtxt(fname, skiprows=4, unpack=True, usecols=[0, 1, 2, 3]) obvs = {'k': k, 'p0k': p0k, 'p2k': p2k, 'p4k': p4k} # readin shot-noise from header f = open(fname, 'r') _ = f.readline() str_sn = f.readline() obvs['shotnoise'] = float(str_sn.strip().split('shotnoise')[-1]) else: gals = Fiducial_Catalog(nreal, nzbin, seed_hod, mneut=mneut, HODrange=HODrange) if obvs == 'plk': # power spectrum multipole plk = FM.Observables(gals, observable='plk', rsd=rsd, Nmesh=Nmesh) # save to file f = open(fname, 'w') f.write("### header ### \n") f.write("# shotnoise %f \n" % plk['shotnoise']) f.write("# columns : k , P0, P2, P4 \n") f.write('### header ### \n') for ik in range(len(plk['k'])): f.write("%f \t %f \t %f \t %f" % (plk['k'][ik], plk['p0k'][ik], plk['p2k'][ik], plk['p4k'][ik])) f.write("\n") f.close() obvs = plk else: raise NotImplementedError('only Plk implemented') return obvs
def Fiducial_Catalog(nreal, nzbin, seed_hod, mneut=0.0, HODrange='sinha2017prior_narrow'): ''' Generate fiducial HOD catalogs from specified m_nu = 0.0eV halo catalog parameters ---------- mneut : float, total neutrino mass nreal : int, realization number nzbin : int, integer specifying the redshift of the snapshot. nzbin = 0 --> z=3 nzbin = 1 --> z=2 nzbin = 2 --> z=1 nzbin = 3 --> z=0.5 nzbin = 4 --> z=0 seed_hod : int, random seed for the HOD ''' if mneut != 0.0: raise ValueError("Fiducial should be calculated at m_nu=0.0eV") folder = ''.join([ UT.dat_dir(), 'lhd/', str(mneut), 'eV_', str(nreal), '_z', str(nzbin), '_fiducial/', 'HOD_seed', str(seed_hod), '/' ]) # read in Neutrino halo with mneut eV, realization # nreal, at z specified by nzbin halos = NeutHalos(mneut, nreal, nzbin) if not np.all([ os.path.exists(folder + subfold + '/') for subfold in ['Position', 'Velocity', 'RSDPosition'] ]): # fiducial HOD parameter values if HODrange in ['sinha2017prior', 'sinha2017prior_narrow']: keylist = ['logMmin', 'sigma_logM', 'logM0', 'logM1', 'alpha'] p_hod = { 'logMmin': 11.60, 'sigma_logM': 0.26, 'logM0': 11.49, 'logM1': 12.83, 'alpha': 1.02 } # populate the halo catalogs using HOD gals = FM.Galaxies(halos, p_hod, seed=seed_hod) # RSD position (hardcoded in the z direction) gals['RSDPosition'] = FM.RSD(gals, LOS=[0, 0, 1]) parent_dir = '/'.join(folder[:-1].split('/')[:-1]) + '/' if not os.path.exists(parent_dir): # make directory os.mkdir(parent_dir) # save to file gals.save(folder, ('Position', 'Velocity', 'RSDPosition')) else: # read from file gals = NBlab.BigFileCatalog(folder, header='Header') gals.cosmo = halos.cosmo # save cosmology return gals