def main(model_gen_func, fiducial, output_fname):
global model
model = model_gen_func()
global fid
fid = np.array(fiducial)
params = fid * np.ones((7 * args.Nparam, 7))
dp_range = np.array((0.11, 0.05, 0.225, 0.9, 0.12))
for i in range(5):
params[args.Nparam * i:args.Nparam * (i + 1),
i] += (2. * np.random.random(args.Nparam) - 1) * min(
dp_range[i], fid[i])
params[args.Nparam * 5:args.Nparam * 6,
5] = 2. * np.random.random(args.Nparam) - 1
params[args.Nparam * 6:args.Nparam * 7,
6] = 2. * np.random.random(args.Nparam) - 1
output_dict = collections.defaultdict(list)
nproc = args.nproc
global halocat
with Pool(nproc) as pool:
if args.simname == 'consuelo20' and args.version == 'all':
for box in consuelo20_box_list:
halocat = CachedHaloCatalog(simname = args.simname, version_name = box,redshift = args.redshift, \
halo_finder = args.halofinder)
model.populate_mock(halocat)
for i, output_data in enumerate(
pool.map(calc_all_observables, params)):
if i % nproc == nproc - 1:
print i
print str(datetime.now())
for name, data in zip(output_names, output_data):
output_dict[name].append(data)
print box
else:
halocat = CachedHaloCatalog(simname = args.simname, version_name = args.version,redshift = args.redshift, \
halo_finder = args.halofinder)
model.populate_mock(halocat)
for i, output_data in enumerate(
pool.map(calc_all_observables, params)):
if i % nproc == nproc - 1:
print i
print str(datetime.now())
for name, data in zip(output_names, output_data):
output_dict[name].append(data)
for name in output_names:
output_dict[name] = np.array(output_dict[name])
np.savez(output_fname, **output_dict)
def main(model_gen_func, params_fname, params_usecols, output_fname):
global model
model = model_gen_func()
median_w = np.median(np.loadtxt(params_fname, usecols=params_usecols),
axis=0)
params = median_w * np.ones((500 + 7 * 1000, 7)) ##take medians
dp_range = np.array((0.5, 0.5, 0.25, 0.5, 0.5, 1, 1))
for i in params_usecols:
params[1000 * i + 500:1000 * i + 1500,
i] += (2. * np.random.random(1000) - 1) * dp_range[i]
output_dict = collections.defaultdict(list)
nproc = args.nproc
global halocat
global c
with Pool(nproc) as pool:
if args.simname == 'consuelo20' and args.version == 'all':
for box in consuelo20_box_list:
halocat = CachedHaloCatalog(simname = args.simname, version_name = box,redshift = args.redshift, \
halo_finder = args.halofinder)
model.populate_mock(halocat)
c = Ngal_estimate(halocat, 150000)
for i, output_data in enumerate(
pool.map(calc_all_observables, params)):
if i % nproc == nproc - 1:
print i
print str(datetime.now())
for name, data in zip(output_names, output_data):
output_dict[name].append(data)
print box
else:
halocat = CachedHaloCatalog(simname = args.simname, version_name = args.version,redshift = args.redshift, \
halo_finder = args.halofinder)
model.populate_mock(halocat)
c = Ngal_estimate(halocat, 150000)
for i, output_data in enumerate(
pool.map(calc_all_observables, params)):
if i % nproc == nproc - 1:
print i
print str(datetime.now())
for name, data in zip(output_names, output_data):
output_dict[name].append(data)
for name in output_names:
output_dict[name] = np.array(output_dict[name])
np.savez(output_fname, **output_dict)
def __init__(self, config):
super().__init__(config)
self.cur_wp = np.zeros(11)
if self['sim'] == "bolplanck":
halocat = CachedHaloCatalog(simname='bolplanck')
elif self['sim'] == "old":
halocat = CachedHaloCatalog(
fname='/home/lom31/Halo/hlist_1.00231.list.halotools_v0p1.hdf5',
update_cached_fname=True)
halocat.redshift = 0.
elif self['sim'] == "smdpl":
halocat = CachedHaloCatalog(
fname=
'/home/lom31/.astropy/cache/halotools/halo_catalogs/SMDPL/rockstar/2019-07-03-18-38-02-9731.dat.my_cosmosim_halos.hdf5',
update_cached_fname=True)
#halocat = CachedHaloCatalog(fname='/home/lom31/.astropy/cache/halotools/halo_catalogs/smdpl/rockstar/2019-07-03-18-38-02-9731.dat.my_cosmosim_halos.hdf5',update_cached_fname = True)
halocat.redshift = 0.
elif self['sim'] == "mdr1":
halocat = CachedHaloCatalog(
fname=
'/home/lom31/.astropy/cache/halotools/halo_catalogs/multidark/rockstar/hlist_0.68215.list.halotools_v0p4.hdf5',
update_cached_fname=True)
if self['param'] == 'mvir':
cens_occ_model = Zheng07Cens(threshold=-19)
cens_prof_model = TrivialPhaseSpace()
sats_occ_model = Zheng07Sats(modulate_with_cenocc=True,
threshold=-19)
sats_prof_model = NFWPhaseSpace()
elif self['param'] == 'vmax':
cens_occ_model = Zheng07Cens(prim_haloprop_key='halo_vmax',
threshold=-19)
cens_prof_model = TrivialPhaseSpace()
sats_occ_model = Zheng07Sats(prim_haloprop_key='halo_vmax',
threshold=-19,
modulate_with_cenocc=True)
sats_prof_model = NFWPhaseSpace()
global model_instance
model_instance = HodModelFactory(centrals_occupation=cens_occ_model,
centrals_profile=cens_prof_model,
satellites_occupation=sats_occ_model,
satellites_profile=sats_prof_model)
try:
model_instance.mock.populate()
except:
model_instance.populate_mock(halocat)
def main(model_gen_func, params_fname, params_usecols, output_fname):
global model
model = model_gen_func()
nparams = args.Nreal * 77
median_w = np.median(np.loadtxt(params_fname, usecols=params_usecols),
axis=0)
params = median_w * np.ones((nparams, 7)) ##take medians
for i in params_usecols:
for j in range(11):
params[11 * args.Nreal * i + args.Nreal * j:11 * args.Nreal * i +
args.Nreal * j + args.Nreal,
i] += args.stepsize[i] * (j - 5)
output_dict = collections.defaultdict(list)
nproc = args.nproc
global halocat
with Pool(nproc) as pool:
if args.simname == 'consuelo20' and args.version == 'all':
for box in consuelo20_box_list:
halocat = CachedHaloCatalog(simname = args.simname, version_name = box,redshift = args.redshift, \
halo_finder = args.halofinder)
model.populate_mock(halocat)
for i, output_data in enumerate(
pool.map(calc_all_observables, params)):
if i % nproc == nproc - 1:
print i
print str(datetime.now())
for name, data in zip(output_names, output_data):
output_dict[name].append(data)
print box
else:
halocat = CachedHaloCatalog(simname = args.simname, version_name = args.version,redshift = args.redshift, \
halo_finder = args.halofinder)
model.populate_mock(halocat)
for i, output_data in enumerate(
pool.map(calc_all_observables, params)):
if i % nproc == nproc - 1:
print i
print str(datetime.now())
for name, data in zip(output_names, output_data):
output_dict[name].append(data)
for name in output_names:
output_dict[name] = np.array(output_dict[name])
np.savez(output_fname, **output_dict)
def build_observations(Mr=21, b_normal=0.25, make=['data', 'covariance']):
''' Wrapper to build all the required fake observations and their
corresponding covariance matrices.
'''
# download the Multidark halo catalog if necessary
try:
halocat = CachedHaloCatalog(simname='multidark',
halo_finder='rockstar',
redshift=0.0)
except InvalidCacheLogEntry:
from halotools.sim_manager import DownloadManager
dman = DownloadManager()
dman.download_processed_halo_table('multidark', 'rockstar', 0.0)
if 'data' in make:
# xi, nbar, gmf
build_xi_bins(Mr=Mr)
print 'Building randoms and RRs for the subvolumes'
build_randoms_RR(Nr=5e5, box='md_sub')
print 'Building nbar, xi(r), GMF data vector... '
build_nbar_xi_gmf(Mr=Mr, b_normal=b_normal)
if 'covariance' in make:
print 'Computing covariance matrix of data...'
print 'building the sample covariance'
build_MCMC_cov_nbar_xi_gmf(Mr=Mr, b_normal=b_normal)
print 'building the poisson covariance'
build_ABC_cov_nbar_xi_gmf(Mr=Mr, b_normal=b_normal)
return None
def halocat_init(halo_catalog, z_median):
"""
Initial population of halo catalog using populate_mock function
Parameters
----------
halo_catalog: string
Path to halo catalog
z_median: float
Median redshift of survey
Returns
---------
model: halotools model instance
Model based on behroozi 2010 SMHM
"""
halocat = CachedHaloCatalog(fname=halo_catalog, update_cached_fname=True)
"""
prim_haloprop_key : String giving the column name of the primary halo
property governing stellar mass.
"""
model = PrebuiltSubhaloModelFactory('behroozi10', redshift=z_median, \
prim_haloprop_key='halo_macc')
model.populate_mock(halocat, seed=5)
return model
def main():
simname = 'smdpl_400'
halocat = CachedHaloCatalog(simname=simname,
halo_finder='Rockstar',
redshift=0.0, dz_tol=0.001,
version_name='custom')
from intrinsic_alignments.utils.value_added_halocat import halocat_to_galaxy_table
from intrinsic_alignments.ia_models.ia_model_components import CentralAlignment, SatelliteAlignment
from intrinsic_alignments.ia_models.occupation_models import SubHaloPositions
rbins = np.logspace(-1,1.8,29)
rbin_centers = (rbins[:-1]+rbins[1:])/2.0
mus = np.linspace(0.99,0.0,10)
N = 1000
for i in range(100, N):
for j in range(1, 10):
print(i, j)
ed, ee = calculate_ed_ee(halocat, 0.99, mus[j], rbins=rbins)
# save measurements
fpath = fpath = PROJECT_DIRECTORY + 'halo_shape_correlations/data/'
fname = simname + '_' + '{0:.2f}'.format(0.99) + '_' + '{0:.2f}'.format(mus[j]) + '_ed_ee_'+str(i).zfill(4)+'.dat'
ascii.write([rbin_centers, ed, ee],
fpath+fname,
names=['r','ed','ee'],
overwrite=True)
def main(model_gen_func, fiducial, output_fname):
global model
model = model_gen_func()
params = np.array(fiducial)*np.ones((args.Nparam,7))
output_dict = collections.defaultdict(list)
nproc = args.nproc
global halocat
with Pool(nproc) as pool:
if 1:
halocat = CachedHaloCatalog(simname = args.simname, version_name = args.version,redshift = args.redshift, \
halo_finder = args.halofinder)
model.populate_mock(halocat)
for i, output_data in enumerate(pool.map(calc_all_observables, params)):
if 1:
print i
print str(datetime.now())
for name, data in zip(output_names, output_data):
output_dict[name].append(data)
for name in output_names:
output_dict[name] = np.array(output_dict[name])
np.savez(output_fname, **output_dict)
def main(model_gen_func, params_fname, params_usecols, output_fname):
global model
model = model_gen_func()
nparams = args.Nparams
params = np.loadtxt(params_fname, usecols=params_usecols)
params = params[np.random.choice(len(params), nparams)]
output_dict = collections.defaultdict(list)
nproc = 55
global halocat
with Pool(nproc) as pool:
if 1:
for box in consuelo20_box_list:
halocat = CachedHaloCatalog(simname = 'consuelo20', version_name = box,redshift = 0, \
halo_finder = 'rockstar')
model.populate_mock(halocat)
for i, output_data in enumerate(
pool.map(calc_all_observables, params)):
if i % 55 == 54:
print i
print str(datetime.now())
for name, data in zip(output_names, output_data):
output_dict[name].append(data)
print box
else:
halocat = CachedHaloCatalog(simname = args.simname, version_name = args.version,redshift = args.redshift, \
halo_finder = args.halofinder)
model.populate_mock(halocat)
for i, output_data in enumerate(
pool.map(calc_all_observables, params)):
if i % 55 == 54:
print i
print str(datetime.now())
for name, data in zip(output_names, output_data):
output_dict[name].append(data)
for name in output_names:
output_dict[name] = np.array(output_dict[name])
np.savez(output_fname, **output_dict)
def __init__(self, Mr=21, b_normal=0.25):
''' Class object that describes our forward model used in MCMC inference.
Our model forward models the galaxy catalog using HOD parameters using HaloTools.
'''
self.Mr = Mr
self.b_normal = b_normal
thr = -1. * np.float(Mr)
self.model = PrebuiltHodModelFactory('zheng07', threshold=thr)
self.halocat = CachedHaloCatalog(simname='multidark',
redshift=0,
halo_finder='rockstar')
def main(model_gen_func, fiducial, output_fname):
global model
model = model_gen_func()
global fid
fid = np.array(fiducial)
params = fid * np.ones((7 * args.Nparam, 7))
dp_range = np.array((0.025, 0.05, 0.02, 0.1, 0.02, 0.05, 0.05))
for i in range(7):
params[args.Nparam * i:args.Nparam * i + args.Nparam / 2,
i] -= dp_range[i]
params[args.Nparam * i + args.Nparam / 2:args.Nparam * i + args.Nparam,
i] += dp_range[i]
output_dict = collections.defaultdict(list)
nproc = args.nproc
global halocat
global ptclpos
global num_ptcls_to_use
with Pool(nproc) as pool:
if 1:
halocat = CachedHaloCatalog(simname = args.simname, version_name = args.version,redshift = args.redshift, \
halo_finder = args.halofinder)
model.populate_mock(halocat)
if args.ptclpos:
ptclpos = np.loadtxt(args.ptclpos)
else:
mask = np.random.rand(len(halocat.ptcl_table)) < args.ptclrate
ptclpos = return_xyz_formatted_array(*(table[ax]
for ax in 'xyz'),
period=Lbox)
num_ptcls_to_use = len(ptclpos)
for i, output_data in enumerate(
pool.map(calc_all_observables, params)):
if i % nproc == nproc - 1:
print i
print str(datetime.now())
for name, data in zip(output_names, output_data):
output_dict[name].append(data)
for name in output_names:
output_dict[name] = np.array(output_dict[name])
np.savez(output_fname, **output_dict)
def __init__(self, Mr=-21, b_normal=0.25):
''' Class object that describes our forward model used in AMC-PMC inference.
Our model forward models the galaxy catalog using HOD parameters using HaloTools.
'''
self.Mr = Mr
self.b_normal = b_normal
thr = -1. * np.float(Mr)
self.model = PrebuiltHodModelFactory('zheng07', threshold=thr)
#self.model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}
self.halocat = CachedHaloCatalog(simname='multidark',
redshift=0,
halo_finder='rockstar')
self.RR = data_RR(box='md_sub')
self.randoms = data_random(box='md_sub')
self.NR = len(self.randoms)
def access_halocat(halocat_file_path):
"""
Accesses already stored halo catalog
Parameters
----------
halocat_file_path: string
Path to halo catalog
Returns
---------
halocat: hdf5 file
Mock halo catalog
"""
halocat = CachedHaloCatalog(fname=halocat_file_path,\
update_cached_fname=True)
broadcast_host_halo_property(halocat.halo_table,'halo_macc')
return halocat
def load_halos(tag, cols, subsample=None):
if 'abacus' in tag:
from AbacusCosmos import Halos
halo_dir = '/mount/sirocco1/ksf293/halo_files_z0'
print("Loading halos from {}".format(halo_dir))
cat = Halos.make_catalog_from_dir(dirname=halo_dir,
halo_type='Rockstar',
load_subsamples=False,
load_pids=False)
halos = cat.halos
if subsample:
halos = halos[:subsample]
arrs = []
for col in cols:
poss = {'x': 0, 'y': 1, 'z': 2}
if col in poss:
arrs.append(halos['pos'][:, poss[col]])
else:
arrs.append(halos[col])
print("Loaded {} halos".format(len(halos)))
elif 'ds14b' in tag:
from halotools.sim_manager import CachedHaloCatalog
simname = 'ds14b'
#version_name = 'rockstar1{}'.format(tag)
version_name = 'rockstar1'
print("Loading halos from {}".format(simname))
halos = CachedHaloCatalog(simname=simname,
halo_finder='rockstar',
version_name=version_name,
redshift=0.0)
arrs = []
for col in cols:
arrs.append(halos.halo_table['halo_{}'.format(col)])
print("Loaded {} halos".format(len(halos.halo_table['halo_id'])))
else:
raise ValueError("Tag '{}' not recognized".format(tag))
return halos, arrs
def load_sim_data(cfg):
"""Load the UniverseMachine data."""
#read in halocat
halocat = CachedHaloCatalog(
simname=cfg['sim_name'],
halo_finder=cfg['sim_halo_finder'],
version_name=cfg['sim_version_name'],
redshift=cfg['sim_z'],
ptcl_version_name=cfg['sim_ptcl_version_name']) # doctest: +SKIP
#read in particle table
ptcl_table = Table.read(os.path.join(cfg['data_location'], cfg['sim_dir'],
cfg['sim_particle_file']),
path='data')
px = ptcl_table['x']
py = ptcl_table['y']
pz = ptcl_table['z']
particles = np.vstack((px, py, pz)).T
ptcl_table = 0
#downsample
num_ptcls_to_use = int(1e4)
particles = randomly_downsample_data(particles, num_ptcls_to_use)
particle_masses = np.zeros(num_ptcls_to_use) + halocat.particle_mass
downsampling_factor = (cfg['sim_particles_per_dimension']**3) / float(
len(particles))
# other parameters
cfg['sim_cosmo'] = FlatLambdaCDM(H0=cfg['sim_h0'] * 100.0,
Om0=cfg['sim_omega_m'])
cfg['sim_volume'] = np.power(cfg['sim_lbox'] / cfg['sim_h0'], 3)
# if verbose:
# print("# The volume of the UniverseMachine mock is %15.2f Mpc^3" %
# cfg['um_volume'])
return {
'halocat': halocat,
'particles': particles,
'particle_masses': particle_masses,
'downsampling_factor': downsampling_factor
}, cfg
def load_baseline_halocat(simname='bolplanck', redshift=0, fixed_seed=411):
halocat = CachedHaloCatalog(simname=simname, redshift=redshift)
halocat.halo_table['halo_zpeak'] = 1./halocat.halo_table['halo_scale_factor_mpeak'] - 1.
rvir_peak_physical_unity_h = halo_mass_to_halo_radius(halocat.halo_table['halo_mpeak'],
halocat.cosmology, halocat.halo_table['halo_zpeak'], 'vir')
rvir_peak_physical = rvir_peak_physical_unity_h/halocat.cosmology.h
halocat.halo_table['halo_rvir_zpeak'] = rvir_peak_physical*1000.
nhalos = len(halocat.halo_table)
with NumpyRNGContext(fixed_seed):
halocat.halo_table['halo_uran'] = np.random.rand(nhalos)
mask = halocat.halo_table['halo_mvir_host_halo'] == 0
halocat.halo_table['halo_mvir_host_halo'][mask] = halocat.halo_table['halo_mpeak'][mask]
vmax_percentile_fname = '/Users/aphearin/work/UniverseMachine/temp_galsize_models/vmax_percentile.npy'
halocat.halo_table['halo_vmax_at_mpeak_percentile'] = np.load(vmax_percentile_fname)
return halocat
def halocat_init(halo_cat, z):
"""
Initial population of halo catalog using populate_mock function
Parameters
----------
halo_cat: string
Path to halo catalog
z: float
Median redshift of survey
Returns
---------
model: halotools model instance
Model based on behroozi 2010 SMHM
"""
halocat = CachedHaloCatalog(fname=halo_cat, update_cached_fname=True)
model = PrebuiltSubhaloModelFactory('behroozi10', redshift=z, \
prim_haloprop_key='halo_macc')
model.populate_mock(halocat, seed=5)
return model
def main():
# get simulation information
if len(sys.argv)>1:
sim_name = sys.argv[1]
snapnum = int(sys.argv[2])
shape_type = sys.argv[3]
sample_name = sys.argv[4]
else:
sim_name = 'TNG300-1' # full physics high-res run
snapnum = 99 # z=0
shape_type = 'reduced' # non-reduced, reduced, iterative
sample_name = 'sample_3'
# load a test halo catalog
from halotools.sim_manager import CachedHaloCatalog
halocat = CachedHaloCatalog(simname='bolplanck', halo_finder='rockstar',
redshift=0.0, dz_tol=0.1, version_name='halotools_v0p4')
from halotools.empirical_models import HodModelFactory
# define the central occupatoion model
from halotools.empirical_models import TrivialPhaseSpace, Zheng07Cens
cens_occ_model = Zheng07Cens()
cens_prof_model = TrivialPhaseSpace()
# define the satellite occupation model
from halotools.empirical_models import Zheng07Sats
from halotools.empirical_models import NFWPhaseSpace, SubhaloPhaseSpace
from intrinsic_alignments.ia_models.anisotropic_nfw_phase_space import AnisotropicNFWPhaseSpace
sats_occ_model = Zheng07Sats()
#sats_prof_model = AnisotropicNFWPhaseSpace()
sats_prof_model = SubhaloPhaseSpace('satellites', np.logspace(10.5, 15.2, 15))
# define the alignment models
from intrinsic_alignments.ia_models.ia_model_components import CentralAlignment,\
RadialSatelliteAlignment, MajorAxisSatelliteAlignment, HybridSatelliteAlignment
central_orientation_model = CentralAlignment()
satellite_orientation_model = RadialSatelliteAlignment()
if sample_name == 'sample_1':
cens_occ_model.param_dict['logMmin'] = 12.54
cens_occ_model.param_dict['sigma_logM'] = 0.26
sats_occ_model.param_dict['alpha'] = 1.0
sats_occ_model.param_dict['logM0'] = 12.68
sats_occ_model.param_dict['logM1'] = 13.48
central_orientation_model.param_dict['central_alignment_strength'] = 0.755
satellite_orientation_model.param_dict['satellite_alignment_strength'] = 0.279
elif sample_name == 'sample_2':
cens_occ_model.param_dict['logMmin'] = 11.93
cens_occ_model.param_dict['sigma_logM'] = 0.26
sats_occ_model.param_dict['alpha'] = 1.0
sats_occ_model.param_dict['logM0'] = 12.05
sats_occ_model.param_dict['logM1'] = 12.85
central_orientation_model.param_dict['central_alignment_strength'] = 0.64
satellite_orientation_model.param_dict['satellite_alignment_strength'] = 0.084
elif sample_name =='sample_3':
cens_occ_model.param_dict['logMmin'] = 11.61
cens_occ_model.param_dict['sigma_logM'] = 0.26
sats_occ_model.param_dict['alpha'] = 1.0
sats_occ_model.param_dict['logM0'] = 11.8
sats_occ_model.param_dict['logM1'] = 12.6
central_orientation_model.param_dict['central_alignment_strength'] = 0.57172919
satellite_orientation_model.param_dict['satellite_alignment_strength'] = 0.01995
# combine model components
model_instance = HodModelFactory(centrals_occupation = cens_occ_model,
centrals_profile = cens_prof_model,
satellites_occupation = sats_occ_model,
satellites_profile = sats_prof_model,
centrals_orientation = central_orientation_model,
satellites_orientation = satellite_orientation_model,
model_feature_calling_sequence = (
'centrals_occupation',
'centrals_profile',
'satellites_occupation',
'satellites_profile',
'centrals_orientation',
'satellites_orientation')
)
# populate mock catalog
model_instance.populate_mock(halocat)
print("number of galaxies: ", len(model_instance.mock.galaxy_table))
mock = model_instance.mock.galaxy_table
# galaxy coordinates and orientations
coords = np.vstack((mock['x'],
mock['y'],
mock['z'])).T
orientations = np.vstack((mock['galaxy_axisA_x'],
mock['galaxy_axisA_y'],
mock['galaxy_axisA_z'])).T
from halotools.mock_observables import tpcf, tpcf_jackknife
rbins = np.logspace(-1,1.5,15)
rbin_centers = (rbins[:-1]+rbins[1:])/2.0
xi = tpcf(coords, rbins, period=halocat.Lbox)
err=np.zeros(len(xi))
# save measurements
fpath = fpath = PROJECT_DIRECTORY + 'modelling_illustris/data/'
fname = sim_name + '_' + str(snapnum) + '-' + sample_name +'_model_xi.dat'
ascii.write([rbin_centers, xi, err],
fpath+fname,
names=['r','xi','err'],
overwrite=True)
def test_precomputed_rr(Nr, Mr=21):
'''
Mr = Luminositty threshold
Nr = Number of randoms
'''
rbins = np.logspace(-1, 1.25, 15)
rmax = rbins.max()
rbin_centers = (rbins[1:] + rbins[0:-1]) / 2.
halocat = CachedHaloCatalog(simname='bolshoi', redshift=0)
model = PrebuiltHodModelFactory("zheng07")
model.populate_mock(halocat=halocat, enforce_PBC=False)
data = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
print data.shape
L = halocat.Lbox
xmin, ymin, zmin = 0., 0., 0.
xmax, ymax, zmax = L, L, L
num_randoms = Nr
xran = np.random.uniform(xmin, xmax, num_randoms)
yran = np.random.uniform(ymin, ymax, num_randoms)
zran = np.random.uniform(zmin, zmax, num_randoms)
randoms = np.vstack((xran, yran, zran)).T
verbose = False
num_threads = cpu_count()
period = None
approx_cell1_size = [rmax, rmax, rmax]
approx_cell2_size = approx_cell1_size
approx_cellran_size = [rmax, rmax, rmax]
normal_result = tpcf(data,
rbins,
data,
randoms=randoms,
period=period,
max_sample_size=int(1e4),
estimator='Landy-Szalay',
approx_cell1_size=approx_cell1_size,
approx_cellran_size=approx_cellran_size)
#count data pairs
DD = npairs(data, data, rbins, period, verbose, num_threads,
approx_cell1_size, approx_cell2_size)
DD = np.diff(DD)
#count random pairs
RR = npairs(randoms, randoms, rbins, period, verbose, num_threads,
approx_cellran_size, approx_cellran_size)
RR = np.diff(RR)
#count data random pairs
DR = npairs(data, randoms, rbins, period, verbose, num_threads,
approx_cell1_size, approx_cell2_size)
DR = np.diff(DR)
print "DD=", DD
print "DR=", DR
print "RR=", RR
ND = len(data)
NR = len(randoms)
factor1 = ND * ND / (NR * NR)
factor2 = ND * NR / (NR * NR)
mult = lambda x, y: x * y
xi_LS = mult(1.0 / factor1, DD / RR) - mult(1.0 / factor2,
2.0 * DR / RR) + 1.0
print "xi=", xi_LS
print "normal=", normal_result
result_with_RR_precomputed = tpcf(data,
rbins,
data,
randoms=randoms,
period=period,
max_sample_size=int(1e5),
estimator='Landy-Szalay',
approx_cell1_size=approx_cell1_size,
approx_cellran_size=approx_cellran_size,
RR_precomputed=RR,
NR_precomputed=NR)
print "xi_pre=", result_with_RR_precomputed
# 6456 galaxies
eco_nobuff = eco_buff.loc[(eco_buff.grpcz.values >= 3000) & \
(eco_buff.grpcz.values <= 7000) & (eco_buff.absrmag.values <= -17.33) &\
(eco_buff.logmstar.values >= 8.9)]
cvar = 0.125
volume = 151829.26 # Survey volume without buffer [Mpc/h]^3
# BMF
eco_nobuff_mstellar = eco_nobuff.logmstar.values
eco_nobuff_mgas = eco_nobuff.logmgas.values
eco_nobuff_mbary = calc_bary(eco_nobuff_mstellar, eco_nobuff_mgas)
maxis, phi, err, bins = diff_bmf(eco_nobuff_mbary, volume, cvar, False)
halocat = CachedHaloCatalog(fname=halo_catalog, update_cached_fname=True)
z_model = np.median(eco_nobuff.grpcz.values) / (3 * 10**5)
model = PrebuiltSubhaloModelFactory('behroozi10', redshift=z_model,
prim_haloprop_key='halo_macc')
model.populate_mock(halocat)
# ensure reproducibility
rseed = 12
np.random.seed(rseed)
behroozi10_param_vals = [12.35,10.72,0.44,0.57,0.15]
nwalkers = 250
ndim = 5
p0 = behroozi10_param_vals + 0.1*np.random.rand(ndim*nwalkers).\
reshape((nwalkers,ndim))
wp_ng_vals = zehavi_data_file_20.get_wp()
bin_edges = zehavi_data_file_20.get_bins()
cov_matrix = zehavi_data_file_20.get_cov()
err = np.array([cov_matrix[i, i] for i in range(len(cov_matrix))])
bin_cen = (bin_edges[1:] + bin_edges[:-1]) / 2.
#cens_occ_model = Zheng07Cens(prim_haloprop_key = 'halo_vmax')
cens_occ_model = Zheng07Cens()
cens_prof_model = TrivialPhaseSpace()
#sats_occ_model = Zheng07Sats(prim_haloprop_key = 'halo_vmax', modulate_with_cenocc=True)
sats_occ_model = Zheng07Sats(modulate_with_cenocc=True)
sats_prof_model = NFWPhaseSpace()
halocat = CachedHaloCatalog(
fname=
'/home/lom31/.astropy/cache/halotools/halo_catalogs/SMDPL/rockstar/2019-07-03-18-38-02-9731.dat.my_cosmosim_halos.hdf5',
update_cached_fname=True)
halocat.redshift = 0.
pi_max = 60.
Lbox = 400.
model_instance = HodModelFactory(centrals_occupation=cens_occ_model,
centrals_profile=cens_prof_model,
satellites_occupation=sats_occ_model,
satellites_profile=sats_prof_model)
try:
model_instance.mock.populate()
except:
model_instance.populate_mock(halocat)
alpha, logM0, logM1 = [1.16, 13.28 - 1.7, 13.28]
def __init__(self, **kwargs):
"""
Initialize a ABHodFitModel.
"""
# first, set up appropriate priors on parameters
if ('priors' in kwargs.keys()):
self.set_prior(kwargs['priors'])
else:
self.set_prior(default_priors)
# set up keys for the parameter names for plotting
self.param_names = [
'alpha', 'logM1', 'siglogM', 'logM0', 'logMmin', 'Acens', 'Asats'
]
self.latex_param_names = [
r'$\alpha$', r'$\log(M_1)$', r'$\sigma_{\log M}$', r'$\log(M_0)$',
r'$\log(M_{\rm min})$', r'$\mathcal{A}_{\rm cens}$',
r'$\mathcal{A}_{\rm sats}'
]
# set up size parameters for any MCMC
self.set_nwalkers(ndim=default_ndim, nwalkers=default_nwalkers)
# if data is specified, load it into memory
if 'rpcut' in kwargs.keys():
self.rpcut = kwargs['rpcut']
else:
self.rpcut = default_rpcut
if ('datafile' in kwargs.keys()):
self.read_datafile(datafile=kwargs['datafile'])
else:
self.read_datafile(datafile=default_wp_datafile)
if ('covarfile' in kwargs.keys()):
self.read_covarfile(covarfile=kwargs['covarfile'])
else:
self.read_covarfile(covarfile=default_wp_covarfile)
# if binfile is specified, load it into memory
# these are Manodeep-style bins
if ('binfile' in kwargs.keys()):
self.binfile = kwargs['binfile']
else:
self.binfile = default_binfile
# set up a default HOD Model
if ('cen_occ_model' in kwargs.keys()):
cen_occ_model = kwargs['cen_occ_model']
else:
cen_occ_model = AssembiasZheng07Cens(
prim_haloprop_key='halo_mvir',
sec_haloprop_key='halo_nfw_conc')
if ('cen_prof_model' in kwargs.keys()):
cen_prof_model = kwargs['cen_prof_model']
else:
cen_prof_model = TrivialPhaseSpace()
if ('sat_occ_model' in kwargs.keys()):
sat_occ_model = kwargs['sat_occ_model']
else:
sat_occ_model = AssembiasZheng07Sats(
prim_haloprop_key='halo_mvir',
sec_haloprop_key='halo_nfw_conc')
if ('sat_prof_model' in kwargs.keys()):
sat_prof_model = kwargs['sat_prof_model']
else:
sat_prof_model = NFWPhaseSpace()
# Default HOD Model is Zheng07 with Heaviside Assembly Bias
self.hod_model = HodModelFactory(centrals_occupation=cen_occ_model,
centrals_profile=cen_prof_model,
satellites_occupation=sat_occ_model,
satellites_profile=sat_prof_model)
# set pi_max for wp(rp) calculations
self.pi_max = default_pi_max
if ('simname' in kwargs.keys()):
simname = kwargs['simname']
else:
simname = default_simname
if ('halo_finder' in kwargs.keys()):
halo_finder = kwargs['halo_finder']
else:
halo_finder = default_halofinder
if ('redshift' in kwargs.keys()):
redshift = kwargs['redshift']
else:
redshift = default_simredshift
if ('version_name' in kwargs.keys()):
version_name = kwargs['version_name']
else:
version_name = default_version_name
# set default simulation halocatalog to work with
self.halocatalog = CachedHaloCatalog(simname=simname,
halo_finder=halo_finder,
redshift=redshift,
version_name=version_name)
return None
def build_data_rr(nr):
from halotools.sim_manager import CachedHaloCatalog
halocat = CachedHaloCatalog(simname = 'bolshoi', redshift = 0)
num_randoms = 1e5
xran = np.random.uniform(xmin, xmax, num_randoms)
yran = np.random.uniform(ymin, ymax, num_randoms)
zran = np.random.uniform(zmin, zmax, num_randoms)
randoms = np.vstack((xran, yran, zran)).T
Lbox = halocat.Lbox
# Build observables ---------------
def build_xi_nbar_gmf(Mr=21):
'''
Build "data" xi, nbar, GMF values and write to file
'''
model = PrebuiltHodModelFactory('zheng07', threshold = -1.0*np.float(Mr))
model.populate_mock(halocat = halocat , enforce_PBC = False) # population mock realization
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# write xi
rbins = hardcoded_xi_bins()
rmax = rbins.max()
approx_cell1_size = [rmax, rmax, rmax]
approx_cell2_size = approx_cell1_size
approx_cellran_size = [rmax, rmax, rmax]
period = np.array([Lbox , Lbox , Lbox])
data_xir = tpcf(
sample1, rbins, sample2 = sample2,
randoms=randoms, period = period,
max_sample_size=int(1e4), estimator='Landy-Szalay',
approx_cell1_size=approx_cell1_size,
approx_cellran_size=approx_cellran_size,
RR_precomputed = RR,
NR_precomputed = NR1)
output_file = ''.join([util.dat_dir(), 'xir.Mr', str(Mr), '.dat'])
np.savetxt(output_file, data_xir)
# write nbar values
nbar = model.mock.number_density
output_file = ''.join([util.dat_dir(), 'nbar.Mr', str(Mr), '.dat'])
np.savetxt(output_file, [nbar])
# write GMF
rich = richness(model.mock.compute_fof_group_ids())
gmf = GMF(rich) # GMF
output_file = ''.join([util.dat_dir(), 'gmf.Mr', str(Mr), '.dat'])
np.savetxt(output_file, gmf)
return None
def build_xi_bins(Mr=21):
''' hardcoded r bins for xi.
'''
model = PrebuiltHodModelFactory('zheng07', threshold = -1.0*np.float(Mr))
model.populate_mock() # population mock realization
r_bin = model.mock.compute_galaxy_clustering(rbins=hardcoded_xi_bins())[0]
output_file = ''.join([util.dat_dir(), 'xir_rbin.Mr', str(Mr), '.dat'])
np.savetxt(output_file, r_bin)
return None
def hardcoded_xi_bins():
''' hardcoded xi bin edges.They are spaced out unevenly due to sparseness
at inner r bins. So the first bin ranges from 0.15 to 0.5
'''
r_bins = np.concatenate([np.array([0.15]), np.logspace(np.log10(0.5), np.log10(20.), 15)])
return r_bins
def build_xi_nbar_gmf_cov(Mr=21, Nmock=500):
'''
Build covariance matrix for xi, variance for nbar, and a bunch of stuff for gmf
...
using Nmock realizations of halotool mocks
'''
xir = []
model = PrebuiltHodModelFactory('zheng07', threshold = -1.0*np.float(Mr))
#nbars = []
#gmfs = []
#gmf_counts = []
for i in xrange(Nmock):
print 'mock#', i
model.populate_mock()
# xi(r)
xir.append(model.mock.compute_galaxy_clustering(rbins=hardcoded_xi_bins())[1])
# nbar
#nbars.append(model.mock.number_density)
# gmf
#rich = richness(model.mock.compute_fof_group_ids())
#gmfs.append(GMF(rich)) # GMF
#gmf_counts.append(GMF(rich, counts=True)) # Group counts
# save xi covariance
xi_covar = np.cov(np.array(xir).T)
output_file = ''.join([util.dat_dir(), 'xir_covariance.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
np.savetxt(output_file, xi_covar)
# save nbar values
#nbar_cov = np.var(nbars, axis=0)
#output_file = ''.join([util.dat_dir(), 'nbar_cov.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(output_file, [nbar_cov])
# write GMF covariance
#gmf_cov = np.cov(np.array(gmfs).T)
#output_file = ''.join([util.dat_dir(), 'gmf_cov.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(output_file, gmf_cov)
# write GMF Poisson
#gmf_counts_mean = np.mean(gmf_counts, axis=0)
#poisson_gmf = np.sqrt(gmf_counts_mean) / 250.**3 # poisson errors
#output_file = ''.join([util.dat_dir(), 'gmf_sigma_poisson.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(output_file, poisson_gmf)
# write GMF standard dev
#sigma_gmf = np.std(gmfs, axis=0) # sample variance
#output_file = ''.join([util.dat_dir(), 'gmf_sigma_stddev.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(output_file, sigma_gmf)
# write GMF total noise
#sigma_tot = (sigma_gmf**2 + poisson_gmf**2)**0.5 # total sigma
#output_file = ''.join([util.dat_dir(), 'gmf_sigma.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(output_file, sigma_tot)
# write full covariance matrix of various combinations of the data
# covariance for all three
#fulldatarr = np.hstack(np.array(nbars).reshape(nbars.shape[0], 1),
# np.array(gmfs), np.array(xir))
#fullcov = np.cov(fulldatarr.T)
#outfn = ''.join([util.dat_dir(), 'nbar_gmf_xir_cov.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(outfn, fullcov)
# covariance for nbar and gmf
##nbgmf_arr = np.hstack(np.array(nbars).reshape(nbars.shape[0], 1),
# np.array(gmfs))
#nbgmf_cov = np.cov(nbgmf_arr.T)
#outfn = ''.join([util.dat_dir(), 'nbar_gmf_cov.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(outfn, nbgmf_cov)
# covariance for nbar and xi
#nbxi_arr = np.hstack(np.array(nbars).reshape(nbars.shape[0], 1),
# np.array(xir))
#nbxi_cov = np.cov(nbxi_arr.T)
#outfn = ''.join([util.dat_dir(), 'nbar_xi_cov.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(outfn, nbxi_cov)
# covariance for gmf and xi
#gmfxi_arr = np.hstack(np.array(gmfs), np.array(xir))
#gmfxi_cov = np.cov(gmfxi_arr.T)
#outfn = ''.join([util.dat_dir(), 'gmf_xi_cov.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
#np.savetxt(outfn, gmfxi_cov)
return None
def build_xi_inv_cov(Mr=21, Nmock=500, unbias=True):
'''
Calculate the inverse covariance of xi multiplied by the unbiased
estimator factor (Nmocks - 2 - Nbins)/(Nmocks - 1).
Mainly used in MCMC inference
'''
xi_cov = data_xi_cov(Mr=Mr, Nmock=Nmock) # covariance matrix of xi
N_bins = int(np.sqrt(xi_cov.size)) # cov matrix is N_bin x N_bin
if unbias:
f_unbias = np.float(Nmock - 2. - N_bins)/np.float(Nmock - 1.)
unbias_str = '.unbias'
else:
f_unbias = 1.0
unbias_str = ''
inv_c = solve(np.eye(N_bins) , xi_cov) * f_unbias
output_file = ''.join([util.dat_dir(),
'xi_inv_cov', unbias_str, '.Mr', str(Mr), '.Nmock', str(Nmock), '.dat'])
np.savetxt(output_file, inv_c)
return None
def build_full_inv_covars(Mr=21, Nmock=50):
'''
Calculate the inverse covariance of full data vectors
for MCMC inference
'''
full_cov = data_full_cov(Mr=Mr, Nmock=Nmock)
N_bins = int(np.sqrt(full_cov.size))
f_unbias = np.float(Nmock - 2. - N_bins)/np.float(Nmock - 1.)
inv_c = solve(np.eye(N_bins) , full_cov) * f_unbias
outfn = ''.join([util.dat_dir(),
'nbar_gmf_xir_inv_cov.Mr', str(Mr),
'.Nmock', str(Nmock), '.dat'])
np.savetxt(outfn, inv_c)
nbgmf_cov = data_nbar_gmf_cov(Mr=Mr, Nmock=Nmock)
N_bins = int(np.sqrt(nbgmf_cov.size))
f_unbias = np.float(Nmock - 2. - N_bins)/np.float(Nmock - 1.)
inv_c = solve(np.eye(N_bins) , nbgmf_cov) * f_unbias
outfn = ''.join([util.dat_dir(),
'nbar_gmf_inv_cov.Mr', str(Mr),
'.Nmock', str(Nmock), '.dat'])
np.savetxt(outfn, inv_c)
nbxi_cov = data_nbar_xi_cov(Mr=Mr, Nmock=Nmock)
N_bins = int(np.sqrt(nbxi_cov.size))
f_unbias = np.float(Nmock - 2. - N_bins)/np.float(Nmock - 1.)
inv_c = solve(np.eye(N_bins) , nbxi_cov) * f_unbias
outfn = ''.join([util.dat_dir(),
'nbar_xi_inv_cov.Mr', str(Mr),
'.Nmock', str(Nmock), '.dat'])
np.savetxt(outfn, inv_c)
gmfxi_cov = data_gmf_xi_cov(Mr=Mr, Nmock=Nmock)
N_bins = int(np.sqrt(gmfxi_cov.size))
f_unbias = np.float(Nmock - 2. - N_bins)/np.float(Nmock - 1.)
inv_c = solve(np.eye(N_bins) , gmfxi_cov) * f_unbias
outfn = ''.join([util.dat_dir(),
'gmf_xi_inv_cov.Mr', str(Mr),
'.Nmock', str(Nmock), '.dat'])
np.savetxt(outfn, inv_c)
return None
def build_observations(Mr=21, Nmock=200):
''' Build all the fake observations
'''
# xi, nbar, gmf
print 'Building xi(r), nbar, GMF ... '
build_xi_bins(Mr=Mr)
build_xi_nbar_gmf(Mr=Mr)
# covariances
print 'Building covariances ... '
build_xi_nbar_gmf_cov(Mr=Mr, Nmock=Nmock)
print 'Build inverse covariance for xi ...'
build_xi_inv_cov(Mr=Mr, Nmock=Nmock, unbias=True)
print 'Building the rest of the inverse covariances...'
#build_full_inv_covars(Mr=Mr, Nmock=Nmock)
if __name__=='__main__':
build_observations()
from halotools.utils import crossmatch
from Ngal_estimate import Ngal_estimate
##########################################################
param_names = ('alpha', 'logM1', 'sigma_logM', 'logM0', 'logMmin',
'mean_occupation_centrals_assembias_param1',
'mean_occupation_satellites_assembias_param1')
output_names = ('ngals', 'deltasigma', 'rp', 'wprp', 'param')
##########################################################
halocat = CachedHaloCatalog(simname = 'diemerL0500', version_name = 'antonio', redshift = 0, \
halo_finder = 'rockstar',ptcl_version_name='antonioz0')
Lbox = 500
particle_portion = 0.1
rp_bins = np.logspace(-1.398, 1.176, 14) ##to match the leauthaud paper
num_ptcls_to_use = int(1e6)
particle_masses = np.zeros(
num_ptcls_to_use) + halocat.particle_mass / particle_portion
total_num_ptcls_in_snapshot = len(halocat.ptcl_table)
downsampling_factor = total_num_ptcls_in_snapshot / float(num_ptcls_to_use)
##ggl
pi_max = 60
r_wp = np.logspace(-1, np.log10(Lbox) - 1, 20)
##wp
def build_ABC_cov_nbar_xi_gmf(Mr=21, b_normal=0.25):
''' Build covariance matrix used in ABC for the full nbar, xi, gmf data vector
using realisations of galaxy mocks for "data" HOD parameters in the
halos from the multidark simulation. Covariance matrices for different sets of observables
can be extracted from the full covariance matrix by slicing through
the indices.
Notes
-----
* This covariance matrix is the covariance matrix calculated from the *entire* multidark
box. So this does _not_ account for the sample variance, which the MCMC covariance does.
'''
nbars, xir, gmfs = [], [], []
thr = -1. * np.float(Mr)
model = PrebuiltHodModelFactory('zheng07', threshold=thr)
halocat = CachedHaloCatalog(simname='multidark',
redshift=0,
halo_finder='rockstar')
rbins = xi_binedges() # some setting for tpcf calculations
rmax = rbins.max()
approx_cell1_size = [rmax, rmax, rmax]
approx_cellran_size = [rmax, rmax, rmax]
# load randoms and RRs for the ENTIRE MultiDark volume
###randoms = data_random(box='md_all')
###RR = data_RR(box='md_all')
###NR = len(randoms)
for i in xrange(1, 125):
print 'mock#', i
# populate the mock subvolume
model.populate_mock(halocat)
# returning the positions of galaxies
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# calculate nbar
nbars.append(len(pos) / 1000**3.)
# calculate xi(r) for the ENTIRE MultiDark volume
# using the natural estimator DD/RR - 1
xi = tpcf(pos,
rbins,
period=model.mock.Lbox,
max_sample_size=int(3e5),
estimator='Natural',
approx_cell1_size=approx_cell1_size)
xir.append(xi)
# calculate gmf
nbar = len(pos) / 1000**3.
b = b_normal * (nbar)**(-1. / 3)
groups = pyfof.friends_of_friends(pos, b)
w = np.array([len(x) for x in groups])
gbins = gmf_bins()
gmf = np.histogram(w, gbins)[0] / (1000.**3.)
gmfs.append(gmf) # GMF
# save nbar variance
nbar_var = np.var(nbars, axis=0, ddof=1)
nbar_file = ''.join([util.obvs_dir(), 'abc_nbar_var.Mr', str(Mr), '.dat'])
np.savetxt(nbar_file, [nbar_var])
# write full covariance matrix of various combinations of the data
# and invert for the likelihood evaluations
# --- covariance for all three ---
fulldatarr = np.hstack(
(np.array(nbars).reshape(len(nbars),
1), np.array(xir), np.array(gmfs)))
fullcov = np.cov(fulldatarr.T)
fullcorr = np.corrcoef(fulldatarr.T)
# and save the covariance matrix
nopoisson_file = ''.join([
util.obvs_dir(), 'ABC.nbar_xi_gmf_cov', '.no_poisson', '.Mr',
str(Mr), '.bnorm',
str(round(b_normal, 2)), '.dat'
])
np.savetxt(nopoisson_file, fullcov)
return None
def build_MCMC_cov_nbar_xi_gmf(Mr=21, b_normal=0.25):
''' Build covariance matrix used in MCMC for the full nbar, xi, gmf data vector
using realisations of galaxy mocks for "data" HOD parameters in the
halos from the other subvolumes (subvolume 1 to subvolume 125) of
the simulation. Covariance matrices for different sets of observables
can be extracted from the full covariance matrix by slicing through
the indices.
'''
nbars = []
xir = []
gmfs = []
thr = -1. * np.float(Mr)
model = PrebuiltHodModelFactory('zheng07', threshold=thr)
halocat = CachedHaloCatalog(simname='multidark',
redshift=0,
halo_finder='rockstar')
###model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}
#some settings for tpcf calculations
rbins = xi_binedges()
rmax = rbins.max()
approx_cell1_size = [rmax, rmax, rmax]
approx_cellran_size = [rmax, rmax, rmax]
#load randoms and RRs
randoms = data_random(box='md_sub')
RR = data_RR(box='md_sub')
NR = len(randoms)
for i in xrange(1, 125):
print 'mock#', i
# populate the mock subvolume
###mocksubvol = lambda x: util.mask_func(x, i)
###model.populate_mock(halocat,
### masking_function=mocksubvol,
### enforce_PBC=False)
model.populate_mock(halocat)
# returning the positions of galaxies in the entire volume
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# masking out the galaxies outside the subvolume i
pos = util.mask_galaxy_table(pos, i)
# calculate nbar
print "shape of pos", pos.shape
nbars.append(len(pos) / 200**3.)
# translate the positions of randoms to the new subbox
xi0, yi0, zi0 = util.random_shifter(i)
temp_randoms = randoms.copy()
temp_randoms[:, 0] += xi0
temp_randoms[:, 1] += yi0
temp_randoms[:, 2] += zi0
#calculate xi(r)
xi = tpcf(pos,
rbins,
pos,
randoms=temp_randoms,
period=None,
max_sample_size=int(3e5),
estimator='Natural',
approx_cell1_size=approx_cell1_size,
approx_cellran_size=approx_cellran_size,
RR_precomputed=RR,
NR_precomputed=NR)
xir.append(xi)
# calculate gmf
nbar = len(pos) / 200**3.
b = b_normal * (nbar)**(-1. / 3)
groups = pyfof.friends_of_friends(pos, b)
w = np.array([len(x) for x in groups])
gbins = gmf_bins()
gmf = np.histogram(w, gbins)[0] / 200.**3.
gmfs.append(gmf)
# save nbar variance
nbar_var = np.var(nbars, axis=0, ddof=1)
nbar_file = ''.join([util.obvs_dir(), 'nbar_var.Mr', str(Mr), '.dat'])
np.savetxt(nbar_file, [nbar_var])
# write full covariance matrix of various combinations of the data
# and invert for the likelihood evaluations
# --- covariance for all three ---
fulldatarr = np.hstack(
(np.array(nbars).reshape(len(nbars),
1), np.array(xir), np.array(gmfs)))
fullcov = np.cov(fulldatarr.T)
fullcorr = np.corrcoef(fulldatarr.T)
# and save the covariance matrix
nopoisson_file = ''.join([
util.obvs_dir(), 'MCMC.nbar_xi_gmf_cov', '.no_poisson', '.Mr',
str(Mr), '.bnorm',
str(round(b_normal, 2)), '.dat'
])
np.savetxt(nopoisson_file, fullcov)
return None
def build_nbar_xi_gmf(Mr=21, b_normal=0.25):
''' Build data vector [nbar, xi, gmf] and save to file
This data vector is built from the zeroth slice of the multidark
The other slices will be used for building the covariance matrix.
Parameters
----------
Mr : (int)
Absolute magnitude cut off M_r. Default M_r = -21.
b_normal : (float)
FoF Linking length
'''
thr = -1. * np.float(Mr)
model = PrebuiltHodModelFactory('zheng07', threshold=thr)
halocat = CachedHaloCatalog(simname='multidark',
redshift=0,
halo_finder='rockstar')
####model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}
####datsubvol = lambda x: util.mask_func(x, 0)
####model.populate_mock(halocat, masking_function=datsubvol, enforce_PBC=False)
model.populate_mock(halocat)
#all the things necessary for tpcf calculation
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
#masking the galaxies outside the subvolume 0
pos = util.mask_galaxy_table(pos, 0)
rbins = xi_binedges()
rmax = rbins.max()
approx_cell1_size = [rmax, rmax, rmax]
approx_cellran_size = [rmax, rmax, rmax]
#compute number density
nbar = len(pos) / 200**3.
# load MD subvolume randoms and RRs
randoms = data_random(box='md_sub')
RR = data_RR(box='md_sub')
NR = len(randoms)
#compue tpcf with Natural estimator
data_xir = tpcf(pos,
rbins,
pos,
randoms=randoms,
period=None,
max_sample_size=int(2e5),
estimator='Natural',
approx_cell1_size=approx_cell1_size,
approx_cellran_size=approx_cellran_size,
RR_precomputed=RR,
NR_precomputed=NR)
fullvec = np.append(nbar, data_xir)
#compute gmf
b = b_normal * (nbar)**(-1. / 3)
groups = pyfof.friends_of_friends(pos, b)
w = np.array([len(x) for x in groups])
gbins = gmf_bins()
gmf = np.histogram(w, gbins)[0] / (200.**3.)
fullvec = np.append(fullvec, gmf)
output_file = data_file(Mr=Mr, b_normal=b_normal)
np.savetxt(output_file, fullvec)
return None
model2 = PrebuiltSubhaloModelFactory('behroozi10',
redshift=z_median,
prim_haloprop_key='halo_macc')
model3 = PrebuiltSubhaloModelFactory('behroozi10',
redshift=z_median,
prim_haloprop_key='halo_macc')
model4 = PrebuiltSubhaloModelFactory('behroozi10',
redshift=z_median,
prim_haloprop_key='halo_macc')
model5 = PrebuiltSubhaloModelFactory('behroozi10',
redshift=z_median,
prim_haloprop_key='halo_macc')
print('Setting up halocats')
###Halocats
halocat1 = CachedHaloCatalog(fname=halo_catalog)
halocat2 = CachedHaloCatalog(fname=halo_catalog)
halocat3 = CachedHaloCatalog(fname=halo_catalog)
halocat4 = CachedHaloCatalog(fname=halo_catalog)
halocat5 = CachedHaloCatalog(fname=halo_catalog)
print('Initial mock population')
###Populate mocks
model1.populate_mock(halocat1)
model2.populate_mock(halocat2)
model3.populate_mock(halocat3)
model4.populate_mock(halocat4)
model5.populate_mock(halocat5)
def gals(Mhalo_value, Mstellar_value, Mlow_slope, Mhigh_slope,
#########################################################
Lbox = args.Lbox
r_vpf = np.logspace(0, 1., args.vpfNbin)
num_sphere = int(1e5)
##vpf
ptcl_accept_rate = 0.1
rp_bins_ggl = np.logspace(-1, 1.5, args.dsNbin + 1)
##ggl
##########################################################
halocat = CachedHaloCatalog(simname = args.simname, version_name = args.version, redshift = args.redshift, \
halo_finder = args.halofinder)
model = decorated_hod_model()
model.param_dict.update(dict(zip(
param_names,
fiducial_p))) ##update model.param_dict with pairs (param_names:params)
try:
model.mock.populate()
except:
model.populate_mock(halocat)
gc.collect()
pos_gals_d = return_xyz_formatted_array(*(model.mock.galaxy_table[ax] for ax in 'xyz'), \
velocity=model.mock.galaxy_table['vz'], velocity_distortion_dimension='z',\
def build_nbar_xi_gmf_cov(Mr=21):
''' Build covariance matrix for the full nbar, xi, gmf data vector
using realisations of galaxy mocks for "data" HOD parameters in the
halos from the multidark simulation. Covariance matrices for different sets of observables
can be extracted from the full covariance matrix by slicing through
the indices.
'''
nbars = []
xir = []
gmfs = []
thr = -1. * np.float(Mr)
model = PrebuiltHodModelFactory('zheng07', threshold=thr)
halocat = CachedHaloCatalog(simname='multidark',
redshift=0,
halo_finder='rockstar')
#some settings for tpcf calculations
rbins = hardcoded_xi_bins()
for i in xrange(1, 125):
print 'mock#', i
# populate the mock subvolume
model.populate_mock(halocat)
# returning the positions of galaxies
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# calculate nbar
nbars.append(len(pos) / 1000**3.)
# translate the positions of randoms to the new subbox
#calculate xi(r)
xi = tpcf(pos,
rbins,
period=model.mock.Lbox,
max_sample_size=int(2e5),
estimator='Landy-Szalay')
xir.append(xi)
# calculate gmf
nbar = len(pos) / 1000**3.
b_normal = 0.75
b = b_normal * (nbar)**(-1. / 3)
groups = pyfof.friends_of_friends(pos, b)
w = np.array([len(x) for x in groups])
gbins = gmf_bins()
gmf = np.histogram(w, gbins)[0] / (1000.**3.)
gmfs.append(gmf) # GMF
# save nbar variance
nbar_var = np.var(nbars, axis=0, ddof=1)
nbar_file = ''.join(
[util.multidat_dir(), 'abc_nbar_var.Mr',
str(Mr), '.dat'])
np.savetxt(nbar_file, [nbar_var])
# write full covariance matrix of various combinations of the data
# and invert for the likelihood evaluations
# --- covariance for all three ---
fulldatarr = np.hstack(
(np.array(nbars).reshape(len(nbars),
1), np.array(xir), np.array(gmfs)))
fullcov = np.cov(fulldatarr.T)
fullcorr = np.corrcoef(fulldatarr.T)
# and save the covariance matrix
nopoisson_file = ''.join([
util.multidat_dir(), 'abc_nbar_xi_gmf_cov.no_poisson.Mr',
str(Mr), '.dat'
])
np.savetxt(nopoisson_file, fullcov)
# and a correlation matrix
full_corr_file = ''.join(
[util.multidat_dir(), 'abc_nbar_xi_gmf_corr.Mr',
str(Mr), '.dat'])
np.savetxt(full_corr_file, fullcorr)
return None