def main(path2config):
# load the yaml parameters
config = yaml.load(open(path2config))
sim_params = config['sim_params']
HOD_params = config['HOD_params']
clustering_params = config['clustering_params']
# additional parameter choices
want_rsd = HOD_params['want_rsd']
write_to_disk = HOD_params['write_to_disk']
# run the HODs (note: the first time you call the function run_hod, the script takes a bit to compile)
newBall = AbacusHOD(sim_params, HOD_params, clustering_params)
mock_dict = newBall.run_hod(tracers=newBall.tracers,
want_rsd=want_rsd,
write_to_disk=write_to_disk,
Nthread=16)
def main(path2config):
# load the yaml parameters
config = yaml.load(open(path2config))
sim_params = config['sim_params']
HOD_params = config['HOD_params']
clustering_params = config['clustering_params']
# additional parameter choices
want_rsd = HOD_params['want_rsd']
write_to_disk = HOD_params['write_to_disk']
bin_params = clustering_params['bin_params']
rpbins = np.logspace(bin_params['logmin'], bin_params['logmax'],
bin_params['nbins'])
pimax = clustering_params['pimax']
pi_bin_size = clustering_params['pi_bin_size']
# create a new abacushod object
newBall = AbacusHOD(sim_params, HOD_params, clustering_params)
# throw away run for jit to compile, write to disk
mock_dict = newBall.run_hod(newBall.tracers,
want_rsd,
write_to_disk=True,
Nthread=16)
# mock_dict = newBall.gal_reader()
# xirppi = newBall.compute_xirppi(mock_dict, rpbins, pimax, pi_bin_size)
# print(xirppi)
# wp = newBall.compute_wp(mock_dict, rpbins, pimax, pi_bin_size)
# print(wp)
# run the fit 10 times for timing
for i in range(10):
print(i)
# example for sandy
newBall.tracers['LRG']['alpha'] += 0.01
print("alpha = ", newBall.tracers['LRG']['alpha'])
start = time.time()
mock_dict = newBall.run_hod(newBall.tracers,
want_rsd,
write_to_disk,
Nthread=64)
print("Done iteration ", i, "took time ", time.time() - start)
def main(path2config):
# load the yaml parameters
config = yaml.load(open(path2config))
sim_params = config['sim_params']
HOD_params = config['HOD_params']
clustering_params = config['clustering_params']
# additional parameter choices
want_rsd = HOD_params['want_rsd']
write_to_disk = HOD_params['write_to_disk']
bin_params = clustering_params['bin_params']
rpbins = np.logspace(bin_params['logmin'], bin_params['logmax'],
bin_params['nbins'] + 1)
pimax = clustering_params['pimax']
pi_bin_size = clustering_params['pi_bin_size']
# run the HODs
newBall = AbacusHOD(sim_params, HOD_params, clustering_params)
mock_dict = newBall.run_hod(tracers=newBall.tracers,
want_rsd=want_rsd,
write_to_disk=write_to_disk,
Nthread=16)
# can change some parameter and run again to time
zs = [0.1]
for i in range(len(zs)):
# create a new abacushod object
sim_params['z_mock'] = zs[i]
newBall = AbacusHOD(sim_params, HOD_params, clustering_params)
start = time.time()
mock_dict = newBall.run_hod(tracers=newBall.tracers,
want_rsd=want_rsd,
write_to_disk=False,
Nthread=16)
print("Done hod, took time ", time.time() - start)
def main(path2config):
# load the yaml parameters
config = yaml.load(open(path2config))
sim_params = config['sim_params']
HOD_params = config['HOD_params']
clustering_params = config['clustering_params']
data_params = config['data_params']
dynesty_config_params = config['dynesty_config_params']
fit_params = config['dynesty_fit_params']
# create a new abacushod object and load the subsamples
newBall = AbacusHOD(sim_params, HOD_params, clustering_params)
# read data parameters
newData = wp_Data(data_params, HOD_params)
# parameters to fit
nparams = len(fit_params.keys())
param_mapping = {}
param_tracer = {}
params = np.zeros((nparams, 2))
for key in fit_params.keys():
mapping_idx = fit_params[key][0]
tracer_type = fit_params[key][-1]
param_mapping[key] = mapping_idx
param_tracer[key] = tracer_type
params[mapping_idx, :] = fit_params[key][1:-1]
# Make path to output
if not os.path.isdir(
os.path.expanduser(dynesty_config_params['path2output'])):
try:
os.makedirs(
os.path.expanduser(dynesty_config_params['path2output']))
except:
pass
# dynesty parameters
nlive = dynesty_config_params['nlive']
maxcall = dynesty_config_params['maxcall']
method = dynesty_config_params['method']
bound = dynesty_config_params['bound']
# where to record
prefix_chain = os.path.join(
os.path.expanduser(dynesty_config_params['path2output']),
dynesty_config_params['chainsPrefix'])
# initiate sampler
found_file = os.path.isfile(prefix_chain + '.dill')
if (not found_file) or (not dynesty_config_params['rerun']):
# initialize our nested sampler
sampler = NestedSampler(
lnprob,
prior_transform,
nparams,
logl_args=[param_mapping, param_tracer, newData, newBall],
ptform_args=[params[:, 0], params[:, 1]],
nlive=nlive,
sample=method,
rstate=np.random.RandomState(dynesty_config_params['rseed']))
# first_update = {'min_eff': 20})
else:
# load sampler to continue the run
with open(prefix_chain + '.dill', "rb") as f:
sampler = dill.load(f)
sampler.rstate = np.load(prefix_chain + '_results.npz')['rstate']
print("run sampler")
sampler.run_nested(maxcall=maxcall)
# save sampler itself
with open(prefix_chain + '.dill', "wb") as f:
dill.dump(sampler, f)
res1 = sampler.results
np.savez(prefix_chain + '_results.npz',
res=res1,
rstate=np.random.get_state())
def main(path2config, time_likelihood):
# load the yaml parameters
config = yaml.load(open(path2config))
sim_params = config['sim_params']
HOD_params = config['HOD_params']
clustering_params = config['clustering_params']
data_params = config['data_params']
ch_config_params = config['ch_config_params']
fit_params = config['fit_params']
# create a new abacushod object and load the subsamples
newBall = AbacusHOD(sim_params, HOD_params, clustering_params)
# read data parameters
newData = PowerData(data_params, HOD_params)
# parameters to fit
nparams = len(fit_params.keys())
param_mapping = {}
param_tracer = {}
params = np.zeros((nparams, 4))
for key in fit_params.keys():
mapping_idx = fit_params[key][0]
tracer_type = fit_params[key][-1]
param_mapping[key] = mapping_idx
param_tracer[key] = tracer_type
params[mapping_idx, :] = fit_params[key][1:-1]
# Make path to output
if not os.path.isdir(os.path.expanduser(ch_config_params['path2output'])):
try:
os.makedirs(os.path.expanduser(ch_config_params['path2output']))
except:
pass
# MPI option
if ch_config_params['use_mpi']:
from schwimmbad import MPIPool
pool = MPIPool()
print("Using MPI")
pool_use = pool
else:
pool = DumPool()
print("Not using MPI")
pool_use = None
if not pool.is_master():
pool.wait()
sys.exit(0)
# just time the likelihood calculation
if time_likelihood:
time_lnprob(params, param_mapping, param_tracer, newData, newBall)
return
# emcee parameters
nwalkers = nparams * ch_config_params['walkersRatio']
nsteps = ch_config_params['burninIterations'] + ch_config_params[
'sampleIterations']
# where to record
prefix_chain = os.path.join(
os.path.expanduser(ch_config_params['path2output']),
ch_config_params['chainsPrefix'])
# fix initial conditions
found_file = os.path.isfile(prefix_chain + '.txt')
if (not found_file) or (not ch_config_params['rerun']):
p_initial = params[:, 0] + np.random.normal(
size=(nwalkers, nparams)) * params[:, 3][None, :]
nsteps_use = nsteps
else:
print("Restarting from a previous run")
old_chain = np.loadtxt(prefix_chain + '.txt')
p_initial = old_chain[-nwalkers:, :]
nsteps_use = max(nsteps - len(old_chain) // nwalkers, 0)
# initializing sampler
chain_file = SampleFileUtil(prefix_chain,
carry_on=ch_config_params['rerun'])
sampler = emcee.EnsembleSampler(nwalkers,
nparams,
lnprob,
args=(params, param_mapping, param_tracer,
newData, newBall),
pool=pool_use)
start = time.time()
print("Running %d samples" % nsteps_use)
# record every iteration
counter = 1
for pos, prob, _ in sampler.sample(p_initial, iterations=nsteps_use):
if pool.is_master():
print('Iteration done. Persisting.')
chain_file.persistSamplingValues(pos, prob)
if counter % 10:
print(f"Finished sample {counter}")
counter += 1
pool.close()
end = time.time()
print("Took ", (end - start), " seconds")
def test_hod(tmp_path, reference_mode = False):
'''Test loading a halo catalog
'''
from abacusnbody.hod import prepare_sim
from abacusnbody.hod.abacus_hod import AbacusHOD
config = yaml.safe_load(open(EXAMPLE_CONFIG))
# inform abacus_hod where the simulation files are, relative to the cwd
config['sim_params']['sim_dir'] = pjoin(TESTDIR, 'halo_light_cones')
sim_params = config['sim_params']
HOD_params = config['HOD_params']
clustering_params = config['clustering_params']
# reference mode
if reference_mode:
print("Generating new reference files...")
prepare_sim.main(EXAMPLE_CONFIG)
# additional parameter choices
want_rsd = HOD_params['want_rsd']
bin_params = clustering_params['bin_params']
# create a new abacushod object
newBall = AbacusHOD(sim_params, HOD_params, clustering_params)
mock_dict = newBall.run_hod(newBall.tracers, want_rsd, write_to_disk = True, Nthread = 2)
# test mode
else:
simname = config['sim_params']['sim_name'] # "AbacusSummit_base_c000_ph006"
simdir = config['sim_params']['sim_dir']
z_mock = config['sim_params']['z_mock']
# all output dirs should be under tmp_path
config['sim_params']['output_dir'] = pjoin(tmp_path, 'data_mocks_summit_new') + '/'
config['sim_params']['subsample_dir'] = pjoin(tmp_path, "data_subs") + '/'
config['sim_params']['scratch_dir'] = pjoin(tmp_path, "data_gals") + '/'
savedir = config['sim_params']['subsample_dir'] + simname+"/z"+str(z_mock).ljust(5, '0')
# check subsample file match
prepare_sim.main(EXAMPLE_CONFIG, params = config)
newhalos = h5py.File(savedir+'/halos_xcom_0_seed600_abacushod_oldfenv_MT_new.h5', 'r')['halos']
temphalos = h5py.File(EXAMPLE_SUBSAMPLE_HALOS, 'r')['halos']
for i in range(len(newhalos)):
for j in range(len(newhalos[i])):
assert check_close(newhalos[i][j], temphalos[i][j])
newparticles = h5py.File(savedir+'/particles_xcom_0_seed600_abacushod_oldfenv_MT_new.h5', 'r')['particles']
tempparticles = h5py.File(EXAMPLE_SUBSAMPLE_PARTS, 'r')['particles']
for i in range(len(newparticles)):
for j in range(len(newparticles[i])):
assert check_close(newparticles[i][j], tempparticles[i][j])
# additional parameter choices
want_rsd = HOD_params['want_rsd']
write_to_disk = HOD_params['write_to_disk']
bin_params = clustering_params['bin_params']
rpbins = np.logspace(bin_params['logmin'], bin_params['logmax'], bin_params['nbins'])
pimax = clustering_params['pimax']
pi_bin_size = clustering_params['pi_bin_size']
# create a new abacushod object
newBall = AbacusHOD(sim_params, HOD_params, clustering_params)
# throw away run for jit to compile, write to disk
mock_dict = newBall.run_hod(newBall.tracers, want_rsd, write_to_disk = True, Nthread = 2)
savedir_gal = config['sim_params']['output_dir']\
+"/"+simname+"/z"+str(z_mock).ljust(5, '0') +"/galaxies_rsd/LRGs.dat"
data = ascii.read(EXAMPLE_LRGS)
data1 = ascii.read(savedir_gal)
for ekey in data.keys():
assert check_close(data[ekey], data1[ekey])
savedir_gal = config['sim_params']['output_dir']\
+"/"+simname+"/z"+str(z_mock).ljust(5, '0') +"/galaxies_rsd/ELGs.dat"
data = ascii.read(EXAMPLE_ELGS)
data1 = ascii.read(savedir_gal)
for ekey in data.keys():
assert check_close(data[ekey], data1[ekey])
def main(path2config):
# load the yaml parameters
config = yaml.load(open(path2config))
sim_params = config['sim_params']
HOD_params = config['HOD_params']
clustering_params = config['clustering_params']
# additional parameter choices
want_rsd = HOD_params['want_rsd']
write_to_disk = HOD_params['write_to_disk']
bin_params = clustering_params['bin_params']
rpbins = np.logspace(bin_params['logmin'], bin_params['logmax'],
bin_params['nbins'] + 1)
pimax = clustering_params['pimax']
pi_bin_size = clustering_params['pi_bin_size']
# create a new abacushod object
newBall = AbacusHOD(sim_params, HOD_params, clustering_params)
# throw away run for jit to compile, write to disk
mock_dict = newBall.run_hod(newBall.tracers,
want_rsd,
write_to_disk=False,
Nthread=16)
# mock_dict = newBall.gal_reader()
start = time.time()
xirppi = newBall.compute_xirppi(mock_dict,
rpbins,
pimax,
pi_bin_size,
Nthread=32)
print("Done xi, total time ", time.time() - start)
# print(xirppi)
# wp = newBall.compute_wp(mock_dict, rpbins, pimax, pi_bin_size)
# print(wp)
# run the fit 10 times for timing
meantime = 0
Ntest = 20
for i in range(Ntest):
print(i)
# # run hod, ngal, xirppi
# newBall.tracers['LRG']['alpha'] += 0.01
# print("alpha = ",newBall.tracers['LRG']['alpha'])
start = time.time()
mock_dict = newBall.run_hod(newBall.tracers,
want_rsd,
write_to_disk,
Nthread=64)
print("Done hod, took time ", time.time() - start)
start = time.time()
# ngal_dict = newBall.compute_ngal()
# print("Done ngal, took time ", time.time() - start, ngal_dict)
xirppi = newBall.compute_xirppi(mock_dict,
rpbins,
pimax,
pi_bin_size,
Nthread=32)
deltat = time.time() - start
print("Done xi, total time ", deltat)
meantime += deltat
print("meantime ", meantime / Ntest)
def test_hod(tmp_path, reference_mode=False):
'''Test loading a halo catalog
'''
from abacusnbody.hod import prepare_sim
from abacusnbody.hod.abacus_hod import AbacusHOD
# reference mode
if reference_mode:
prepare_sim.main(path2config)
# load the yaml parameters
config = yaml.load(open(path2config))
sim_params = config['sim_params']
HOD_params = config['HOD_params']
clustering_params = config['clustering_params']
# additional parameter choices
want_rsd = HOD_params['want_rsd']
write_to_disk = HOD_params['write_to_disk']
bin_params = clustering_params['bin_params']
rpbins = np.logspace(bin_params['logmin'], bin_params['logmax'],
bin_params['nbins'])
pimax = clustering_params['pimax']
pi_bin_size = clustering_params['pi_bin_size']
# create a new abacushod object
newBall = AbacusHOD(sim_params, HOD_params, clustering_params)
mock_dict = newBall.run_hod(newBall.tracers,
want_rsd,
write_to_disk=True,
Nthread=2)
# test mode
else:
config = yaml.load(open(EXAMPLE_CONFIG))
sim_params = config['sim_params']
HOD_params = config['HOD_params']
clustering_params = config['clustering_params']
simname = config['sim_params'][
'sim_name'] # "AbacusSummit_base_c000_ph006"
simdir = config['sim_params']['sim_dir']
z_mock = config['sim_params']['z_mock']
config['sim_params']['subsample_dir'] = str(tmp_path) + "/data_subs/"
config['sim_params']['scratch_dir'] = str(tmp_path) + "/data_gals/"
savedir = config['sim_params']['subsample_dir'] + simname + "/z" + str(
z_mock).ljust(5, '0')
# check subsample file match
prepare_sim.main(EXAMPLE_CONFIG, params=config)
newhalos = h5py.File(
savedir + '/halos_xcom_2_seed600_abacushod_new.h5', 'r')['halos']
temphalos = h5py.File(EXAMPLE_SUBSAMPLE_HALOS, 'r')['halos']
for i in range(len(newhalos)):
for j in range(len(newhalos[i])):
assert np.array_equal(newhalos[i][j], temphalos[i][j])
newparticles = h5py.File(
savedir + '/particles_xcom_2_seed600_abacushod_new.h5',
'r')['particles']
tempparticles = h5py.File(EXAMPLE_SUBSAMPLE_PARTS, 'r')['particles']
for i in range(len(newparticles)):
for j in range(len(newparticles[i])):
assert np.array_equal(newparticles[i][j], tempparticles[i][j])
# additional parameter choices
want_rsd = HOD_params['want_rsd']
write_to_disk = HOD_params['write_to_disk']
bin_params = clustering_params['bin_params']
rpbins = np.logspace(bin_params['logmin'], bin_params['logmax'],
bin_params['nbins'])
pimax = clustering_params['pimax']
pi_bin_size = clustering_params['pi_bin_size']
# create a new abacushod object
newBall = AbacusHOD(sim_params, HOD_params, clustering_params)
# throw away run for jit to compile, write to disk
mock_dict = newBall.run_hod(newBall.tracers,
want_rsd,
write_to_disk=True,
Nthread=2)
savedir_gal = config['sim_params']['scratch_dir']\
+"/"+simname+"/z"+str(z_mock).ljust(5, '0') +"/galaxies_rsd/LRGs.dat"
data = ascii.read(EXAMPLE_GALS)
data1 = ascii.read(savedir_gal)
for ekey in data.keys():
assert np.allclose(data[ekey], data1[ekey])