def test_subvol_gmf(Mr):
"""
"""
thr = -1.0 * np.float(Mr)
model = PrebuiltHodModelFactory("zheng07", threshold=thr, halocat="multidark", redshift=0.0)
model.new_haloprop_func_dict = {"sim_subvol": util.mk_id_column}
datsubvol = lambda x: util.mask_func(x, 0)
model.populate_mock(simname="multidark", masking_function=datsubvol, enforce_PBC=False)
# compute group richness
# These are wrong because they assume periodicity
# rich = richness(model.mock.compute_fof_group_ids())
# gmf = GMF(rich) #GMF
# print gmf
# print GMF(rich , counts = True)
galaxy_sample = model.mock.galaxy_table
x = galaxy_sample["x"]
y = galaxy_sample["y"]
z = galaxy_sample["z"]
vz = galaxy_sample["vz"]
pos = three_dim_pos_bundle(model.mock.galaxy_table, "x", "y", "z", velocity=vz, velocity_distortion_dimension="z")
b_para, b_perp = 0.2, 0.2
groups = FoFGroups(pos, b_perp, b_para, period=None, Lbox=200, num_threads="max")
gids = groups.group_ids
rich = richness(gids)
gmf = GMF(rich)
print gmf
print GMF(rich, counts=True)
return None
def test_GMFbinning(Mr):
""" Tests for the GMF binning scheme in order to make it sensible.
"""
gids_saved = "gids_saved.p"
if not os.path.isfile(gids_saved):
thr = -1.0 * np.float(Mr)
model = PrebuiltHodModelFactory("zheng07", threshold=thr, halocat="multidark", redshift=0.0)
model.new_haloprop_func_dict = {"sim_subvol": util.mk_id_column}
datsubvol = lambda x: util.mask_func(x, 0)
model.populate_mock(simname="multidark", masking_function=datsubvol, enforce_PBC=False)
galaxy_sample = model.mock.galaxy_table
x = galaxy_sample["x"]
y = galaxy_sample["y"]
z = galaxy_sample["z"]
vz = galaxy_sample["vz"]
pos = three_dim_pos_bundle(
model.mock.galaxy_table, "x", "y", "z", velocity=vz, velocity_distortion_dimension="z"
)
b_para, b_perp = 0.2, 0.2
groups = FoFGroups(pos, b_perp, b_para, period=None, Lbox=200, num_threads="max")
pickle.dump(groups, open(gids_saved, "wb"))
else:
groups = pickle.load(open(gids_saved, "rb"))
gids = groups.group_ids
rich = richness(gids)
# print "rich=" , rich
rbins = np.logspace(np.log10(3.0), np.log10(20), 10)
rbins = np.array([1, 2.0, 3.0, 4.0, 5.0, 6.0, 7, 9, 11, 14, 17, 20])
gmf = GMF(rich, counts=False, bins=rbins)
gmf_counts = GMF(rich, counts=True, bins=rbins)
print rbins
print gmf
print gmf_counts
fig = plt.figure(1)
sub = fig.add_subplot(111)
sub.plot(0.5 * (rbins[:-1] + rbins[1:]), gmf)
sub.set_xscale("log")
sub.set_yscale("log")
plt.show()
# print gmf
# print GMF(rich , counts = True)
return None
def test_subvol_gmf(Mr):
'''
'''
thr = -1. * np.float(Mr)
model = PrebuiltHodModelFactory('zheng07',
threshold=thr,
halocat='multidark',
redshift=0.)
model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}
datsubvol = lambda x: util.mask_func(x, 0)
model.populate_mock(simname='multidark',
masking_function=datsubvol,
enforce_PBC=False)
#compute group richness
# These are wrong because they assume periodicity
#rich = richness(model.mock.compute_fof_group_ids())
#gmf = GMF(rich) #GMF
#print gmf
#print GMF(rich , counts = True)
galaxy_sample = model.mock.galaxy_table
x = galaxy_sample['x']
y = galaxy_sample['y']
z = galaxy_sample['z']
vz = galaxy_sample['vz']
pos = three_dim_pos_bundle(model.mock.galaxy_table,
'x',
'y',
'z',
velocity=vz,
velocity_distortion_dimension="z")
b_para, b_perp = 0.2, 0.2
groups = FoFGroups(pos,
b_perp,
b_para,
period=None,
Lbox=200,
num_threads='max')
gids = groups.group_ids
rich = richness(gids)
gmf = GMF(rich)
print gmf
print GMF(rich, counts=True)
return None
def main():
###############################Multi-Dark###########################
model = PrebuiltHodModelFactory('zheng07', threshold=-21)
halocat = CachedHaloCatalog(simname = 'multidark', redshift = 0, halo_finder = 'rockstar')
t0 = time.time()
model.populate_mock(halocat, enforce_PBC = False)
print(time.time() - t0)
t0 = time.time()
model.populate_mock(halocat, enforce_PBC = False)
print(time.time() - t0)
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
nthreads = 1
binfile = path.join(path.dirname(path.abspath(__file__)),
"/home/mj/Corrfunc/xi_theory/tests/", "bins")
autocorr = 1
pos = pos.astype(np.float32)
x , y , z = pos[:,0] , pos[:,1] , pos[:,2]
DD = _countpairs.countpairs(autocorr, nthreads, binfile, x, y, z,
x , y , z)
ND = len(pos)
NR = 50*800000
DD = np.array(DD)[:,3]
num_randoms = 50 * 800000
xran = np.random.uniform(0, 1000, num_randoms)
yran = np.random.uniform(0, 1000, num_randoms)
zran = np.random.uniform(0, 1000, num_randoms)
randoms = np.vstack((xran, yran, zran)).T
randoms = randoms.astype(np.float32)
xran = randoms[:,0]
yran = randoms[:,1]
zran = randoms[:,2]
results_RR = _countpairs.countpairs(autocorr, nthreads, binfile, xran, yran, zran,
xran, yran, zran)
RR = np.array(results_RR)[:,3]
factor1 = 1.*ND*ND/(NR*NR)
mult = lambda x,y: x*y
xi_MD = mult(1.0/factor1 , DD/RR) - 1.
print(xi_MD)
###############################Subvolume of Multi-Dark###########################
num_srandoms = 50 * 8000
xran = np.random.uniform(0, 200, num_srandoms)
yran = np.random.uniform(0, 200, num_srandoms)
zran = np.random.uniform(0, 200, num_srandoms)
randoms = np.vstack((xran, yran, zran)).T
randoms = randoms.astype(np.float32)
xran = randoms[:,0]
yran = randoms[:,1]
zran = randoms[:,2]
results_RR = _countpairs.countpairs(autocorr, nthreads, binfile, xran, yran, zran,
xran, yran, zran)
RR_sub = np.array(results_RR)[:,3]
import util
sub_model = PrebuiltHodModelFactory('zheng07' , threshold = -21)
sub_model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}
sub_halocat = CachedHaloCatalog(simname='multidark', redshift=0, halo_finder='rockstar')
xi_subs = []
for i in range(10):
simsubvol = lambda x: util.mask_func(x, i)
sub_model.populate_mock(sub_halocat, masking_function=simsubvol, enforce_PBC=False)
sub_pos = three_dim_pos_bundle(sub_model.mock.galaxy_table, 'x', 'y', 'z')
nthreads = 1
binfile = path.join(path.dirname(path.abspath(__file__)),
"/home/mj/Corrfunc/xi_theory/tests/", "bins")
autocorr = 1
sub_pos = sub_pos.astype(np.float32)
x , y , z = sub_pos[:,0] , sub_pos[:,1] , sub_pos[:,2]
DD_sub = _countpairs.countpairs(autocorr, nthreads, binfile, x, y, z,
x, y, z)
ND_sub = len(sub_pos)
NR_sub = 50 * 8000
DD_sub = np.array(DD_sub)[:,3]
factor1 = 1.*ND_sub*ND_sub/(NR_sub*NR_sub)
mult = lambda x,y: x*y
xi_n = mult(1.0/factor1 , DD_sub/RR_sub) - 1.
xi_subs.append(xi_n)
xi_subs = np.array(xi_subs)
np.savetxt("xi_subs.dat" , xi_subs)
import matplotlib.pyplot as plt
from ChangTools.plotting import prettyplot
from ChangTools.plotting import prettycolors
binfile = path.join(path.dirname(path.abspath(__file__)),
"/home/mj/Corrfunc/xi_theory/tests/", "bins")
rbins = np.loadtxt(binfile)
rbins_centers = np.mean(rbins , axis = 1)
xi_subs = np.loadtxt("xi_subs.dat")
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(111)
for i in range(10):
ax.semilogx(rbins_centers , xi_subs[i,:] / xi_MD , alpha = 0.2)
plt.xlabel(r"$r$" , fontsize = 20)
plt.ylabel(r"$\xi_{\rm subvolume}(r) / \xi_{\rm MD}(r)$" , fontsize = 20)
plt.savefig("xi_ratios.pdf")
def Subvolume_FullvolumeCut(N_sub, ratio=False):
''' Test the 2PCF estimates from MultiDark subvolume versus the
2PCF for the entire MultiDark volume WITHOUT periodic boundary conditions
and actual pair counts, CUT into subvolumes of the same size *AFTER*
populate mock
Parameters
----------
N_sub : (int)
Number of subvolumes to sample
'''
prettyplot()
pretty_colors = prettycolors()
pickle_file = ''.join([
'/export/bbq2/hahn/ccppabc/dump/',
'xi_subvolume_fullvolume_cut_test',
'.Nsub', str(N_sub),
'.p'])
fig = plt.figure(1)
sub = fig.add_subplot(111)
xi_bin = xi_binedges()
# Entire MultiDark Volume (No Periodic Boundary Conditions)
model = PrebuiltHodModelFactory('zheng07', threshold=-21)
halocat = CachedHaloCatalog(simname = 'multidark', redshift = 0, halo_finder = 'rockstar')
sub_RR = data_RR(box='md_sub')
sub_randoms = data_random(box='md_sub')
sub_NR = len(sub_randoms)
rmax = xi_bin.max()
full_approx_cell1_size = [rmax , rmax , rmax]
full_approx_cellran_size = [rmax , rmax , rmax]
model.populate_mock(halocat, enforce_PBC=False)
subvol_id = util.mk_id_column(table=model.mock.galaxy_table)
full_pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# Full Volume
if os.path.isfile(pickle_file):
data_dump = pickle.load(open(pickle_file, 'rb'))
full_xi = data_dump['full_xi']
else:
model = PrebuiltHodModelFactory('zheng07', threshold=-21)
halocat = CachedHaloCatalog(simname = 'multidark', redshift = 0, halo_finder = 'rockstar')
full_randoms = data_random(box='md_all')
full_RR = data_RR(box='md_all')
full_NR = len(full_randoms)
rmax = xi_bin.max()
full_approx_cell1_size = [rmax , rmax , rmax]
full_approx_cellran_size = [rmax , rmax , rmax]
model.populate_mock(halocat, enforce_PBC=False)
full_pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
full_xi = tpcf(
full_pos, xi_bin,
randoms=full_randoms, period=None,
do_auto=True,
do_cross=False,
num_threads=5,
max_sample_size=int(full_pos.shape[0]), estimator='Natural',
approx_cell1_size=full_approx_cell1_size,
approx_cellran_size=full_approx_cellran_size,
RR_precomputed = full_RR,
NR_precomputed = full_NR)
data_dump = {}
data_dump['full_xi'] = full_xi
if not ratio:
sub.plot(0.5*(xi_bin[:-1]+xi_bin[1:]), full_xi,
lw=2, ls='-', c='k', label=r'Full Volume')
if os.path.isfile(pickle_file):
fullcut_xi_list = data_dump['fullcut_xi']['fullcut_xi_list']
fullcut_xi_avg = data_dump['fullcut_xi']['fullcut_xi_avg']
else:
data_dump['fullcut_xi'] = {}
fullcut_xi_list = []
fullcut_xi_tot = np.zeros(len(xi_bin)-1)
for id in np.unique(subvol_id)[:N_sub]:
print 'Subvolume ', id
in_cut = np.where(subvol_id == id)
fullcut_pos = full_pos[in_cut]
fullcut_xi = tpcf(
fullcut_pos, xi_bin,
randoms=sub_randoms, period=None,
do_auto=True,
do_cross=False,
num_threads=5,
max_sample_size=int(fullcut_pos.shape[0]), estimator='Natural',
approx_cell1_size=full_approx_cell1_size,
approx_cellran_size=full_approx_cellran_size,
RR_precomputed=sub_RR,
NR_precomputed=sub_NR)
fullcut_xi_list.append(fullcut_xi)
fullcut_xi_tot += fullcut_xi
fullcut_xi_avg = fullcut_xi_tot / np.float(N_sub)
data_dump['fullcut_xi']['fullcut_xi_list']= fullcut_xi_list
data_dump['fullcut_xi']['fullcut_xi_avg']= fullcut_xi_avg
if not ratio:
sub.plot(0.5*(xi_bin[:-1]+xi_bin[1:]), fullcut_xi_avg,
lw=2, ls='-', c='k', label=r'Full Volume Cut Average')
else:
sub.plot(0.5*(xi_bin[:-1]+xi_bin[1:]), fullcut_xi_avg/full_xi,
lw=2, ls='-', c='k', label=r'Full Volume Cut Average')
if not os.path.isfile(pickle_file):
# MultiDark SubVolume (precomputed RR pairs)
sub_model = PrebuiltHodModelFactory('zheng07', threshold=-21)
sub_model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}
sub_halocat = CachedHaloCatalog(simname = 'multidark', redshift = 0, halo_finder = 'rockstar')
sub_RR = data_RR(box='md_sub')
sub_randoms = data_random(box='md_sub')
sub_NR = len(sub_randoms)
sub_xis_list = []
sub_xis = np.zeros(len(full_xi))
for ii in range(1,N_sub):
print 'Subvolume ', ii
# randomly sample one of the subvolumes
rint = ii #np.random.randint(1, 125)
simsubvol = lambda x: util.mask_func(x, rint)
sub_model.populate_mock(sub_halocat, masking_function=simsubvol, enforce_PBC=False)
pos = three_dim_pos_bundle(sub_model.mock.galaxy_table, 'x', 'y', 'z')
xi, yi , zi = util.random_shifter(rint)
temp_randoms = sub_randoms.copy()
temp_randoms[:,0] += xi
temp_randoms[:,1] += yi
temp_randoms[:,2] += zi
rmax = xi_bin.max()
sub_approx_cell1_size = [rmax , rmax , rmax]
sub_approx_cellran_size = [rmax , rmax , rmax]
sub_xi = tpcf(
pos, xi_bin,
randoms=temp_randoms,
period = None,
do_auto=True,
do_cross=False,
num_threads=5,
max_sample_size=int(pos.shape[0]),
estimator='Natural',
approx_cell1_size = sub_approx_cell1_size,
approx_cellran_size = sub_approx_cellran_size,
RR_precomputed=sub_RR,
NR_precomputed=sub_NR)
label = None
if ii == N_sub - 1:
label = 'Subvolumes'
sub_xis += sub_xi
sub_xis_list.append(sub_xi)
sub_xi_avg = sub_xis/np.float(N_sub)
data_dump['Natural'] = {}
data_dump['Natural']['sub_xi_avg'] = sub_xi_avg
data_dump['Natural']['sub_xis_list'] = sub_xis_list
else:
sub_xis_list = data_dump['Natural']['sub_xis_list']
sub_xi_avg = data_dump['Natural']['sub_xi_avg']
if not os.path.isfile(pickle_file):
pickle.dump(data_dump, open(pickle_file, 'wb'))
if not ratio:
sub.plot(0.5*(xi_bin[:-1]+xi_bin[1:]), sub_xi_avg,
lw=2, ls='--', c=pretty_colors[3], label='Subvolume')
else:
sub.plot(0.5*(xi_bin[:-1]+xi_bin[1:]), sub_xi_avg/full_xi,
lw=2, ls='--', c=pretty_colors[3], label='Subvolume')
sub.set_xlim([0.1, 50.])
sub.set_xlabel('r', fontsize=30)
sub.set_xscale('log')
if not ratio:
sub.set_ylabel(r"$\xi \mathtt{(r)}$", fontsize=25)
sub.set_yscale('log')
else:
sub.set_ylabel(r"$\overline{\xi^\mathtt{sub}}/\xi^\mathtt{all}$", fontsize=25)
sub.legend(loc='lower left')
if ratio:
fig_file = ''.join([util.fig_dir(), 'test_xi_subvolume_fullvolume_cut.Nsub', str(N_sub), '.ratio.png'])
else:
fig_file = ''.join([util.fig_dir(), 'test_xi_subvolume_fullvolume_cut.Nsub', str(N_sub), '.png'])
fig.savefig(fig_file, bbox_inches='tight', dpi=100)
plt.close()
return None
def Subvolume_Analytic(N_sub, ratio=False):
''' Test the 2PCF estimates from MultiDark subvolume versus the
analytic 2PCF for the entire MultiDark volume
Parameters
----------
N_sub : (int)
Number of subvolumes to sample
'''
prettyplot()
pretty_colors = prettycolors()
pickle_file = ''.join([
'/export/bbq2/hahn/ccppabc/dump/',
'xi_subvolume_test',
'.Nsub', str(N_sub),
'.p'])
fig = plt.figure(1)
sub = fig.add_subplot(111)
xi_bin = xi_binedges()
if os.path.isfile(pickle_file):
data_dump = pickle.load(open(pickle_file, 'rb'))
full_xi = data_dump['full_xi']
else:
# Entire MultiDark Volume (Analytic xi)
model = PrebuiltHodModelFactory('zheng07', threshold=-21)
halocat = CachedHaloCatalog(simname = 'multidark', redshift = 0, halo_finder = 'rockstar')
model.populate_mock(halocat)
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# while the estimator claims to be Landy-Szalay, I highly suspect it
# actually uses Landy-Szalay since DR pairs cannot be calculated from
# analytic randoms
full_xi = tpcf(pos, xi_bin, period=model.mock.Lbox, max_sample_size=int(2e5), estimator='Landy-Szalay', num_threads=1)
data_dump = {}
data_dump['full_xi'] = full_xi
if not ratio:
sub.plot(0.5*(xi_bin[:-1]+xi_bin[1:]), full_xi, lw=2, ls='-', c='k', label=r'Analytic $\xi$ Entire Volume')
if not os.path.isfile(pickle_file):
# MultiDark SubVolume (precomputed RR pairs)
sub_model = PrebuiltHodModelFactory('zheng07', threshold=-21)
sub_model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}
sub_halocat = CachedHaloCatalog(simname = 'multidark', redshift = 0, halo_finder = 'rockstar')
RR = data_RR()
randoms = data_random()
NR = len(randoms)
for method in ['Landy-Szalay', 'Natural']:
if method == 'Landy-Szalay':
iii = 3
elif method == 'Natural':
iii = 5
if not os.path.isfile(pickle_file):
sub_xis_list = []
sub_xis = np.zeros(len(full_xi))
for ii in range(1,N_sub+1):
# randomly sample one of the subvolumes
rint = ii #np.random.randint(1, 125)
simsubvol = lambda x: util.mask_func(x, rint)
sub_model.populate_mock(sub_halocat, masking_function=simsubvol, enforce_PBC=False)
pos = three_dim_pos_bundle(sub_model.mock.galaxy_table, 'x', 'y', 'z')
xi, yi , zi = util.random_shifter(rint)
temp_randoms = randoms.copy()
temp_randoms[:,0] += xi
temp_randoms[:,1] += yi
temp_randoms[:,2] += zi
rmax = xi_bin.max()
approx_cell1_size = [rmax , rmax , rmax]
approx_cellran_size = [rmax , rmax , rmax]
sub_xi = tpcf(
pos, xi_bin, pos,
randoms=temp_randoms,
period = None,
max_sample_size=int(1e5),
estimator=method,
approx_cell1_size = approx_cell1_size,
approx_cellran_size = approx_cellran_size,
RR_precomputed=RR,
NR_precomputed=NR)
label = None
if ii == N_sub - 1:
label = 'Subvolumes'
#if not ratio:
# sub.plot(0.5*(xi_bin[:-1]+xi_bin[1:]), sub_xi, lw=0.5, ls='--', c=pretty_colors[iii])
sub_xis += sub_xi
sub_xis_list.append(sub_xi)
sub_xi_avg = sub_xis/np.float(N_sub)
data_dump[method] = {}
data_dump[method]['sub_xi_avg'] = sub_xi_avg
data_dump[method]['sub_xis_list'] = sub_xis_list
else:
sub_xis_list = data_dump[method]['sub_xis_list']
sub_xi_avg = data_dump[method]['sub_xi_avg']
if not ratio:
sub.plot(0.5*(xi_bin[:-1]+xi_bin[1:]), sub_xi_avg,
lw=2, ls='--', c=pretty_colors[iii], label='Subvolume '+method)
else:
sub.plot(0.5*(xi_bin[:-1]+xi_bin[1:]), sub_xi_avg/full_xi,
lw=2, ls='--', c=pretty_colors[iii], label='Subvolume '+method)
if not os.path.isfile(pickle_file):
pickle.dump(data_dump, open(pickle_file, 'wb'))
sub.set_xlim([0.1, 50.])
sub.set_xlabel('r', fontsize=30)
sub.set_xscale('log')
if not ratio:
sub.set_ylabel(r"$\xi \mathtt{(r)}$", fontsize=25)
sub.set_yscale('log')
else:
sub.set_ylabel(r"$\overline{\xi^\mathtt{sub}}/\xi^\mathtt{all}$", fontsize=25)
sub.legend(loc='lower left')
if ratio:
fig_file = ''.join([util.fig_dir(), 'test_xi_subvolume_analytic.Nsub', str(N_sub), '.ratio.png'])
else:
fig_file = ''.join([util.fig_dir(), 'test_xi_subvolume_analytic.Nsub', str(N_sub), '.png'])
fig.savefig(fig_file, bbox_inches='tight', dpi=100)
plt.close()
return None
estimator='Natural',
approx_cell1_size=[rmax, rmax, rmax],
approx_cellran_size=[rmax, rmax, rmax])
print "done"
Nsub = 8
# Now set up for subvol boxes
num_randoms = 50 * 8000
xran = np.random.uniform(0, 250, num_randoms)
yran = np.random.uniform(0, 250, num_randoms)
zran = np.random.uniform(0, 250, num_randoms)
sub_randoms = np.vstack((xran, yran, zran)).T
sub_model = PrebuiltHodModelFactory('zheng07')
sub_model.new_haloprop_func_dict = {'sim_subvol': mk_id_column}
sub_halocat = CachedHaloCatalog(simname='multidark', redshift=0,
halo_finder='rockstar')
print "starting with 250 mpc subvols:"
sub_xi_list = []
for ii in range(1,Nsub+1):
print ii
simsubvol = lambda x: mask_func(x, ii)
sub_model.populate_mock(sub_halocat,
masking_function=simsubvol, enforce_PBC=False)
sub_pos = three_dim_pos_bundle(sub_model.mock.galaxy_table,
'x', 'y', 'z')
xi, yi, zi = random_shifter(ii)
def test_GMFbinning(Mr):
''' Tests for the GMF binning scheme in order to make it sensible.
'''
gids_saved = 'gids_saved.p'
if not os.path.isfile(gids_saved):
thr = -1. * np.float(Mr)
model = PrebuiltHodModelFactory('zheng07',
threshold=thr,
halocat='multidark',
redshift=0.)
model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}
datsubvol = lambda x: util.mask_func(x, 0)
model.populate_mock(simname='multidark',
masking_function=datsubvol,
enforce_PBC=False)
galaxy_sample = model.mock.galaxy_table
x = galaxy_sample['x']
y = galaxy_sample['y']
z = galaxy_sample['z']
vz = galaxy_sample['vz']
pos = three_dim_pos_bundle(model.mock.galaxy_table,
'x',
'y',
'z',
velocity=vz,
velocity_distortion_dimension="z")
b_para, b_perp = 0.2, 0.2
groups = FoFGroups(pos,
b_perp,
b_para,
period=None,
Lbox=200,
num_threads='max')
pickle.dump(groups, open(gids_saved, 'wb'))
else:
groups = pickle.load(open(gids_saved, 'rb'))
gids = groups.group_ids
rich = richness(gids)
#print "rich=" , rich
rbins = np.logspace(np.log10(3.), np.log10(20), 10)
rbins = np.array([1, 2., 3., 4., 5., 6., 7, 9, 11, 14, 17, 20])
gmf = GMF(rich, counts=False, bins=rbins)
gmf_counts = GMF(rich, counts=True, bins=rbins)
print rbins
print gmf
print gmf_counts
fig = plt.figure(1)
sub = fig.add_subplot(111)
sub.plot(0.5 * (rbins[:-1] + rbins[1:]), gmf)
sub.set_xscale('log')
sub.set_yscale('log')
plt.show()
#print gmf
#print GMF(rich , counts = True)
return None
def Subvolume_FullvolumeCut(N_sub, ratio=False):
''' Test the 2PCF estimates from MultiDark subvolume versus the
2PCF for the entire MultiDark volume WITHOUT periodic boundary conditions
and actual pair counts, CUT into subvolumes of the same size *AFTER*
populate mock
Parameters
----------
N_sub : (int)
Number of subvolumes to sample
'''
prettyplot()
pretty_colors = prettycolors()
pickle_file = ''.join([
'/export/bbq2/hahn/ccppabc/dump/', 'xi_subvolume_fullvolume_cut_test',
'.Nsub',
str(N_sub), '.p'
])
fig = plt.figure(1)
sub = fig.add_subplot(111)
xi_bin = xi_binedges()
# Entire MultiDark Volume (No Periodic Boundary Conditions)
model = PrebuiltHodModelFactory('zheng07', threshold=-21)
halocat = CachedHaloCatalog(simname='multidark',
redshift=0,
halo_finder='rockstar')
sub_RR = data_RR(box='md_sub')
sub_randoms = data_random(box='md_sub')
sub_NR = len(sub_randoms)
rmax = xi_bin.max()
full_approx_cell1_size = [rmax, rmax, rmax]
full_approx_cellran_size = [rmax, rmax, rmax]
model.populate_mock(halocat, enforce_PBC=False)
subvol_id = util.mk_id_column(table=model.mock.galaxy_table)
full_pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# Full Volume
if os.path.isfile(pickle_file):
data_dump = pickle.load(open(pickle_file, 'rb'))
full_xi = data_dump['full_xi']
else:
model = PrebuiltHodModelFactory('zheng07', threshold=-21)
halocat = CachedHaloCatalog(simname='multidark',
redshift=0,
halo_finder='rockstar')
full_randoms = data_random(box='md_all')
full_RR = data_RR(box='md_all')
full_NR = len(full_randoms)
rmax = xi_bin.max()
full_approx_cell1_size = [rmax, rmax, rmax]
full_approx_cellran_size = [rmax, rmax, rmax]
model.populate_mock(halocat, enforce_PBC=False)
full_pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
full_xi = tpcf(full_pos,
xi_bin,
randoms=full_randoms,
period=None,
do_auto=True,
do_cross=False,
num_threads=5,
max_sample_size=int(full_pos.shape[0]),
estimator='Natural',
approx_cell1_size=full_approx_cell1_size,
approx_cellran_size=full_approx_cellran_size,
RR_precomputed=full_RR,
NR_precomputed=full_NR)
data_dump = {}
data_dump['full_xi'] = full_xi
if not ratio:
sub.plot(0.5 * (xi_bin[:-1] + xi_bin[1:]),
full_xi,
lw=2,
ls='-',
c='k',
label=r'Full Volume')
if os.path.isfile(pickle_file):
fullcut_xi_list = data_dump['fullcut_xi']['fullcut_xi_list']
fullcut_xi_avg = data_dump['fullcut_xi']['fullcut_xi_avg']
else:
data_dump['fullcut_xi'] = {}
fullcut_xi_list = []
fullcut_xi_tot = np.zeros(len(xi_bin) - 1)
for id in np.unique(subvol_id)[:N_sub]:
print 'Subvolume ', id
in_cut = np.where(subvol_id == id)
fullcut_pos = full_pos[in_cut]
fullcut_xi = tpcf(fullcut_pos,
xi_bin,
randoms=sub_randoms,
period=None,
do_auto=True,
do_cross=False,
num_threads=5,
max_sample_size=int(fullcut_pos.shape[0]),
estimator='Natural',
approx_cell1_size=full_approx_cell1_size,
approx_cellran_size=full_approx_cellran_size,
RR_precomputed=sub_RR,
NR_precomputed=sub_NR)
fullcut_xi_list.append(fullcut_xi)
fullcut_xi_tot += fullcut_xi
fullcut_xi_avg = fullcut_xi_tot / np.float(N_sub)
data_dump['fullcut_xi']['fullcut_xi_list'] = fullcut_xi_list
data_dump['fullcut_xi']['fullcut_xi_avg'] = fullcut_xi_avg
if not ratio:
sub.plot(0.5 * (xi_bin[:-1] + xi_bin[1:]),
fullcut_xi_avg,
lw=2,
ls='-',
c='k',
label=r'Full Volume Cut Average')
else:
sub.plot(0.5 * (xi_bin[:-1] + xi_bin[1:]),
fullcut_xi_avg / full_xi,
lw=2,
ls='-',
c='k',
label=r'Full Volume Cut Average')
if not os.path.isfile(pickle_file):
# MultiDark SubVolume (precomputed RR pairs)
sub_model = PrebuiltHodModelFactory('zheng07', threshold=-21)
sub_model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}
sub_halocat = CachedHaloCatalog(simname='multidark',
redshift=0,
halo_finder='rockstar')
sub_RR = data_RR(box='md_sub')
sub_randoms = data_random(box='md_sub')
sub_NR = len(sub_randoms)
sub_xis_list = []
sub_xis = np.zeros(len(full_xi))
for ii in range(1, N_sub):
print 'Subvolume ', ii
# randomly sample one of the subvolumes
rint = ii #np.random.randint(1, 125)
simsubvol = lambda x: util.mask_func(x, rint)
sub_model.populate_mock(sub_halocat,
masking_function=simsubvol,
enforce_PBC=False)
pos = three_dim_pos_bundle(sub_model.mock.galaxy_table, 'x', 'y',
'z')
xi, yi, zi = util.random_shifter(rint)
temp_randoms = sub_randoms.copy()
temp_randoms[:, 0] += xi
temp_randoms[:, 1] += yi
temp_randoms[:, 2] += zi
rmax = xi_bin.max()
sub_approx_cell1_size = [rmax, rmax, rmax]
sub_approx_cellran_size = [rmax, rmax, rmax]
sub_xi = tpcf(pos,
xi_bin,
randoms=temp_randoms,
period=None,
do_auto=True,
do_cross=False,
num_threads=5,
max_sample_size=int(pos.shape[0]),
estimator='Natural',
approx_cell1_size=sub_approx_cell1_size,
approx_cellran_size=sub_approx_cellran_size,
RR_precomputed=sub_RR,
NR_precomputed=sub_NR)
label = None
if ii == N_sub - 1:
label = 'Subvolumes'
sub_xis += sub_xi
sub_xis_list.append(sub_xi)
sub_xi_avg = sub_xis / np.float(N_sub)
data_dump['Natural'] = {}
data_dump['Natural']['sub_xi_avg'] = sub_xi_avg
data_dump['Natural']['sub_xis_list'] = sub_xis_list
else:
sub_xis_list = data_dump['Natural']['sub_xis_list']
sub_xi_avg = data_dump['Natural']['sub_xi_avg']
if not os.path.isfile(pickle_file):
pickle.dump(data_dump, open(pickle_file, 'wb'))
if not ratio:
sub.plot(0.5 * (xi_bin[:-1] + xi_bin[1:]),
sub_xi_avg,
lw=2,
ls='--',
c=pretty_colors[3],
label='Subvolume')
else:
sub.plot(0.5 * (xi_bin[:-1] + xi_bin[1:]),
sub_xi_avg / full_xi,
lw=2,
ls='--',
c=pretty_colors[3],
label='Subvolume')
sub.set_xlim([0.1, 50.])
sub.set_xlabel('r', fontsize=30)
sub.set_xscale('log')
if not ratio:
sub.set_ylabel(r"$\xi \mathtt{(r)}$", fontsize=25)
sub.set_yscale('log')
else:
sub.set_ylabel(r"$\overline{\xi^\mathtt{sub}}/\xi^\mathtt{all}$",
fontsize=25)
sub.legend(loc='lower left')
if ratio:
fig_file = ''.join([
util.fig_dir(), 'test_xi_subvolume_fullvolume_cut.Nsub',
str(N_sub), '.ratio.png'
])
else:
fig_file = ''.join([
util.fig_dir(), 'test_xi_subvolume_fullvolume_cut.Nsub',
str(N_sub), '.png'
])
fig.savefig(fig_file, bbox_inches='tight', dpi=100)
plt.close()
return None
def Subvolume_Analytic(N_sub, ratio=False):
''' Test the 2PCF estimates from MultiDark subvolume versus the
analytic 2PCF for the entire MultiDark volume
Parameters
----------
N_sub : (int)
Number of subvolumes to sample
'''
prettyplot()
pretty_colors = prettycolors()
pickle_file = ''.join([
'/export/bbq2/hahn/ccppabc/dump/', 'xi_subvolume_test', '.Nsub',
str(N_sub), '.p'
])
fig = plt.figure(1)
sub = fig.add_subplot(111)
xi_bin = xi_binedges()
if os.path.isfile(pickle_file):
data_dump = pickle.load(open(pickle_file, 'rb'))
full_xi = data_dump['full_xi']
else:
# Entire MultiDark Volume (Analytic xi)
model = PrebuiltHodModelFactory('zheng07', threshold=-21)
halocat = CachedHaloCatalog(simname='multidark',
redshift=0,
halo_finder='rockstar')
model.populate_mock(halocat)
pos = three_dim_pos_bundle(model.mock.galaxy_table, 'x', 'y', 'z')
# while the estimator claims to be Landy-Szalay, I highly suspect it
# actually uses Landy-Szalay since DR pairs cannot be calculated from
# analytic randoms
full_xi = tpcf(pos,
xi_bin,
period=model.mock.Lbox,
max_sample_size=int(2e5),
estimator='Landy-Szalay',
num_threads=1)
data_dump = {}
data_dump['full_xi'] = full_xi
if not ratio:
sub.plot(0.5 * (xi_bin[:-1] + xi_bin[1:]),
full_xi,
lw=2,
ls='-',
c='k',
label=r'Analytic $\xi$ Entire Volume')
if not os.path.isfile(pickle_file):
# MultiDark SubVolume (precomputed RR pairs)
sub_model = PrebuiltHodModelFactory('zheng07', threshold=-21)
sub_model.new_haloprop_func_dict = {'sim_subvol': util.mk_id_column}
sub_halocat = CachedHaloCatalog(simname='multidark',
redshift=0,
halo_finder='rockstar')
RR = data_RR()
randoms = data_random()
NR = len(randoms)
for method in ['Landy-Szalay', 'Natural']:
if method == 'Landy-Szalay':
iii = 3
elif method == 'Natural':
iii = 5
if not os.path.isfile(pickle_file):
sub_xis_list = []
sub_xis = np.zeros(len(full_xi))
for ii in range(1, N_sub + 1):
# randomly sample one of the subvolumes
rint = ii #np.random.randint(1, 125)
simsubvol = lambda x: util.mask_func(x, rint)
sub_model.populate_mock(sub_halocat,
masking_function=simsubvol,
enforce_PBC=False)
pos = three_dim_pos_bundle(sub_model.mock.galaxy_table, 'x',
'y', 'z')
xi, yi, zi = util.random_shifter(rint)
temp_randoms = randoms.copy()
temp_randoms[:, 0] += xi
temp_randoms[:, 1] += yi
temp_randoms[:, 2] += zi
rmax = xi_bin.max()
approx_cell1_size = [rmax, rmax, rmax]
approx_cellran_size = [rmax, rmax, rmax]
sub_xi = tpcf(pos,
xi_bin,
pos,
randoms=temp_randoms,
period=None,
max_sample_size=int(1e5),
estimator=method,
approx_cell1_size=approx_cell1_size,
approx_cellran_size=approx_cellran_size,
RR_precomputed=RR,
NR_precomputed=NR)
label = None
if ii == N_sub - 1:
label = 'Subvolumes'
#if not ratio:
# sub.plot(0.5*(xi_bin[:-1]+xi_bin[1:]), sub_xi, lw=0.5, ls='--', c=pretty_colors[iii])
sub_xis += sub_xi
sub_xis_list.append(sub_xi)
sub_xi_avg = sub_xis / np.float(N_sub)
data_dump[method] = {}
data_dump[method]['sub_xi_avg'] = sub_xi_avg
data_dump[method]['sub_xis_list'] = sub_xis_list
else:
sub_xis_list = data_dump[method]['sub_xis_list']
sub_xi_avg = data_dump[method]['sub_xi_avg']
if not ratio:
sub.plot(0.5 * (xi_bin[:-1] + xi_bin[1:]),
sub_xi_avg,
lw=2,
ls='--',
c=pretty_colors[iii],
label='Subvolume ' + method)
else:
sub.plot(0.5 * (xi_bin[:-1] + xi_bin[1:]),
sub_xi_avg / full_xi,
lw=2,
ls='--',
c=pretty_colors[iii],
label='Subvolume ' + method)
if not os.path.isfile(pickle_file):
pickle.dump(data_dump, open(pickle_file, 'wb'))
sub.set_xlim([0.1, 50.])
sub.set_xlabel('r', fontsize=30)
sub.set_xscale('log')
if not ratio:
sub.set_ylabel(r"$\xi \mathtt{(r)}$", fontsize=25)
sub.set_yscale('log')
else:
sub.set_ylabel(r"$\overline{\xi^\mathtt{sub}}/\xi^\mathtt{all}$",
fontsize=25)
sub.legend(loc='lower left')
if ratio:
fig_file = ''.join([
util.fig_dir(), 'test_xi_subvolume_analytic.Nsub',
str(N_sub), '.ratio.png'
])
else:
fig_file = ''.join([
util.fig_dir(), 'test_xi_subvolume_analytic.Nsub',
str(N_sub), '.png'
])
fig.savefig(fig_file, bbox_inches='tight', dpi=100)
plt.close()
return None