def load_core_cat(core_loc):
print("loading: ", core_loc)
t1 = time.time()
cat = {}
if ".hdf5" in core_loc:
hfile = h5py.File(core_loc, 'r')
cat['infall_mass'] = hfile['m_peak'].value
cat['radius'] = hfile['r_peak'].value
cat['infall_step'] = hfile['infall_step'].value
else:
dtk.gio_inspect(core_loc)
cat['infall_mass'] = dtk.gio_read(core_loc, 'infall_mass')
cat['radius'] = dtk.gio_read(core_loc, 'radius')
cat['infall_step'] = dtk.gio_read(core_loc, 'infall_step')
print(t1 - time.time())
plt.figure()
slct = cat['infall_step'] == 401
h, xbins, ybins = np.histogram2d(
cat['infall_mass'][slct],
cat['radius'][slct],
bins=[np.logspace(10, 15, 100),
np.logspace(-4, 0.0, 100)])
plt.pcolor(xbins, ybins, h.T, cmap='Blues', norm=clr.LogNorm())
plt.xlabel('Core M_infall [Msun/h]')
plt.ylabel('Core Radius [Mpc/h]')
plt.xscale('log')
plt.yscale('log')
return cat
def corner_plot_mcmc(labels,
mcmc_loc,
fig,
axs,
colors=['tab:blue', 'k', 'tab:red'],
plot_hist=True,
alpha=0.3):
mcmc_m_i = dtk.gio_read(mcmc_loc, "mcmc_mass_infall")
mcmc_r_d = dtk.gio_read(mcmc_loc, "mcmc_r_disrupt")
mcmc_id = dtk.gio_read(mcmc_loc, "mcmc_walker_id")
mcmc_step = dtk.gio_read(mcmc_loc, "mcmc_walker_step")
mcmc_val = dtk.gio_read(mcmc_loc, "mcmc_value")
size = len(labels)
data = []
slct = mcmc_step > np.max(mcmc_step) / 2.0
data.append(np.log10(mcmc_m_i[slct]))
data.append(mcmc_r_d[slct])
best_fit_indx = np.argmin(mcmc_val[slct])
#Diagonal Covariance
for i in range(0, len(labels)):
ax = axs[i][i]
h, xbins = np.histogram(data[i], bins=50, density=True)
ax.plot(dtk.bins_avg(xbins), h, c=colors[0])
# Calculate 1 simga limits
b1, b2 = get_bounds_limits(h, xbins, 0)
b2 += 0
b1 += 0
b1a = b1 - 1
ylim = ax.get_ylim()
ax.fill_between(dtk.bins_avg(bbins[b1a:b2]),
0,
h[b1:b2],
lw=0.0,
alpha=alpha,
color=colors[0])
ax.set_xlim(np.min(xbins), np.max(xbins))
ax.set_ylim(ylim) #restore old ylims before fill_bewteen
ax.axvline(data[i][best_fit_indx], c=colors[2], ls='--')
for i in range(size):
for j in range(size):
if i <= j:
continue
ax = axs[i][j]
h, xbins, ybins = np.histogram2d(data[j], data[i], bins=50)
if plot_hist:
ax.pcolor(xbins, ybins, h.T, cmap='Greys')
dtk.quick_contour(xbins,
ybins,
h,
ax=ax,
levels=(0.68, 0.87),
colors=colors[1],
label=False,
smoothen=False,
bins_edges=True)
ax.axvline(data[j][best_fit_indx], c=colors[2], ls='--')
ax.axhline(data[i][best_fit_indx], c=colors[2], ls='--')
plt.sca(ax)
plt.xticks(rotation=45)
def velDisp_mass_rel_z0(sod_file):
'''
Function to gather mass and dark matter velocity dispersion data columns from
sodproperties halo files, scale masses by redshift, and return a .csv of the data
:param sod_file: path to a sodproperties file
:return: path to .csv file with mass and DM velocity dispersion columns
'''
masses = dtk.gio_read(sod_file, 'sod_halo_mass')
vDisp = dtk.gio_read(sod_file, 'sod_halo_vel_disp')
h_1 = 1 / (cosmo.h)
mask = (masses * h_1) >= 1e14
masses_big = masses[mask] * h_1
vDisp_big = vDisp[mask]
today = str(datetime.date.today())
with open('velDisp_vs_SodMass_{}.csv'.format(today), 'wb') as output:
writer = csv.writer(output,
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
for row in zip(masses_big, vDisp_big):
writer.writerow(list(row))
output.flush()
return os.path.abspath(output.name)
def test_core_catalog(core_loc, rank_num):
print(core_loc)
dtk.gio_inspect(core_loc)
print("loading host htag")
host_htag = dtk.gio_read(core_loc, 'fof_halo_tag', rank_num)
print("loading infall step")
infall_step = dtk.gio_read(core_loc, 'infall_step', rank_num)
print("loading infall htag")
infall_htag = dtk.gio_read(core_loc, 'infall_fof_halo_tag', rank_num)
print("loading infall mass")
infall_mass = dtk.gio_read(core_loc, 'infall_mass', rank_num)
print("loading central")
core_central = dtk.gio_read(core_loc, 'central', rank_num)
mi = np.argmax(infall_mass)
print(infall_mass[mi], np.log10(infall_mass[mi]))
htag = host_htag[mi]
slct = host_htag == htag
host_htag = host_htag[slct]
infall_htag = infall_htag[slct]
infall_step = infall_step[slct]
infall_mass = infall_mass[slct]
core_central = core_central[slct]
for i in range(0, np.sum(slct)):
if infall_step[i] == 401:
print(host_htag[i], infall_step[i], core_central[i])
def test(fname):
dtk.gio_inspect(fname)
centrals = dtk.gio_read(fname, 'central')
infall_step = dtk.gio_read(fname, 'infall_step')
print(centrals)
print(np.unique(centrals))
print(np.sum(centrals), len(centrals),
np.float(np.sum(centrals)) / len(centrals))
def load_core_data(core_file, step):
print "Loading core step:", step
fname = core_file.replace("${step}", str(step))
core_htag.append(dtk.gio_read(fname, "fof_halo_tag"))
core_x.append(dtk.gio_read(fname, "x"))
core_y.append(dtk.gio_read(fname, "y"))
core_z.append(dtk.gio_read(fname, "z"))
core_size.append(dtk.gio_read(fname, "radius"))
def load(self, fof_catalog_file, s):
print "Loading fof catalog:", s
fname = fof_catalog_file.replace("${step}", str(s))
self.steps.append(s)
htags = dtk.gio_read(fname, "fof_halo_tag")
mass = dtk.gio_read(fname, "fof_halo_mass")
indx = np.argsort(htags)
self.fof_htags.append(htags[indx])
self.fof_mass.append(mass[indx])
def load_sod(self, sod_loc, sod_hdf5):
if sod_hdf5:
hfile = h5py.File(sod_loc, 'r')
self.rad = hfile['sod_halo_radius_r200m'].value
self.mass = hfile['sod_halo_mass_m200m'].value
self.rad_c = hfile['sod_halo_radius_r200m'].value
self.mass_c = hfile['sod_halo_radius_r200c'].value
else:
self.rad = dtk.gio_read(sod_loc, 'sod_halo_radius')
self.mass = dtk.gio_read(sod_loc, 'sod_halo_mass')
def load(self, sod_file, s):
print "Loading sod catalog:", s
fname = sod_file.replace("${num}", str(s))
self.core_steps.append(s)
htags = dtk.gio_read(fname, "fof_halo_tag")
radius = dtk.gio_read(fname, "sod_radius")
mass = dtk.gio_read(fname, "sod_mass")
indx = np.arg_sort(htags)
self.htag.append(htags[indx])
self.sod_mass.append(mass[indx])
self.sod_radius.append(radius[indx])
def load_cores(core_fname, step):
core_fname = core_fname.replace("${step}", str(step))
dtk.gio_inspect(core_fname)
result = {
# "x": dtk.gio_read(core_fname, "x"),
# "y": dtk.gio_read(core_fname, "y"),
# "z": dtk.gio_read(core_fname, "z"),
# "r": dtk.gio_read(core_fname, "r"),
"infall_mass": dtk.gio_read(core_fname, "infall_mass"),
"radius": dtk.gio_read(core_fname, "radius"),
}
return result
def read_gio(self,verbose=False):
if(verbose):
print("from file",self.file_source_)
for step in self.step_data.keys():
if(verbose):
print("\treading in step: ",step)
file_name = self.file_source_.replace(self.step_string_,"%d"%step)
for name in self.data_names:
if(verbose):
print("\treading in ",name)
data = dtk.gio_read(file_name,name)
self.step_data[step][name] = data
def load_gio(file_loc, step):
start = time.time()
print "\treading in gio..."
file_step_loc = file_loc.replace("${step}", str(step))
dic = {}
dic['nodeIndex'] = dtk.gio_read(file_step_loc, 'nodeIndex')
dic['hostIndex'] = dtk.gio_read(file_step_loc, 'hostIndex')
dic['hostHaloTag'] = dtk.gio_read(file_step_loc, "host_halo_tag")
dic['hostHaloMass'] = dtk.gio_read(file_step_loc, 'host_halo_mass') / h
dic['infallHaloTag'] = dtk.gio_read(file_step_loc, 'infall_halo_tag')
dic['infallHaloMass'] = dtk.gio_read(file_step_loc, 'infall_halo_mass') / h
dic['step'] = np.ones(dic['nodeIndex'].size, dtype='i4') * step
dic['placementType'] = dtk.gio_read(file_step_loc, 'positionType')
print "\t\tdone. time:", time.time() - start
return dic
def load_halo_cat(sod_location, h_scaling):
print("loading the catalog")
print("H_scaling: ", h_scaling)
print("mass factor: ", 1/h_scaling)
cat = {}
dtk.gio_inspect(sod_location)
cat['fof_halo_tag'] = dtk.gio_read(sod_location, 'fof_halo_tag');
cat['fof_halo_center_x'] = dtk.gio_read(sod_location, 'fof_halo_center_x');
cat['fof_halo_center_y'] = dtk.gio_read(sod_location, 'fof_halo_center_y');
cat['fof_halo_center_z'] = dtk.gio_read(sod_location, 'fof_halo_center_z');
cat['sod_halo_mass'] = dtk.gio_read(sod_location, 'sod_halo_mass');
cat['sod_halo_radius'] = dtk.gio_read(sod_location, 'sod_halo_radius');
cat['sod_halo_cdelta'] = dtk.gio_read(sod_location, 'sod_halo_cdelta');
slct = cat['sod_halo_cdelta'] < 1
cat['sod_halo_cdelta'][slct] = 5.75
slct = cat['sod_halo_cdelta'] > 20
cat['sod_halo_cdelta'][slct] = 20
print("done loading")
return cat
#!/usr/bin/env python2.7
import numpy as np
import matplotlib as plt
import dtk
import os.path
file_loc = '/home/dkorytov/data/AlphaQ/core_catalog3/01_12_17.AlphaQ.${step}.coreproperties'
steps = np.arange(0, 500, dtype=int)
for step in steps:
file_loc2 = file_loc.replace('${step}', str(step))
if (os.path.isfile(file_loc2)):
infall_step = dtk.gio_read(file_loc2, 'infall_step')
slct = infall_step != step
print "\t", step
print "\t\t", np.sum(slct)
def load_prop(fname, var, cat):
cat[var] = dtk.gio_read(fname, var)
#!/usr/bin/env python2.7
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.colors as clr
import dtk
import sys
if __name__ == "__main__":
param = dtk.Param(sys.argv[1])
steps = param.get_int_list("steps")
core_loc = param.get_string("core_loc")
for s in steps:
print "step: ", s
cfname = core_loc.replace("${step}", str(s))
radius = dtk.gio_read(cfname, "radius")
infall_mass = dtk.gio_read(cfname, "infall_mass")
infall_step = dtk.gio_read(cfname, "infall_step")
#!/usr/bin/env python2.7 import numpy as np import matplotlib.pyplot as plt import matplotlib.colors as clrs import h5py import sys import dtk from util import * #param = dtk.Param(sys.argv[1]) output_gio_pos = "output/gal_499_pos.gio" output_gio = "output/gal_499.gio" gal_id = dtk.gio_read(output_gio, "nodeIndex") gal_x = dtk.gio_read(output_gio, "x") gal_y = dtk.gio_read(output_gio, "y") gal_z = dtk.gio_read(output_gio, "z") mask = 0x000FFFFF00000000 mask2 = 0x0FF0000000000000 print mask, mask2, gal_id.dtype gal_id2 = (gal_id & mask) >> 32 gal_id3 = (gal_id & mask2) >> 52 print gal_id2[:10] print gal_id3[:10] target = 2247296544770364691 slct = gal_id == target print gal_id[:10] print gal_id.size print np.sum(slct)
param_cnt += 1
def autocorr(x):
x2 = x - np.average(x)
x3 = x2 / np.sqrt(np.sum(x2 * x2))
result = np.correlate(x3, x3, mode='full')
return (np.arange(0, x.size - 1), result[x.size:])
mcmc_loc = "output/" + param.file_name + "/mcmc.gio"
#mcmc_loc = '/home/dkorytov/tmp/mcmc.gio'
#mcmc_loc = "jupiter_result/mcmc.gio"
print "loading mi"
mcmc_m_i = dtk.gio_read(mcmc_loc, "mcmc_mass_infall")
print "loading rd"
mcmc_r_d = dtk.gio_read(mcmc_loc, "mcmc_r_disrupt")
print "loading id"
mcmc_id = dtk.gio_read(mcmc_loc, "mcmc_walker_id")
print "loading ws"
mcmc_step = dtk.gio_read(mcmc_loc, "mcmc_walker_step")
print "loading val"
mcmc_val = dtk.gio_read(mcmc_loc, "mcmc_value")
hfile = h5py.File("output/" + param.file_name + "/fit_core_params.hdf5")
fitted_m_infall = hfile['m_infall']
fitted_r_disrupt = hfile['r_disrupt']
def stats(data):
mean = np.average(data)
def load_gio(file_loc,step,box=False):
start = time.time()
print "\treading in gio..."
file_step_loc = file_loc.replace("${step}",str(step))
dic = {}
dic['nodeIndex'] = dtk.gio_read(file_step_loc,'nodeIndex')
dic['hostIndex'] = dtk.gio_read(file_step_loc,'hostIndex')
dic['hostHaloTag'] = dtk.gio_read(file_step_loc,"host_halo_tag")
dic['hostHaloMass']= dtk.gio_read(file_step_loc,'host_halo_mass')/h
dic['infallHaloTag']=dtk.gio_read(file_step_loc,'infall_halo_tag')
dic['infallHaloMass']=dtk.gio_read(file_step_loc,'infall_halo_mass')/h
dic['step']=np.ones(dic['nodeIndex'].size,dtype='i4')*step
dic['placementType']=dtk.gio_read(file_step_loc,'positionType')
if box:
dic['x'] = dtk.gio_read(gio_loc.replace('${step}',str(step)),'x')
dic['y'] = dtk.gio_read(gio_loc.replace('${step}',str(step)),'y')
dic['z'] = dtk.gio_read(gio_loc.replace('${step}',str(step)),'z')
dic['vx'] = dtk.gio_read(gio_loc.replace('${step}',str(step)),'vx')
dic['vy'] = dtk.gio_read(gio_loc.replace('${step}',str(step)),'vy')
dic['vz'] = dtk.gio_read(gio_loc.replace('${step}',str(step)),'vz')
dic['redshift'] = np.ones(pos_dict['nodeIndex'].size)*stepz.get_z(step)
print "\t\tdone. time:",time.time()-start
return dic
halo_egr3_y = np.zeros_like(steps, dtype='float')
halo_egr3_z = np.zeros_like(steps, dtype='float')
print steps
steps_unique = np.unique(steps)
print "loading in halo shapes and extra halo info"
for step in steps_unique:
print "working on step: ", step
slct_step = steps == step
print step, sod_loc
sod_step_loc = sod_loc.replace('${step}', str(step))
print sod_step_loc
halo_shape_step_loc = halo_shape_loc.replace('${step}', str(step))
halo_shape_red_step_loc = halo_shape_red_loc.replace('${step}', str(step))
sod_cat_htag = dtk.gio_read(sod_step_loc, 'fof_halo_tag')
sod_cat_mass = dtk.gio_read(sod_step_loc, 'sod_halo_mass')
srt = np.argsort(sod_cat_htag)
indx = dtk.search_sorted(sod_cat_htag, htag_real[slct_step], sorter=srt)
slct_indx = indx != -1
slct = slct_step
slct[slct_step] = slct_indx
#Assign so masses to the ones that have it
so_mass[slct] = sod_cat_mass[indx[slct_indx]]
#Assign a dumby value to those that don't have it
#slct[slct_step]=(slct_indx==0)
#so_mass[slct] = -1.0
eg_cat_htag = dtk.gio_read(halo_shape_step_loc, 'halo_id')
eg_cat_eg1 = dtk.gio_read(halo_shape_step_loc, 'eval1')
eg_cat_eg2 = dtk.gio_read(halo_shape_step_loc, 'eval2')
eg_cat_eg3 = dtk.gio_read(halo_shape_step_loc, 'eval3')
#!/usr/bin/env python
import numpy as np
import matplotlib.pyplot as plt
import dtk
print 'reading'
step = 487
#tags = dtk.gio_read('/media01_12_17.AlphaQ.'+str(step)+'.treenodes','fof_halo_tag')
#mass = dtk.gio_read('/home/dkorytov/tmp/01_12_17.AlphaQ.'+str(step)+'.treenodes','fof_halo_mass')
tags = dtk.gio_read(
'/media/luna1/rangel/AlphaQ/temp/01_12_17.AlphaQ.' + str(step) +
'.treenodes', 'fof_halo_tag')
mass = dtk.gio_read(
'/media/luna1/rangel/AlphaQ/temp/01_12_17.AlphaQ.' + str(step) +
'.treenodes', 'fof_halo_mass')
print tags
print np.min(tags), np.max(tags)
print 'done reading'
mass_bins = np.logspace(10, 16, 20)
mass_bins_avg = (mass_bins[:-1] + mass_bins[1:]) / 2.0
lim = 1024 * 1024 * 1024
fof_slct = tags < lim
frag_slct = tags > lim
print lim
print np.sum(fof_slct)
print np.sum(frag_slct)
fof, _ = np.histogram(mass[fof_slct], bins=mass_bins)
frag, _ = np.histogram(mass[frag_slct], bins=mass_bins)
tot = frag + fof
print 'done histing'
chain_avg = np.mean(data[slct])
B += (chain_avg - global_avg)**2
W = np.mean(ssq)
B = n / (m - 1.0) * B
var = (n - 1.0) / n * W + 1.0 / n * B
R = np.sqrt(var / W)
return R
if (__name__ == '__main__'):
print "Gelman Ruben Test"
param_file_name = sys.argv[1]
param = dtk.Param(param_file_name)
file_loc = "output/" + param_file_name + "/mcmc.gio"
print "loading data"
mcmc_id = dtk.gio_read(file_loc, "mcmc_walker_id")
mcmc_step = dtk.gio_read(file_loc, "mcmc_walker_step")
mcmc_mi = dtk.gio_read(file_loc, "mcmc_mass_infall")
mcmc_rd = dtk.gio_read(file_loc, "mcmc_r_disrupt")
print "data loaded"
slct_half = mcmc_step > np.max(mcmc_step) / 2.0
print "M_infall: ", gelman_ruben_test(mcmc_mi[slct_half],
mcmc_id[slct_half])
print "R_dispupt: ", gelman_ruben_test(mcmc_rd[slct_half],
mcmc_id[slct_half])
if (param.get_bool("fit_r_merger")):
mcmc_rm = dtk.gio_read(file_loc, "mcmc_r_merger")
print "R_merger: ", gelman_ruben_test(mcmc_mi[slct_half],
mcmc_id[slct_half])
if (param.get_bool("fit_r_fof")):
dtype='f4',
as_name='m0',
z_str=step2nan_str_z[step])
add_step(lc_dic, step)
df1 = pd.DataFrame(lc_dic)
gio_df = pd.DataFrame(load_gio(gio_loc, step))
print "\t", df1.shape, gio_df.shape, "merged into",
df = pd.merge(df1, gio_df, on=['nodeIndex', 'step'])
print df.shape
dfs.append(df)
print "\t done. ", time.time() - start
dfs = []
for step in gltcs_steps:
print "working on step: ", step
pos_dict = {}
pos_dict['x'] = dtk.gio_read(gio_loc.replace('${step}', str(step)), 'x')
pos_dict['y'] = dtk.gio_read(gio_loc.replace('${step}', str(step)), 'y')
pos_dict['z'] = dtk.gio_read(gio_loc.replace('${step}', str(step)), 'z')
pos_dict['vx'] = dtk.gio_read(gio_loc.replace('${step}', str(step)), 'vx')
pos_dict['vy'] = dtk.gio_read(gio_loc.replace('${step}', str(step)), 'vy')
pos_dict['vz'] = dtk.gio_read(gio_loc.replace('${step}', str(step)), 'vz')
pos_dict['nodeIndex'] = dtk.gio_read(gio_loc.replace('${step}', str(step)),
'nodeIndex')
pos_dict['step'] = np.ones(pos_dict['nodeIndex'].size, dtype=int) * step
pos_dict['redshift'] = np.ones(
pos_dict['nodeIndex'].size) * stepz.get_z(step)
df1 = pd.DataFrame(pos_dict)
gio_df = pd.DataFrame(load_gio(gio_loc, step))
df = pd.merge(df1, gio_df, on=['nodeIndex', 'step'])
dfs.append(df)
