def load_core_cat(core_loc): print("loading: ", core_loc) t1 = time.time() cat = {} if ".hdf5" in core_loc: hfile = h5py.File(core_loc, 'r') cat['infall_mass'] = hfile['m_peak'].value cat['radius'] = hfile['r_peak'].value cat['infall_step'] = hfile['infall_step'].value else: dtk.gio_inspect(core_loc) cat['infall_mass'] = dtk.gio_read(core_loc, 'infall_mass') cat['radius'] = dtk.gio_read(core_loc, 'radius') cat['infall_step'] = dtk.gio_read(core_loc, 'infall_step') print(t1 - time.time()) plt.figure() slct = cat['infall_step'] == 401 h, xbins, ybins = np.histogram2d( cat['infall_mass'][slct], cat['radius'][slct], bins=[np.logspace(10, 15, 100), np.logspace(-4, 0.0, 100)]) plt.pcolor(xbins, ybins, h.T, cmap='Blues', norm=clr.LogNorm()) plt.xlabel('Core M_infall [Msun/h]') plt.ylabel('Core Radius [Mpc/h]') plt.xscale('log') plt.yscale('log') return cat
def corner_plot_mcmc(labels, mcmc_loc, fig, axs, colors=['tab:blue', 'k', 'tab:red'], plot_hist=True, alpha=0.3): mcmc_m_i = dtk.gio_read(mcmc_loc, "mcmc_mass_infall") mcmc_r_d = dtk.gio_read(mcmc_loc, "mcmc_r_disrupt") mcmc_id = dtk.gio_read(mcmc_loc, "mcmc_walker_id") mcmc_step = dtk.gio_read(mcmc_loc, "mcmc_walker_step") mcmc_val = dtk.gio_read(mcmc_loc, "mcmc_value") size = len(labels) data = [] slct = mcmc_step > np.max(mcmc_step) / 2.0 data.append(np.log10(mcmc_m_i[slct])) data.append(mcmc_r_d[slct]) best_fit_indx = np.argmin(mcmc_val[slct]) #Diagonal Covariance for i in range(0, len(labels)): ax = axs[i][i] h, xbins = np.histogram(data[i], bins=50, density=True) ax.plot(dtk.bins_avg(xbins), h, c=colors[0]) # Calculate 1 simga limits b1, b2 = get_bounds_limits(h, xbins, 0) b2 += 0 b1 += 0 b1a = b1 - 1 ylim = ax.get_ylim() ax.fill_between(dtk.bins_avg(bbins[b1a:b2]), 0, h[b1:b2], lw=0.0, alpha=alpha, color=colors[0]) ax.set_xlim(np.min(xbins), np.max(xbins)) ax.set_ylim(ylim) #restore old ylims before fill_bewteen ax.axvline(data[i][best_fit_indx], c=colors[2], ls='--') for i in range(size): for j in range(size): if i <= j: continue ax = axs[i][j] h, xbins, ybins = np.histogram2d(data[j], data[i], bins=50) if plot_hist: ax.pcolor(xbins, ybins, h.T, cmap='Greys') dtk.quick_contour(xbins, ybins, h, ax=ax, levels=(0.68, 0.87), colors=colors[1], label=False, smoothen=False, bins_edges=True) ax.axvline(data[j][best_fit_indx], c=colors[2], ls='--') ax.axhline(data[i][best_fit_indx], c=colors[2], ls='--') plt.sca(ax) plt.xticks(rotation=45)
def velDisp_mass_rel_z0(sod_file): ''' Function to gather mass and dark matter velocity dispersion data columns from sodproperties halo files, scale masses by redshift, and return a .csv of the data :param sod_file: path to a sodproperties file :return: path to .csv file with mass and DM velocity dispersion columns ''' masses = dtk.gio_read(sod_file, 'sod_halo_mass') vDisp = dtk.gio_read(sod_file, 'sod_halo_vel_disp') h_1 = 1 / (cosmo.h) mask = (masses * h_1) >= 1e14 masses_big = masses[mask] * h_1 vDisp_big = vDisp[mask] today = str(datetime.date.today()) with open('velDisp_vs_SodMass_{}.csv'.format(today), 'wb') as output: writer = csv.writer(output, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL) for row in zip(masses_big, vDisp_big): writer.writerow(list(row)) output.flush() return os.path.abspath(output.name)
def test_core_catalog(core_loc, rank_num): print(core_loc) dtk.gio_inspect(core_loc) print("loading host htag") host_htag = dtk.gio_read(core_loc, 'fof_halo_tag', rank_num) print("loading infall step") infall_step = dtk.gio_read(core_loc, 'infall_step', rank_num) print("loading infall htag") infall_htag = dtk.gio_read(core_loc, 'infall_fof_halo_tag', rank_num) print("loading infall mass") infall_mass = dtk.gio_read(core_loc, 'infall_mass', rank_num) print("loading central") core_central = dtk.gio_read(core_loc, 'central', rank_num) mi = np.argmax(infall_mass) print(infall_mass[mi], np.log10(infall_mass[mi])) htag = host_htag[mi] slct = host_htag == htag host_htag = host_htag[slct] infall_htag = infall_htag[slct] infall_step = infall_step[slct] infall_mass = infall_mass[slct] core_central = core_central[slct] for i in range(0, np.sum(slct)): if infall_step[i] == 401: print(host_htag[i], infall_step[i], core_central[i])
def test(fname): dtk.gio_inspect(fname) centrals = dtk.gio_read(fname, 'central') infall_step = dtk.gio_read(fname, 'infall_step') print(centrals) print(np.unique(centrals)) print(np.sum(centrals), len(centrals), np.float(np.sum(centrals)) / len(centrals))
def load_core_data(core_file, step): print "Loading core step:", step fname = core_file.replace("${step}", str(step)) core_htag.append(dtk.gio_read(fname, "fof_halo_tag")) core_x.append(dtk.gio_read(fname, "x")) core_y.append(dtk.gio_read(fname, "y")) core_z.append(dtk.gio_read(fname, "z")) core_size.append(dtk.gio_read(fname, "radius"))
def load(self, fof_catalog_file, s): print "Loading fof catalog:", s fname = fof_catalog_file.replace("${step}", str(s)) self.steps.append(s) htags = dtk.gio_read(fname, "fof_halo_tag") mass = dtk.gio_read(fname, "fof_halo_mass") indx = np.argsort(htags) self.fof_htags.append(htags[indx]) self.fof_mass.append(mass[indx])
def load_sod(self, sod_loc, sod_hdf5): if sod_hdf5: hfile = h5py.File(sod_loc, 'r') self.rad = hfile['sod_halo_radius_r200m'].value self.mass = hfile['sod_halo_mass_m200m'].value self.rad_c = hfile['sod_halo_radius_r200m'].value self.mass_c = hfile['sod_halo_radius_r200c'].value else: self.rad = dtk.gio_read(sod_loc, 'sod_halo_radius') self.mass = dtk.gio_read(sod_loc, 'sod_halo_mass')
def load(self, sod_file, s): print "Loading sod catalog:", s fname = sod_file.replace("${num}", str(s)) self.core_steps.append(s) htags = dtk.gio_read(fname, "fof_halo_tag") radius = dtk.gio_read(fname, "sod_radius") mass = dtk.gio_read(fname, "sod_mass") indx = np.arg_sort(htags) self.htag.append(htags[indx]) self.sod_mass.append(mass[indx]) self.sod_radius.append(radius[indx])
def load_cores(core_fname, step): core_fname = core_fname.replace("${step}", str(step)) dtk.gio_inspect(core_fname) result = { # "x": dtk.gio_read(core_fname, "x"), # "y": dtk.gio_read(core_fname, "y"), # "z": dtk.gio_read(core_fname, "z"), # "r": dtk.gio_read(core_fname, "r"), "infall_mass": dtk.gio_read(core_fname, "infall_mass"), "radius": dtk.gio_read(core_fname, "radius"), } return result
def read_gio(self,verbose=False): if(verbose): print("from file",self.file_source_) for step in self.step_data.keys(): if(verbose): print("\treading in step: ",step) file_name = self.file_source_.replace(self.step_string_,"%d"%step) for name in self.data_names: if(verbose): print("\treading in ",name) data = dtk.gio_read(file_name,name) self.step_data[step][name] = data
def load_gio(file_loc, step): start = time.time() print "\treading in gio..." file_step_loc = file_loc.replace("${step}", str(step)) dic = {} dic['nodeIndex'] = dtk.gio_read(file_step_loc, 'nodeIndex') dic['hostIndex'] = dtk.gio_read(file_step_loc, 'hostIndex') dic['hostHaloTag'] = dtk.gio_read(file_step_loc, "host_halo_tag") dic['hostHaloMass'] = dtk.gio_read(file_step_loc, 'host_halo_mass') / h dic['infallHaloTag'] = dtk.gio_read(file_step_loc, 'infall_halo_tag') dic['infallHaloMass'] = dtk.gio_read(file_step_loc, 'infall_halo_mass') / h dic['step'] = np.ones(dic['nodeIndex'].size, dtype='i4') * step dic['placementType'] = dtk.gio_read(file_step_loc, 'positionType') print "\t\tdone. time:", time.time() - start return dic
def load_halo_cat(sod_location, h_scaling): print("loading the catalog") print("H_scaling: ", h_scaling) print("mass factor: ", 1/h_scaling) cat = {} dtk.gio_inspect(sod_location) cat['fof_halo_tag'] = dtk.gio_read(sod_location, 'fof_halo_tag'); cat['fof_halo_center_x'] = dtk.gio_read(sod_location, 'fof_halo_center_x'); cat['fof_halo_center_y'] = dtk.gio_read(sod_location, 'fof_halo_center_y'); cat['fof_halo_center_z'] = dtk.gio_read(sod_location, 'fof_halo_center_z'); cat['sod_halo_mass'] = dtk.gio_read(sod_location, 'sod_halo_mass'); cat['sod_halo_radius'] = dtk.gio_read(sod_location, 'sod_halo_radius'); cat['sod_halo_cdelta'] = dtk.gio_read(sod_location, 'sod_halo_cdelta'); slct = cat['sod_halo_cdelta'] < 1 cat['sod_halo_cdelta'][slct] = 5.75 slct = cat['sod_halo_cdelta'] > 20 cat['sod_halo_cdelta'][slct] = 20 print("done loading") return cat
#!/usr/bin/env python2.7 import numpy as np import matplotlib as plt import dtk import os.path file_loc = '/home/dkorytov/data/AlphaQ/core_catalog3/01_12_17.AlphaQ.${step}.coreproperties' steps = np.arange(0, 500, dtype=int) for step in steps: file_loc2 = file_loc.replace('${step}', str(step)) if (os.path.isfile(file_loc2)): infall_step = dtk.gio_read(file_loc2, 'infall_step') slct = infall_step != step print "\t", step print "\t\t", np.sum(slct)
def load_prop(fname, var, cat): cat[var] = dtk.gio_read(fname, var)
#!/usr/bin/env python2.7 import numpy as np import matplotlib.pyplot as plt import matplotlib.colors as clr import dtk import sys if __name__ == "__main__": param = dtk.Param(sys.argv[1]) steps = param.get_int_list("steps") core_loc = param.get_string("core_loc") for s in steps: print "step: ", s cfname = core_loc.replace("${step}", str(s)) radius = dtk.gio_read(cfname, "radius") infall_mass = dtk.gio_read(cfname, "infall_mass") infall_step = dtk.gio_read(cfname, "infall_step")
#!/usr/bin/env python2.7 import numpy as np import matplotlib.pyplot as plt import matplotlib.colors as clrs import h5py import sys import dtk from util import * #param = dtk.Param(sys.argv[1]) output_gio_pos = "output/gal_499_pos.gio" output_gio = "output/gal_499.gio" gal_id = dtk.gio_read(output_gio, "nodeIndex") gal_x = dtk.gio_read(output_gio, "x") gal_y = dtk.gio_read(output_gio, "y") gal_z = dtk.gio_read(output_gio, "z") mask = 0x000FFFFF00000000 mask2 = 0x0FF0000000000000 print mask, mask2, gal_id.dtype gal_id2 = (gal_id & mask) >> 32 gal_id3 = (gal_id & mask2) >> 52 print gal_id2[:10] print gal_id3[:10] target = 2247296544770364691 slct = gal_id == target print gal_id[:10] print gal_id.size print np.sum(slct)
param_cnt += 1 def autocorr(x): x2 = x - np.average(x) x3 = x2 / np.sqrt(np.sum(x2 * x2)) result = np.correlate(x3, x3, mode='full') return (np.arange(0, x.size - 1), result[x.size:]) mcmc_loc = "output/" + param.file_name + "/mcmc.gio" #mcmc_loc = '/home/dkorytov/tmp/mcmc.gio' #mcmc_loc = "jupiter_result/mcmc.gio" print "loading mi" mcmc_m_i = dtk.gio_read(mcmc_loc, "mcmc_mass_infall") print "loading rd" mcmc_r_d = dtk.gio_read(mcmc_loc, "mcmc_r_disrupt") print "loading id" mcmc_id = dtk.gio_read(mcmc_loc, "mcmc_walker_id") print "loading ws" mcmc_step = dtk.gio_read(mcmc_loc, "mcmc_walker_step") print "loading val" mcmc_val = dtk.gio_read(mcmc_loc, "mcmc_value") hfile = h5py.File("output/" + param.file_name + "/fit_core_params.hdf5") fitted_m_infall = hfile['m_infall'] fitted_r_disrupt = hfile['r_disrupt'] def stats(data): mean = np.average(data)
def load_gio(file_loc,step,box=False): start = time.time() print "\treading in gio..." file_step_loc = file_loc.replace("${step}",str(step)) dic = {} dic['nodeIndex'] = dtk.gio_read(file_step_loc,'nodeIndex') dic['hostIndex'] = dtk.gio_read(file_step_loc,'hostIndex') dic['hostHaloTag'] = dtk.gio_read(file_step_loc,"host_halo_tag") dic['hostHaloMass']= dtk.gio_read(file_step_loc,'host_halo_mass')/h dic['infallHaloTag']=dtk.gio_read(file_step_loc,'infall_halo_tag') dic['infallHaloMass']=dtk.gio_read(file_step_loc,'infall_halo_mass')/h dic['step']=np.ones(dic['nodeIndex'].size,dtype='i4')*step dic['placementType']=dtk.gio_read(file_step_loc,'positionType') if box: dic['x'] = dtk.gio_read(gio_loc.replace('${step}',str(step)),'x') dic['y'] = dtk.gio_read(gio_loc.replace('${step}',str(step)),'y') dic['z'] = dtk.gio_read(gio_loc.replace('${step}',str(step)),'z') dic['vx'] = dtk.gio_read(gio_loc.replace('${step}',str(step)),'vx') dic['vy'] = dtk.gio_read(gio_loc.replace('${step}',str(step)),'vy') dic['vz'] = dtk.gio_read(gio_loc.replace('${step}',str(step)),'vz') dic['redshift'] = np.ones(pos_dict['nodeIndex'].size)*stepz.get_z(step) print "\t\tdone. time:",time.time()-start return dic
halo_egr3_y = np.zeros_like(steps, dtype='float') halo_egr3_z = np.zeros_like(steps, dtype='float') print steps steps_unique = np.unique(steps) print "loading in halo shapes and extra halo info" for step in steps_unique: print "working on step: ", step slct_step = steps == step print step, sod_loc sod_step_loc = sod_loc.replace('${step}', str(step)) print sod_step_loc halo_shape_step_loc = halo_shape_loc.replace('${step}', str(step)) halo_shape_red_step_loc = halo_shape_red_loc.replace('${step}', str(step)) sod_cat_htag = dtk.gio_read(sod_step_loc, 'fof_halo_tag') sod_cat_mass = dtk.gio_read(sod_step_loc, 'sod_halo_mass') srt = np.argsort(sod_cat_htag) indx = dtk.search_sorted(sod_cat_htag, htag_real[slct_step], sorter=srt) slct_indx = indx != -1 slct = slct_step slct[slct_step] = slct_indx #Assign so masses to the ones that have it so_mass[slct] = sod_cat_mass[indx[slct_indx]] #Assign a dumby value to those that don't have it #slct[slct_step]=(slct_indx==0) #so_mass[slct] = -1.0 eg_cat_htag = dtk.gio_read(halo_shape_step_loc, 'halo_id') eg_cat_eg1 = dtk.gio_read(halo_shape_step_loc, 'eval1') eg_cat_eg2 = dtk.gio_read(halo_shape_step_loc, 'eval2') eg_cat_eg3 = dtk.gio_read(halo_shape_step_loc, 'eval3')
#!/usr/bin/env python import numpy as np import matplotlib.pyplot as plt import dtk print 'reading' step = 487 #tags = dtk.gio_read('/media01_12_17.AlphaQ.'+str(step)+'.treenodes','fof_halo_tag') #mass = dtk.gio_read('/home/dkorytov/tmp/01_12_17.AlphaQ.'+str(step)+'.treenodes','fof_halo_mass') tags = dtk.gio_read( '/media/luna1/rangel/AlphaQ/temp/01_12_17.AlphaQ.' + str(step) + '.treenodes', 'fof_halo_tag') mass = dtk.gio_read( '/media/luna1/rangel/AlphaQ/temp/01_12_17.AlphaQ.' + str(step) + '.treenodes', 'fof_halo_mass') print tags print np.min(tags), np.max(tags) print 'done reading' mass_bins = np.logspace(10, 16, 20) mass_bins_avg = (mass_bins[:-1] + mass_bins[1:]) / 2.0 lim = 1024 * 1024 * 1024 fof_slct = tags < lim frag_slct = tags > lim print lim print np.sum(fof_slct) print np.sum(frag_slct) fof, _ = np.histogram(mass[fof_slct], bins=mass_bins) frag, _ = np.histogram(mass[frag_slct], bins=mass_bins) tot = frag + fof print 'done histing'
chain_avg = np.mean(data[slct]) B += (chain_avg - global_avg)**2 W = np.mean(ssq) B = n / (m - 1.0) * B var = (n - 1.0) / n * W + 1.0 / n * B R = np.sqrt(var / W) return R if (__name__ == '__main__'): print "Gelman Ruben Test" param_file_name = sys.argv[1] param = dtk.Param(param_file_name) file_loc = "output/" + param_file_name + "/mcmc.gio" print "loading data" mcmc_id = dtk.gio_read(file_loc, "mcmc_walker_id") mcmc_step = dtk.gio_read(file_loc, "mcmc_walker_step") mcmc_mi = dtk.gio_read(file_loc, "mcmc_mass_infall") mcmc_rd = dtk.gio_read(file_loc, "mcmc_r_disrupt") print "data loaded" slct_half = mcmc_step > np.max(mcmc_step) / 2.0 print "M_infall: ", gelman_ruben_test(mcmc_mi[slct_half], mcmc_id[slct_half]) print "R_dispupt: ", gelman_ruben_test(mcmc_rd[slct_half], mcmc_id[slct_half]) if (param.get_bool("fit_r_merger")): mcmc_rm = dtk.gio_read(file_loc, "mcmc_r_merger") print "R_merger: ", gelman_ruben_test(mcmc_mi[slct_half], mcmc_id[slct_half]) if (param.get_bool("fit_r_fof")):
dtype='f4', as_name='m0', z_str=step2nan_str_z[step]) add_step(lc_dic, step) df1 = pd.DataFrame(lc_dic) gio_df = pd.DataFrame(load_gio(gio_loc, step)) print "\t", df1.shape, gio_df.shape, "merged into", df = pd.merge(df1, gio_df, on=['nodeIndex', 'step']) print df.shape dfs.append(df) print "\t done. ", time.time() - start dfs = [] for step in gltcs_steps: print "working on step: ", step pos_dict = {} pos_dict['x'] = dtk.gio_read(gio_loc.replace('${step}', str(step)), 'x') pos_dict['y'] = dtk.gio_read(gio_loc.replace('${step}', str(step)), 'y') pos_dict['z'] = dtk.gio_read(gio_loc.replace('${step}', str(step)), 'z') pos_dict['vx'] = dtk.gio_read(gio_loc.replace('${step}', str(step)), 'vx') pos_dict['vy'] = dtk.gio_read(gio_loc.replace('${step}', str(step)), 'vy') pos_dict['vz'] = dtk.gio_read(gio_loc.replace('${step}', str(step)), 'vz') pos_dict['nodeIndex'] = dtk.gio_read(gio_loc.replace('${step}', str(step)), 'nodeIndex') pos_dict['step'] = np.ones(pos_dict['nodeIndex'].size, dtype=int) * step pos_dict['redshift'] = np.ones( pos_dict['nodeIndex'].size) * stepz.get_z(step) df1 = pd.DataFrame(pos_dict) gio_df = pd.DataFrame(load_gio(gio_loc, step)) df = pd.merge(df1, gio_df, on=['nodeIndex', 'step']) dfs.append(df)