def check_unique_core_tags(file1): ''' Check that core tags and infall tree nodes are unique criteria: no repeating core tags or infall tree nodes inputs: file1 = core catalog file outputs: criteria (pass=0) display: Uniqueness of core tag and infall tree node ''' try: core_tag = gio.gio_read(file1, "core_tag") infall_tree_node_index = gio.gio_read(file1, "infall_tree_node_index") core_tag_unique = _is_unique_array(core_tag) tree_node_unique = _is_unique_array(infall_tree_node_index) if core_tag_unique: print("Core tag is unique") else: print("Core tag is not unique") if tree_node_unique: print("Infall tree node index is unique") else: print("Infall tree node index is not unique") if core_tag_unique & tree_node_unique: return 0 else: return 1 except: print("test failed")
def return_idx(): tag_new = gio.gio_read(file_new, 'fof_halo_tag') tag_old = gio.gio_read(file_old, 'fof_halo_tag') idx_new = np.argsort(tag_new) idx_old = np.argsort(tag_old) print((tag_new[idx_new] == tag_old[idx_old]).all()) return idx_new, idx_old
def check_positions(file1): ''' Check that the positions of the cores look sensible criteria: _check_nan_in_positions test passess for all position variables inputs: file1 = core catalog file outputs: criteria (pass=0) plot: 1d histogram of x,y,z values, 2d histogram of (x,y) values display: output of _check_nan_in_positions test ''' try: test = 0 x = gio.gio_read(file1, 'x').flatten() y = gio.gio_read(file1, 'y').flatten() z = gio.gio_read(file1, 'z').flatten() test += _check_nan_in_positions('x', x) test += _check_nan_in_positions('y', y) test += _check_nan_in_positions('z', z) if vis: plt.figure() plt.hist(x, bins=100, alpha=0.4) plt.hist(y, bins=100, alpha=0.4) plt.hist(z, bins=100, alpha=0.4) plt.xlabel('position (Mpc)') plt.figure() plt.hist2d(x, y, bins=100) plt.xlabel('x (Mpc)') plt.ylabel('y (Mpc)') plt.show() return test except: print("can't run position test")
def check_central(file1, step): ''' Check that all halos have a central core criteria: a core with infall step = current step exists for all fof halos inputs: file1 = core catalog file step = time step associated with core catalog file outputs: criteria (pass=0) display: Maximum number of cores in a halo Total number of fof halos in file ''' try: halo_tag = gio.gio_read(file1, 'fof_halo_tag') infall_step = gio.gio_read(file1, 'infall_step') unique_tags, tag_counts = np.unique(halo_tag, return_counts=True) print("Maximum number of cores in a halo is " + str(max(tag_counts))) print("Number of halos is " + str(len(unique_tags))) unique_tags_central, tag_counts_central = np.unique( halo_tag[infall_step == step], return_counts=True) criteria = (np.sort(unique_tags) == np.sort(unique_tags_central)).all() if criteria: print("All halos have a central core") return 0 else: print("Some halos are missing a central core") return 1 except: print("test failed")
def check_mt_files(file1, file_mt, step): ''' Check that the number of halos in the merger tree file is consistent with the number of central cores criteria: number of halos in MT file and number of central cores agree to within 10% inputs: file1 = core catalog file file_mt = merger tree file step = current time step outputs: criteria (pass=0) display: Number of halos in merger tree file Number of central cores in core file option: plot_missing if plot_missing, then print out the number of 'missing objects', check how many of these have zero-masses, and plot the positions of the first 10,000 objects. (note at step=499 this shows boundary issues, at higher redshifts this shows fragments) ''' try: plot_missing = True # useful to turn on if step = 499, so you can see if all missing objects are at mass resolution if (step != 499): print( "Warning, merger tree files should only match properly at step 499 (at higher redshifts N_mergertree > N_centralcores due to fragmentation). As you are looking at a higher redshift, expect a ~5% level of missing cores with a range of masses (including 0 mass fragments)." ) halo_tag = gio.gio_read(file_mt, 'fof_halo_tag') halo_mass = gio.gio_read(file_mt, 'fof_halo_mass') halo_tag_c = gio.gio_read(file1, 'fof_halo_tag') infall_step = gio.gio_read(file1, 'infall_step') halo_tag_c = halo_tag_c[infall_step == step] print("Number of halos in merger tree file = " + str(len(halo_tag))) print("Number of central cores in core file = " + str(len(halo_tag_c))) diff_tot = (len(halo_tag) - len(halo_tag_c) + 0.0) / len(halo_tag_c) * 100. criteria = (diff_tot < 10) if vis & plot_missing: print("Plotting missing cores - this may take a minute.") import random missing_cores = np.setdiff1d(halo_tag, halo_tag_c) print("Number of missing objects = " + str(len(missing_cores))) random.shuffle(missing_cores) masses = [] for i in range(min(1000, len(missing_cores))): #print(i) idx = np.where(halo_tag == missing_cores[i])[0][0] #print(idx) masses.append(halo_mass[idx]) masses = np.array(masses) if 0 in masses: print("Fraction of zero-mass objects = " + str(np.sum(masses == 0) / (len(masses) + 0.0))) plt.figure() plt.hist((masses[masses > 0] / 1.09e9), bins=100) plt.xlabel('non-zero masses (log10)') plt.show() if criteria: return 0 else: return 1 except: print("test_failed")
def read_mass_cdelta(): ''' read and mask the SOD mass and concentration ''' mask12, fof_old = make_mask() print("made mask") sod_old = gio.gio_read(file_old, 'sod_halo_mass')[mask12] sod_new = gio.gio_read(file_new, 'sod_halo_mass')[mask12] cdelta_old = gio.gio_read(file_old, 'sod_halo_cdelta')[mask12] cdelta_new = gio.gio_read(file_new, 'sod_halo_cdelta')[mask12] return fof_old, sod_old, sod_new, cdelta_old, cdelta_new
def check_tags(): ''' Double check ordering is the same from the tags ''' tag_old = gio.gio_read(file_old, 'fof_halo_tag') tag_new = gio.gio_read(file_new, 'fof_halo_tag') if (tag_old == tag_new).all(): print("ordering is the same") else: print("ordering has changed - double check this before continuing") return
def check_core_evolution(file1, file2): ''' Check that core tags present in an earlier timestep are retained in the later timestep criteria: less than 5% of the core tags present in the earlier timestep are missing in the later step inputs: file1 = core catalog file file2 = core catalog file at lower step number (higher redshift) outputs: criteria (pass=0) display: Percentage of cores present in earlier timestep missing in subsequent timestep option: plot_missing if plot_missing, then print out the number of 'missing objects', and plot the positions of the first 10,000 objects alongside a core density map for comparison. ''' try: plot_missing = False core_tag1 = gio.gio_read(file1, "core_tag") core_tag2 = gio.gio_read(file2, "core_tag") # file1 should be later in time (higher step number) than file2 missing_cores = np.setdiff1d(core_tag2, core_tag1, assume_unique=False) perc_missing = float(len(missing_cores)) / len(core_tag2) * 100 criteria = (perc_missing < 5) print( "Percentage of cores present in earlier timestep missing in subsequent timestep = " + str(perc_missing)) if vis & plot_missing: import random random.shuffle(missing_cores) print( "Plotting positions of 10,000 missing cores - this may take a minute" ) x = gio.gio_read(file2, 'x').flatten() y = gio.gio_read(file2, 'y').flatten() z = gio.gio_read(file2, 'z').flatten() xx = [] yy = [] zz = [] for i in range(10000): idx = np.where(core_tag2 == missing_cores[i])[0][0] xx.append(x[idx]) yy.append(y[idx]) zz.append(z[idx]) xx = np.array(xx) yy = np.array(yy) zz = np.array(zz) plt.hist2d(xx, yy, bins=100) plt.title('location of missing cores') plt.figure() plt.hist2d(x, y, bins=100) plt.title('location of all cores - for comparison') plt.show() if criteria: return 0 else: return 1 except: print("test_failed")
def main(): arrx = gio.gio_read(file_name, 'x', rank, size) arry = gio.gio_read(file_name, 'y', rank, size) i = 0 for elem in arrx[:,0]: if elem != 0: i +=1 perc= np.around(i/float(arrx.shape[0]), decimals=3) print i,"/",arrx.shape[0],"(",perc*100,"%)", " nonzero elements in rank ",rank plt.scatter(arrx, arry) plt.show()
def check_prop(var, idx_new, idx_old): var_new = gio.gio_read(file_new, var) var_old = gio.gio_read(file_old, var) ident = (var_new[idx_new] == var_old[idx_old]).all() if ident: print("values are identical for variable ", var) else: print("values are not identical for variable ", var) print(np.sum(var_new[idx_new] != var_old[idx_old])) print(np.max(var_new[idx_new] - var_old[idx_old])) print(np.min(var_new[idx_new] - var_old[idx_old])) print(np.max((var_new[idx_new] - var_old[idx_old]) / var_old[idx_old])) print(np.min((var_new[idx_new] - var_old[idx_old]) / var_old[idx_old])) return
def check_infall_timesteps(file1): ''' Check the distribution of infall steps criteria: minimum infall step > 0 (sanity check on time step assignment) inputs: file1 = core catalog file outputs: criteria (pass=0) plots: histogram of infall time step display: list of unique infall time steps ''' try: infall_step = gio.gio_read(file1, 'infall_step') criteria = (np.min(infall_step) > 0) print("List of infall steps:") print(np.unique(infall_step)) if criteria: print("Minimum infall step has positive value: " + str(np.min(infall_step))) if vis: plt.figure() plt.hist(infall_step, bins=100) plt.xlabel('infall step') plt.show() return 0 else: print("Minimum infall step not sensible: " + str(np.min(infall_step))) return 1 except: print("can't run infall timestep test") return 1
def check_infall_masses(file1): ''' Check how many particles make up the minimum infall mass, and plot the distribution criteria: minimum infall mass > 0 (this is effectively a sanity check on mass assignment) inputs: file1 = core catalog file outputs: criteria (pass=0) plots: histogram of log10(mass) ''' try: infall_mass = gio.gio_read(file1, 'infall_mass') criteria = (np.min(infall_mass) > 0) if criteria: print("Minimum infall mass has positive value: " + str(round(np.min(infall_mass) / 1.09086e9)) + " particles") if vis: plt.figure() plt.hist(np.log10(infall_mass), bins=100) plt.xlabel('log10(infall mass)') plt.yscale('log') plt.show() return 0 else: print("minimum infall mass not sensible: " + str(np.min(infall_mass))) return 1 except: print("can't run infall mass test") return 1
def read_in_lightcone(input_file, vars=['x', 'y', 'z', 'vx', 'vy', 'vz', 'id']): ''' read in lightcone as a dictionary ''' lightcone_object = {} for i in vars: lightcone_object[i] = gio.gio_read(input_file, i) return lightcone_object
def getData( dataFile, rank, size, *args ): #Takes a data file and a list of attributes as input, and returns an array whose columns are the attributes from the file (e.g. mass, position, etc) nObjects = gio.gio_read(dataFile, args[0], rank, size).shape[ 0] #gets number of objects in file by checking the shape of the numpy array returned from the first arg . data_array = np.empty( shape=(nObjects, len(args))) #one row per object, one column per arg names = [] formats = [] for index, arg in enumerate(args): arr = gio.gio_read(dataFile, arg, rank, size).flatten() data_array[:, index] = arr names.append(arg) formats.append(type(arr[0])) dtype = dict(names=names, formats=formats) new_array = np.core.records.fromarrays(data_array.transpose(), dtype) return new_array
def read_centers(): mask12, fof_old = make_mask() x_old = gio.gio_read(file_old, 'fof_halo_center_x')[mask12] y_old = gio.gio_read(file_old, 'fof_halo_center_y')[mask12] z_old = gio.gio_read(file_old, 'fof_halo_center_z')[mask12] x_new = gio.gio_read(file_new, 'fof_halo_center_x')[mask12] y_new = gio.gio_read(file_new, 'fof_halo_center_y')[mask12] z_new = gio.gio_read(file_new, 'fof_halo_center_z')[mask12] radius = gio.gio_read(file_old, 'sod_halo_radius')[mask12] return fof_old, x_old, y_old, z_old, x_new, y_new, z_new, radius
def check_heirarchy(file1): ''' Check that host core tag is within the same tree node index as the core tag. Check that infall step of the host is higher than infall step of the core ''' core_tag = gio.gio_read(file1, "core_tag") host_core_tag = gio.gio_read(file1, "host_core") vals, ar1, ar2 = np.intersect1d(core_tag, host_core_tag, assume_unique=False, return_indices=True) tni = gio.gio_read(file1, 'tree_node_index') infall_step = gio.gio_read(file1, 'infall_step') if (tni[ar1] == tni[ar2]).all() & (infall_step[ar1] > infall_step[ar2]).all(): return else: print(np.sum(tni[ar1] != tni[ar2]), np.sum(infall_step[ar1] <= infall_step[ar2])) return 1
def norm_mass(file1): ''' Checking that core abundance increases with host halo mass as a power law with index unity ''' infall_mass = gio.gio_read(file1, 'infall_mass').flatten() infall_step = gio.gio_read(file1, 'infall_step').flatten() host_tag = gio.gio_read(file1, 'fof_halo_tag').flatten() host_mass = infall_mass infall_mass = infall_mass[host_mass >= 1.e12] host_tag = host_tag[host_mass >= 1.e12] infall_step = infall_step[host_mass >= 1.e12] host_tag_u = host_tag[infall_step == 499] host_mass = host_mass[host_mass >= 1.e12][infall_step == 499] dict_values = dict(zip(host_tag_u, host_mass)) def apply_dict(i): return dict_values[i] infall_mass = gio.gio_read(file1, 'infall_mass').flatten() infall_step = gio.gio_read(file1, 'infall_step').flatten() host_tag = gio.gio_read(file1, 'fof_halo_tag').flatten() mask = (infall_mass > 1.e12) & (infall_step != 499) tags, counts = np.unique(host_tag[mask], return_counts=True) mass_tot = np.array(map(apply_dict, tags)) mean_sats, edges, nums = bs(np.log10(mass_tot), counts - 1., statistic='mean', bins=100) bin_mean = (edges[1:] - edges[:-1]) / 2. + edges[:-1] idx_13 = np.where( np.abs(bin_mean - 13.5) == np.min(np.abs(bin_mean - 13.5))) idx_15 = np.where( np.abs(bin_mean - 14.5) == np.min(np.abs(bin_mean - 14.5))) diff_val = (np.log10(mean_sats[idx_15]) - np.log10(mean_sats[idx_13])) criteria = (diff_val < 1.2) & (diff_val > 0.8) print((np.log10(mean_sats[idx_15]) - np.log10(mean_sats[idx_13]))) if vis: plt.figure() plt.plot(bin_mean, mean_sats) plt.ylabel('N_cores (>1.e12 mass)') plt.yscale('log') plt.xlabel('log10(host mass)') plt.xlim([12.5, 15]) plt.show() if criteria: print( "power law index of core abundance as a function of host halo mass has an index of nearly unity" ) return 0 else: print("core abundance as a function of host halo mass needs checking") return 1
def read_outerrim(snapshot, redshift, file_number, path=""): """ Reads halo position, velocity and mass from OuterRim file Args: snapshot: OuterRim snapshot number redshift: redshift of OuterRim snapshot file_number: file number of OuterRim file to read path: path of directory where OuterRim files are stored Returns: pos: array of comoving positions in units [Mpc/h] vel: array of proper velocities in units [km/s] mass: array of masses in units [Msun/h] """ halo_cat = path + "02_17_2016.OuterRim.%i.fofproperties#%i" % (snapshot, file_number) # read halo mass mass = genericio.gio_read(halo_cat, 'fof_halo_mass')[0] # read comoving positions x = genericio.gio_read(halo_cat, 'fof_halo_center_x')[0] y = genericio.gio_read(halo_cat, 'fof_halo_center_y')[0] z = genericio.gio_read(halo_cat, 'fof_halo_center_z')[0] pos = np.array([x, y, z]).transpose() del x, y, z # read comoving velocities vx = genericio.gio_read(halo_cat, 'fof_halo_mean_vx')[0] vy = genericio.gio_read(halo_cat, 'fof_halo_mean_vy')[0] vz = genericio.gio_read(halo_cat, 'fof_halo_mean_vz')[0] vel = np.array([vx, vy, vz]).transpose() del vx, vy, vz # convert velocities from comoving to proper vel = vel / (1. + redshift) return pos, vel, mass
def create_core_catalog_mevolved(virialFlag=False, writeOutputFlag=True, useLocalHost=False): """ Appends mevolved to core catalog and saves output in HDF5. Works by computing mevolved for step+1 at each step and saving that in memory. """ if writeOutputFlag: print 'Reading data from {} and writing output to {}'.format( cc_data_dir, cc_output_dir) cc = {} cc_prev = {} for step in tqdm(steps): # Read in cc for step cc = {v: gio.gio_read(fname_cc(step, 'input'), v)[0] for v in vars_cc} if virialFlag: # Convert all mass to virial cc['infall_mass'] = SHMLM.m_vir(cc['infall_mass'], step) satellites_mask = cc['central'] == 0 centrals_mask = cc['central'] == 1 # assert np.sum(satellites_mask) + np.sum(centrals_mask) == len(cc['infall_mass']), 'central flag not 0 or 1.' numSatellites = np.sum(satellites_mask) # Verify there are no satellites at first step if step == steps[0]: assert numSatellites == 0, 'Satellites found at first step.' # Add column for m_evolved and initialize to 0 for A in A_arr: for zeta in zeta_arr: cc[m_evolved_col(A, zeta)] = np.zeros_like(cc['infall_mass']) ''' #cc['mergedCoreTag'] = np.zeros_like(cc['core_tag']) #cc['mergedCoreStep'] = np.zeros_like(cc['infall_step']) cc['phaseSpaceMerged'] = np.zeros_like(cc['central']) # wasInFragment: persistent flag that is False by default and becomes True when core is inside a fragment halo cc['wasInFragment'] = cc['fof_halo_tag']<0 if step != steps[0]: _, i1, i2 = np.intersect1d( cc['core_tag'], cc_prev['core_tag'], return_indices=True ) cc['wasInFragment'][i1] = np.logical_or( cc['wasInFragment'][i1], cc_prev['wasInFragment'][i2] ) #cc['mergedCoreTag'][i1] = cc_prev['mergedCoreTag'][i2] #cc['mergedCoreStep'][i1] = cc_prev['mergedCoreStep'][i2] cc['phaseSpaceMerged'][i1] = cc_prev['phaseSpaceMerged'][i2] ''' # If there are satellites (not applicable for first step) if numSatellites != 0: # Match satellites to prev step cores using core tag. vals, idx1, idx2 = np.intersect1d(cc['core_tag'][satellites_mask], cc_prev['core_tag'], return_indices=True) # assert len(vals) == len(cc['core_tag'][satellites_mask]), 'All cores from prev step did not carry over.' # assert len(cc['core_tag'][satellites_mask]) == len(np.unique(cc['core_tag'][satellites_mask])), 'Satellite core tags not unique for this time step.' # Initialize M array (corresponds with cc[satellites_mask]) to be host tree node mass of each satellite idx_m21 = many_to_one(cc['tree_node_index'][satellites_mask], cc['tree_node_index'][centrals_mask]) M, X, Y, Z = (cc[k][centrals_mask][idx_m21] for k in ['infall_mass', 'x', 'y', 'z']) for A in A_arr: for zeta in zeta_arr: # Set m_evolved of all satellites that have core tag match on prev step to next_m_evolved of prev step. # DOUBLE MASK ASSIGNMENT: cc['m_evolved'][satellites_mask][idx1] = cc_prev['next_m_evolved'][idx2] cc[m_evolved_col(A, zeta)][np.flatnonzero(satellites_mask) [idx1]] = cc_prev[m_evolved_col( A, zeta, next=True)][idx2] # Initialize m array (corresponds with cc[satellites_mask]) to be either infall_mass (step after infall) or m_evolved (subsequent steps). initMask = cc[m_evolved_col(A, zeta)][satellites_mask] == 0 minfall = cc['infall_mass'][satellites_mask][initMask] cc[m_evolved_col(A, zeta)][np.flatnonzero( satellites_mask)[initMask]] = SHMLM.m_evolved( m0=minfall, M0=M[initMask], step=step, step_prev=steps[steps.index(step) - 1], A=A, zeta=zeta, dtFactorFlag=True) # m = (cc[m_evolved_col(A, zeta)][satellites_mask] == 0)*cc['infall_mass'][satellites_mask] + (cc[m_evolved_col(A, zeta)][satellites_mask] != 0)*cc[m_evolved_col(A, zeta)][satellites_mask] # Set m_evolved of satellites with m_evolved=0 to infall mass. # cc[m_evolved_col(A, zeta)][satellites_mask] = m ''' x, y, z = cc['x'].copy(), cc['y'].copy(), cc['z'].copy() x[satellites_mask] = SHMLM.periodic_bcs(cc['x'][satellites_mask], X, SHMLM.BOXSIZE) y[satellites_mask] = SHMLM.periodic_bcs(cc['y'][satellites_mask], Y, SHMLM.BOXSIZE) z[satellites_mask] = SHMLM.periodic_bcs(cc['z'][satellites_mask], Z, SHMLM.BOXSIZE) nonMergeMask = np.flatnonzero(cc['phaseSpaceMerged']==0) c1, c2 = np.array( list(combinations(nonMergeMask, 2)) ).T #c1, c2 are indices relative of cc Delta_x = ((x[c1]-x[c2])**2 + (y[c1]-y[c2])**2 + (z[c1]-z[c2])**2)**0.5 Delta_v = ((cc['vx'][c1]-cc['vx'][c2])**2 + (cc['vy'][c1]-cc['vy'][c2])**2 + (cc['vz'][c1]-cc['vz'][c2])**2)**0.5 mass1bigger = cc['infall_mass'][c1] > cc['infall_mass'][c2] massbiggeridx = np.where(mass1bigger, c1, c2) masssmalleridx = np.where(mass1bigger, c2, c1) sigma_x = 3*cc['radius'][massbiggeridx] sigma_v = 3*cc['vel_disp'][massbiggeridx] mergeFound = ((Delta_x/sigma_x + Delta_v/sigma_v) < 2) cc['phaseSpaceMerged'][np.unique(masssmalleridx[mergeFound])] = 1 print "Step", step print "Merged pairs found: {} out of {} pairs ({}%)".format( np.sum(mergeFound), len(mergeFound), np.sum(mergeFound)/len(mergeFound)*100. ) print "Total number of cores marked as merged:", np.sum(cc['phaseSpaceMerged']==1) print "Number of central cores marked as merged:", np.sum(cc['phaseSpaceMerged'][centrals_mask]==1) cc['phaseSpaceMerged'][centrals_mask] = 0 ''' if writeOutputFlag: # Write output to disk h5_write_dict(fname_cc(step, 'output'), cc, 'coredata') # Compute m_evolved of satellites according to SHMLModel for NEXT time step and save as cc_prev['next_m_evolved'] in memory. # Mass loss assumed to begin at step AFTER infall detected. if step != steps[-1]: # cc_prev = { k:cc[k].copy() for k in ['core_tag', 'wasInFragment', 'phaseSpaceMerged'] } cc_prev = {'core_tag': cc['core_tag'].copy()} if numSatellites != 0: localhost_m21 = many_to_one(cc['host_core'][satellites_mask], cc['core_tag']) for A in A_arr: for zeta in zeta_arr: cc_prev[m_evolved_col(A, zeta, next=True)] = np.zeros_like( cc['infall_mass']) if numSatellites != 0: # If there are satellites (not applicable for first step) m = cc[m_evolved_col(A, zeta)][satellites_mask] if useLocalHost: Mlocal = cc[m_evolved_col(A, zeta)][localhost_m21] M_A_zeta = (Mlocal == 0) * M + (Mlocal != 0) * Mlocal cc_prev[m_evolved_col( A, zeta, next=True)][satellites_mask] = SHMLM.m_evolved( m0=m, M0=M_A_zeta, step=steps[steps.index(step) + 1], step_prev=step, A=A, zeta=zeta) else: cc_prev[m_evolved_col( A, zeta, next=True)][satellites_mask] = SHMLM.m_evolved( m0=m, M0=M, step=steps[steps.index(step) + 1], step_prev=step, A=A, zeta=zeta) return cc
import sys import numpy as np import pandas as pd sys.path.append("/home/pascal/projects/VizAly-Vis_IO/genericio/python") import genericio as gio ## Input Section input_file_name = "/bigData/Halos/b0168/m001-499.sodproperties" query = "['fof_halo_count'] > 400000" display_values = [ 'fof_halo_tag', 'fof_halo_count', 'fof_halo_center_x', 'fof_halo_center_y', 'fof_halo_center_y' ] ##### Script Starts ##### num_vars = gio.gio_get_num_variables(input_file_name) # Read in data for each variable df = pd.DataFrame() for i in range(num_vars): var_name = gio.gio_get_variable(input_file_name, i) data = gio.gio_read(input_file_name, var_name) df.insert(i, var_name, data) query_string = "df" + query row_selection = eval(query_string) col_selection = display_values print(df.loc[row_selection, col_selection])
# permission. # # ***************************************************************************** # # DISCLAIMER # THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND. NEITHER THE # UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR # UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, # EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE # ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS, # PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE # PRIVATELY OWNED RIGHTS. # # ***************************************************************************** import sys import numpy as np import genericio as gio name = sys.argv[1] gio.gio_inspect(name) if gio.gio_has_variable(name, "x"): x = gio.gio_read(name, "x") y = gio.gio_read(name, "y") z = gio.gio_read(name, "z") print (np.column_stack((x, y, z))) else: pos = gio.gio_read(name, "pos") print (pos)
# File nroot = halodir + 'HaloCatalog/STEP' + str(istep) files = glob.glob(nroot + '/*' + str(istep) + '*.fofproperties#' + ifile) if (len(files) > 1): print files print 'STOP: More than one file for ifile ', ifile sys.exit() else: infile = files[0] gio.gio_inspect(infile) print '----------------------------' print 'Output: ', outfile # Read the file count = gio.gio_read(infile, "fof_halo_count") tag = gio.gio_read(infile, "fof_halo_tag") mass = gio.gio_read(infile, "fof_halo_mass") xc = gio.gio_read(infile, "fof_halo_center_x") yc = gio.gio_read(infile, "fof_halo_center_y") zc = gio.gio_read(infile, "fof_halo_center_z") xm = gio.gio_read(infile, "fof_halo_mean_x") ym = gio.gio_read(infile, "fof_halo_mean_y") zm = gio.gio_read(infile, "fof_halo_mean_z") vx = gio.gio_read(infile, "fof_halo_mean_vx") vy = gio.gio_read(infile, "fof_halo_mean_vy") vz = gio.gio_read(infile, "fof_halo_mean_vz") # Output size = len(vz)
def dndmu(file1): print file1 gio.gio_inspect(file1) infall_mass = gio.gio_read(file1, 'infall_mass').flatten() infall_step = gio.gio_read(file1, 'infall_step').flatten() host_tag = gio.gio_read(file1, 'fof_halo_tag').flatten() host_mass = infall_mass infall_mass = infall_mass[host_mass >= 1.e12] host_tag = host_tag[host_mass >= 1.e12] infall_step = infall_step[host_mass >= 1.e12] host_tag_u = host_tag[infall_step == 499] host_mass = host_mass[host_mass >= 1.e12][infall_step == 499] dict_values = dict(zip(host_tag_u, host_mass)) infall_mass = infall_mass[infall_step != 499] host_tag = host_tag[infall_step != 499] def apply_dict(i): return dict_values[i] print(host_tag[0]) print(dict_values[host_tag[0]]) print(apply_dict(host_tag[0])) mass_tot = np.array(map(apply_dict, host_tag)) ratio_tot = infall_mass / mass_tot print(len(mass_tot)) print(len(ratio_tot)) print(mass_tot[:100]) ''' print(len(host_tag_u)) ratio_tot=np.array([]).astype(float) mass_tot=np.array([]).astype(float) for i in range(len(host_tag_u)): if i%1000==0: print(i) core_masses = infall_mass[(host_tag==host_tag_u[i])&(infall_step!=499)] ratio = core_masses/host_mass[i] mass = ratio*0.0 + host_mass[i] ratio_tot = np.concatenate((ratio_tot,ratio)) mass_tot = np.concatenate((mass_tot,mass)) print(len(host_mass)) ''' bin12 = (mass_tot > 1.e12) & (mass_tot < 5.e12) bin13 = (mass_tot > 1.e13) & (mass_tot < 5.e13) bin14 = (mass_tot > 1.e14) & (mass_tot < 5.e14) bin125 = (mass_tot > 5.e12) & (mass_tot < 1.e13) bin135 = (mass_tot > 5.e13) & (mass_tot < 1.e14) bin145 = (mass_tot > 5.e14) & (mass_tot < 1.e15) print(np.sum(bin12), np.sum(bin125), np.sum(bin13), np.sum(bin135), np.sum(bin14), np.sum(bin145)) bins = np.logspace(-2, 0, 100) hist12, edges = np.histogram( ratio_tot[bin12], normed=True, bins=bins) #np.logspace(-1,0,100))#np.linspace(0.0,1.0,100)) hist13, edges = np.histogram( ratio_tot[bin13], normed=True, bins=bins) #np.logspace(-1,0,100))#linspace(0.0,1.0,100)) hist14, edges = np.histogram( ratio_tot[bin14], normed=False, bins=bins) #np.logspace(-1,0,100))#inspace(0.0,1.0,100)) hist125, edges = np.histogram( ratio_tot[bin125], normed=True, bins=bins) # np.logspace(-1,0,100))#np.linspace(0.0,1.0,100)) hist135, edges = np.histogram( ratio_tot[bin135], normed=False, bins=bins) #np.logspace(-1,0,100))#linspace(0.0,1.0,100)) hist145, edges = np.histogram( ratio_tot[bin145], normed=True, bins=bins) #np.logspace(-3,0,100))#inspace(0.0,1.0,100)) #plt.plot(edges[1:], hist12,label='bin 12') #plt.plot(edges[1:], hist13,label = 'bin 13') plt.plot(edges[1:], hist14, label='bin 14') #plt.plot(edges[1:], hist125,label='bin 12.5') plt.plot(edges[1:], hist135, label='bin 13.5') #plt.plot(edges[1:], hist145,label= 'bin 14.5') plt.yscale('log') plt.xscale('log') plt.legend() plt.xlabel('mass ratio') plt.ylabel('N_cores (normalized)') plt.show() bins = np.logspace(-0.5, 0, 100) hist12, edges = np.histogram( ratio_tot[bin12], normed=True, bins=bins) #np.logspace(-1,0,100))#np.linspace(0.0,1.0,100)) hist13, edges = np.histogram( ratio_tot[bin13], normed=True, bins=bins) #np.logspace(-1,0,100))#linspace(0.0,1.0,100)) hist14, edges = np.histogram( ratio_tot[bin14], normed=True, bins=bins) #np.logspace(-1,0,100))#inspace(0.0,1.0,100)) hist125, edges = np.histogram( ratio_tot[bin125], normed=True, bins=bins) # np.logspace(-1,0,100))#np.linspace(0.0,1.0,100)) hist135, edges = np.histogram( ratio_tot[bin135], normed=True, bins=bins) #np.logspace(-1,0,100))#linspace(0.0,1.0,100)) hist145, edges = np.histogram( ratio_tot[bin145], normed=True, bins=bins) #np.logspace(-3,0,100))#inspace(0.0,1.0,100)) plt.plot(edges[1:], hist12, label='bin 12') plt.plot(edges[1:], hist13, label='bin 13') # plt.plot(edges[1:], hist14,label= 'bin 14') plt.plot(edges[1:], hist125, label='bin 12.5') plt.plot(edges[1:], hist135, label='bin 13.5') #plt.plot(edges[1:], hist145,label= 'bin 14.5') plt.yscale('log') plt.xscale('log') plt.legend() plt.xlabel('mass ratio') plt.ylabel('N_cores (normalized)') plt.show() # except: # print('test_failed') return
def central_flip(file1, file2, step): ''' Check for flipping of centrals from one step to the next This test is in development, and may need to read in the accum files to see if there is truly missing substructures ''' try: x = gio.gio_read(file1, 'x').flatten() y = gio.gio_read(file1, 'y').flatten() z = gio.gio_read(file1, 'z').flatten() halo_tag = gio.gio_read(file1, 'fof_halo_tag').flatten() infall_step = gio.gio_read(file1, 'infall_step').flatten() infall_mass = gio.gio_read(file1, 'infall_mass').flatten() core_tag = gio.gio_read(file1, "core_tag").flatten() core_tag2 = gio.gio_read(file2, "core_tag").flatten() x2 = gio.gio_read(file2, 'x').flatten() y2 = gio.gio_read(file2, 'y').flatten() z2 = gio.gio_read(file2, 'z').flatten() infall_mass2 = gio.gio_read(file2, 'infall_mass').flatten() centrals = (infall_step == step) fof_mass_mask = ( infall_mass[centrals] > 500. * 1.1e9 ) # at least 500 particles (let's not bother with low mass halos for now) nhalos = len(halo_tag[centrals][fof_mass_mask]) print("Number of halos above mass cut = " + str(nhalos)) count = 0 maxcount = min(nhalos, 5000) distarr = [] for j in range(maxcount): fof_halo_tag = halo_tag[centrals][fof_mass_mask][j] mask_fof = (halo_tag == fof_halo_tag) flip, dist = _halo_central_flip_light( x[mask_fof], y[mask_fof], z[mask_fof], infall_step[mask_fof], infall_mass[mask_fof], core_tag[mask_fof], step, fof_halo_tag) # uncomment if you want for the worst cases to see comparison plots with the previous step #flip,dist = _halo_central_flip_detailed(x[mask_fof],y[mask_fof],z[mask_fof],infall_step[mask_fof],infall_mass[mask_fof],core_tag[mask_fof],step,x2,y2,z2,core_tag2,infall_mass2,fof_halo_tag) if flip: count += 1 distarr.append(dist) distarr = np.array(distarr) np.savetxt('/media/luna1/prlarsen/dist1.txt', np.array(distarr)) print("Number of pairs closer than 30kpc = " + str(np.sum(distarr < 0.03)) + " out of a possible " + str(maxcount)) except: print("test_failed") return
import itertools as it massbinsize = 0.5 step = 499 #step = 247 assert SHMLM_HM.step2z[step] == SHMLM_SV.step2z[step], "Redshifts of HM and SV don't match." z = SHMLM_HM.step2z[step] cc_HM = h5_read_dict('/home/isultan/projects/halomassloss/core_catalog_mevolved/output_ALCC_localhost_dtfactor_0.5/{}.corepropertiesextend.hdf5'.format(step), 'coredata') cc_SV = h5_read_dict('/home/isultan/projects/halomassloss/core_catalog_mevolved/output_LJDS_localhost_dtfactor_0.5/m000p-{}.corepropertiesextend.hdf5'.format(step), 'coredata') # mt_cores = gio_read_dict('/home/isultan/data/LJDS/MergerTrees_updated/m000p-{}.treenodes'.format(step), ['fof_halo_tag', 'fof_halo_mass', 'fof_halo_center_x', 'fof_halo_center_y', 'fof_halo_center_z']) sh = gio_read_dict('/home/isultan/data/LJDS/subhalos/m000p-{}.subhaloproperties'.format(step), ['fof_halo_tag','subhalo_mean_x','subhalo_mean_y','subhalo_mean_z','subhalo_mean_vx', 'subhalo_mean_vy', 'subhalo_mean_vz', 'subhalo_count', 'subhalo_tag', 'subhalo_mass']) mt = gio_read_dict('/home/isultan/data/LJDS/subhalos/m000p-{}.haloproperties'.format(step), ['fof_halo_tag', 'fof_halo_mass', 'fof_halo_center_x', 'fof_halo_center_y', 'fof_halo_center_z']) cc_HM['infall_mass'] = gio.gio_read('/home/isultan/data/ALCC/CoreCatalog/{}.coreproperties'.format(step), 'infall_fof_halo_mass')[0] #cc_HM['infall_fof_halo_mass'] cc_SV['infall_mass'] = cc_SV['infall_fof_halo_mass'] # Subhalo-core matching: to look at the subhalo mass:core evolved mass ratio def generate_cores_kdtree(M1, M2, cc, SHMLM, s1=False, disrupt=None, onlyFiltered=False, giveFragmentsFofMass=False, mt_cores=None): # idx_filteredsatcores, M, X, Y, Z, nHalo = SHMLM.core_mask(cc, M1=10**8, M2=10**16, s1=s1, disrupt=disrupt, z=z) idx_filteredsatcores, M, X, Y, Z, nHalo = SHMLM.core_mask(cc, M1, M2, s1=s1, disrupt=disrupt, z=z) cc_filtered = { k:cc[k][idx_filteredsatcores].copy() for k in cc.keys() } cc_filtered['M'] = M if giveFragmentsFofMass: fragmask = cc_filtered['fof_halo_tag']<0 frag_realfht = np.bitwise_and(cc_filtered['fof_halo_tag'][fragmask]*-1, 0xffffffffffff) idx_m21 = many_to_one( frag_realfht, mt_cores['fof_halo_tag'] ) cc_filtered['M'][fragmask] = mt_cores['fof_halo_mass'][idx_m21] cc_filtered['x'] = SHMLM.periodic_bcs(cc_filtered['x'], X, SHMLM.BOXSIZE)
def create_core_catalog_mevolved(virialFlag, writeOutputFlag=True, useLocalHost=False): """ Appends mevolved to core catalog and saves output in HDF5. Works by computing mevolved for step+1 at each step and saving that in memory. """ if writeOutputFlag: print 'Reading data from {} and writing output to {}'.format( cc_data_dir, cc_output_dir) global cc, coresMaskedArray, distance_upper_bound, pool_mergedCoreTag, coresKDTree, KDTree_idx cc = {} cc_prev = {} for step in tqdm(steps): # Read in cc for step cc = {v: gio.gio_read(fname_cc(step, 'input'), v)[0] for v in vars_cc} if virialFlag: # Convert all mass to virial cc['infall_mass'] = SHMLM.m_vir(cc['infall_mass'], step) satellites_mask = cc['central'] == 0 centrals_mask = cc['central'] == 1 # assert np.sum(satellites_mask) + np.sum(centrals_mask) == len(cc['infall_mass']), 'central flag not 0 or 1.' numSatellites = np.sum(satellites_mask) # Verify there are no satellites at first step if step == steps[0]: assert numSatellites == 0, 'Satellites found at first step.' # Add column for m_evolved and initialize to 0 for A in A_arr: for zeta in zeta_arr: cc[m_evolved_col(A, zeta)] = np.zeros_like(cc['infall_mass']) cc['mergedCoreTag'] = np.zeros_like(cc['core_tag']) cc['mergedCoreStep'] = np.zeros_like(cc['infall_step']) # wasInFragment: persistent flag that is False by default and becomes True when core is inside a fragment halo cc['wasInFragment'] = cc['fof_halo_tag'] < 0 if step != steps[0]: _, i1, i2 = np.intersect1d(cc['core_tag'], cc_prev['core_tag'], return_indices=True) cc['wasInFragment'][i1] = np.logical_or( cc['wasInFragment'][i1], cc_prev['wasInFragment'][i2]) cc['mergedCoreTag'][i1] = cc_prev['mergedCoreTag'][i2] cc['mergedCoreStep'][i1] = cc_prev['mergedCoreStep'][i2] # If there are satellites (not applicable for first step) if numSatellites != 0: # Match satellites to prev step cores using core tag. vals, idx1, idx2 = np.intersect1d(cc['core_tag'][satellites_mask], cc_prev['core_tag'], return_indices=True) # assert len(vals) == len(cc['core_tag'][satellites_mask]), 'All cores from prev step did not carry over.' # assert len(cc['core_tag'][satellites_mask]) == len(np.unique(cc['core_tag'][satellites_mask])), 'Satellite core tags not unique for this time step.' for A in A_arr: for zeta in zeta_arr: # Set m_evolved of all satellites that have core tag match on prev step to next_m_evolved of prev step. # DOUBLE MASK ASSIGNMENT: cc['m_evolved'][satellites_mask][idx1] = cc_prev['next_m_evolved'][idx2] cc[m_evolved_col(A, zeta)][np.flatnonzero(satellites_mask) [idx1]] = cc_prev[m_evolved_col( A, zeta, next=True)][idx2] # Initialize m array (corresponds with cc[satellites_mask]) to be either infall_mass (step after infall) or m_evolved (subsequent steps). m = (cc[m_evolved_col(A, zeta)][satellites_mask] == 0) * cc['infall_mass'][satellites_mask] + ( cc[m_evolved_col(A, zeta)][satellites_mask] != 0) * cc[m_evolved_col(A, zeta)][satellites_mask] # Set m_evolved of satellites with m_evolved=0 to infall mass. cc[m_evolved_col(A, zeta)][satellites_mask] = m # Initialize M array (corresponds with cc[satellites_mask]) to be host tree node mass of each satellite idx_m21 = many_to_one(cc['tree_node_index'][satellites_mask], cc['tree_node_index'][centrals_mask]) M, X, Y, Z = (cc[k][centrals_mask][idx_m21] for k in ['infall_mass', 'x', 'y', 'z']) KDTreemask = cc['mergedCoreTag'][satellites_mask] == 0 x = SHMLM.periodic_bcs(cc['x'][satellites_mask][KDTreemask], X[KDTreemask], SHMLM.BOXSIZE) y = SHMLM.periodic_bcs(cc['y'][satellites_mask][KDTreemask], Y[KDTreemask], SHMLM.BOXSIZE) z = SHMLM.periodic_bcs(cc['z'][satellites_mask][KDTreemask], Z[KDTreemask], SHMLM.BOXSIZE) # coresMaskedArray = np.vstack((x, y, z)).T coresMaskedArray_Tree = np.vstack( (np.r_[x, cc['x'][centrals_mask]], np.r_[y, cc['y'][centrals_mask]], np.r_[z, cc['z'][centrals_mask]])).T coresMaskedArray = coresMaskedArray_Tree KDTree_idx = np.r_[np.flatnonzero(satellites_mask)[KDTreemask], np.flatnonzero(centrals_mask)] coresKDTree = spatial.cKDTree(coresMaskedArray_Tree) # Set mergedCoreTag as central core of KDTreemask cores with no mergedCoreTag that are `merged` ''' disruptMask = cc['merged'][satellites_mask][KDTreemask]|(cc['radius'][satellites_mask][KDTreemask] >= SHMLM.CORERADIUSCUT) cc['mergedCoreTag'][ np.flatnonzero(satellites_mask)[KDTreemask] ] += disruptMask*cc['core_tag'][centrals_mask][idx_m21][KDTreemask] cc['mergedCoreStep'][ np.flatnonzero(satellites_mask)[KDTreemask] ] += disruptMask*step ''' ### core merging detection # rvir_KDTreemask = SHMLM.getRvir(cc[m_evolved_col(1.1, 0.068)][satellites_mask][KDTreemask], SHMLM.step2z[step]) #infall should be exact m_evolved in production # rvir_KDTreemask = SHMLM.getRvir(cc['infall_mass'][satellites_mask][KDTreemask], SHMLM.step2z[step]) #infall should be exact m_evolved in production # distance_upper_bound = SHMLM.COREVIRIALRADIUSFACTOR * rvir_KDTreemask distance_upper_bound = SHMLM.COREVIRIALRADIUSFACTOR * SHMLM.getRvir( cc['infall_mass'][KDTree_idx], SHMLM.step2z[step]) import ctypes # pool_mergedCoreTag = { i:np.full(len(KDTree_idx), 0, dtype=np.int64) for i in range(1,pool_size+1) } pool_mergedCoreTag = { i: multiprocessing.RawArray( ctypes.c_int64, np.zeros(len(KDTree_idx), dtype=np.int64)) for i in range(1, pool_size + 1) } # Search radius of 2*rvir around each core from datetime import datetime tstart = datetime.now() work = np.arange(len(distance_upper_bound)) t1 = datetime.now() - tstart print "work=", t1 tstart = datetime.now() p = Pool(pool_size) t2 = datetime.now() - tstart print "Pool=", t2 tstart = datetime.now() p.map(query, np.array_split(work, pool_size)) t3 = datetime.now() - tstart print "mapp=", t3 tstart = datetime.now() p.close() t4 = datetime.now() - tstart print "close=", t4 tstart = datetime.now() p.join() t5 = datetime.now() - tstart print "join=", t5 tstart = datetime.now() # mergedCoreTag = np.full_like(pool_mergedCoreTag[1], 0) mergedCoreTag = np.zeros(len(KDTree_idx), dtype=np.int64) # pool_mergedCoreTag = {k:np.array(pool_mergedCoreTag[k]) for k in pool_mergedCoreTag.keys()} for i in range(1, pool_size + 1): mCT = np.array(pool_mergedCoreTag[i]) newMergedMask = mCT != 0 # print np.sum(newMergedMask) mergedCoreTag[newMergedMask] = mCT[newMergedMask] t6 = datetime.now() - tstart newMergedMask = mergedCoreTag != 0 cc['mergedCoreTag'][ KDTree_idx[newMergedMask]] = mergedCoreTag[newMergedMask] cc['mergedCoreStep'][KDTree_idx[newMergedMask]] = step print np.sum(cc['mergedCoreTag'] != 0) t7 = datetime.now() - tstart print t1, t2, t3, t4, t5, t6, t7 ### print 'Merged centrals: ', np.sum( cc['mergedCoreTag'][centrals_mask] != 0) cc['mergedCoreTag'][centrals_mask] = 0 cc['mergedCoreStep'][centrals_mask] = 0 if writeOutputFlag: # Write output to disk h5_write_dict(fname_cc(step, 'output'), cc, 'coredata') # Compute m_evolved of satellites according to SHMLModel for NEXT time step and save as cc_prev['next_m_evolved'] in memory. # Mass loss assumed to begin at step AFTER infall detected. if step != steps[-1]: # cc_prev = { 'core_tag':cc['core_tag'].copy(), 'wasInFragment':cc['wasInFragment'].copy() } cc_prev = { k: cc[k].copy() for k in [ 'core_tag', 'wasInFragment', 'mergedCoreTag', 'mergedCoreStep' ] } if numSatellites != 0: localhost_m21 = many_to_one(cc['host_core'][satellites_mask], cc['core_tag']) for A in A_arr: for zeta in zeta_arr: cc_prev[m_evolved_col(A, zeta, next=True)] = np.zeros_like( cc['infall_mass']) if numSatellites != 0: # If there are satellites (not applicable for first step) m = cc[m_evolved_col(A, zeta)][satellites_mask] if useLocalHost: Mlocal = cc[m_evolved_col(A, zeta)][localhost_m21] M_A_zeta = (Mlocal == 0) * M + (Mlocal != 0) * Mlocal cc_prev[m_evolved_col( A, zeta, next=True)][satellites_mask] = SHMLM.m_evolved( m0=m, M0=M_A_zeta, step=steps[steps.index(step) + 1], step_prev=step, A=A, zeta=zeta) else: cc_prev[m_evolved_col( A, zeta, next=True)][satellites_mask] = SHMLM.m_evolved( m0=m, M0=M, step=steps[steps.index(step) + 1], step_prev=step, A=A, zeta=zeta) return cc
def create_core_catalog_mevolved(writeOutputFlag=True, useLocalHost=False): """ Appends mevolved to core catalog and saves output in HDF5. Works by computing mevolved for step+1 at each step and saving that in memory. """ if writeOutputFlag: print 'Reading data from {} and writing output to {}'.format( cc_data_dir, cc_output_dir) cc = {} cc_prev = {} for step in tqdm(steps): # Read in cc for step cc = { v: gio.gio_read(fname_cc(step, 'input'), v)[0] for v in vars_cc(step) } satellites_mask = cc['central'] == 0 centrals_mask = cc['central'] == 1 # assert np.sum(satellites_mask) + np.sum(centrals_mask) == len(cc['infall_tree_node_mass']), 'central flag not 0 or 1.' numSatellites = np.sum(satellites_mask) # Verify there are no satellites at first step if step == steps[0]: assert numSatellites == 0, 'Satellites found at first step.' # Add column for m_evolved and initialize to 0 cc[m_evolved_col(A, zeta)] = np.zeros_like(cc['infall_tree_node_mass']) # If there are satellites (not applicable for first step) if numSatellites != 0: # Match satellites to prev step cores using core tag. vals, idx1, idx2 = np.intersect1d(cc['core_tag'][satellites_mask], cc_prev['core_tag'], return_indices=True) idx_m21 = many_to_one(cc['tree_node_index'][satellites_mask], cc['tree_node_index'][centrals_mask]) M = cc['infall_tree_node_mass'][centrals_mask][idx_m21] # Set m_evolved of all satellites that have core tag match on prev step to next_m_evolved of prev step. cc[m_evolved_col(A, zeta)][np.flatnonzero(satellites_mask) [idx1]] = cc_prev[m_evolved_col( A, zeta, next=True)][idx2] # Initialize m array (corresponds with cc[satellites_mask]) to be either infall_mass (step after infall) or m_evolved (subsequent steps). initMask = cc[m_evolved_col(A, zeta)][satellites_mask] == 0 minfall = cc['infall_tree_node_mass'][satellites_mask][initMask] cc[m_evolved_col(A, zeta)][np.flatnonzero( satellites_mask)[initMask]] = SHMLM.m_evolved( m0=minfall, M0=M[initMask], step=step, step_prev=steps[steps.index(step) - 1], A=A, zeta=zeta, dtFactorFlag=True) if writeOutputFlag and (step == 499 or step == 247): # Write output to disk h5_write_dict(fname_cc(step, 'output'), cc, 'coredata') # Compute m_evolved of satellites according to SHMLModel for NEXT time step and save as cc_prev['next_m_evolved'] in memory. # Mass loss assumed to begin at step AFTER infall detected. if step != steps[-1]: cc_prev = {'core_tag': cc['core_tag'].copy()} if numSatellites != 0: localhost_m21 = many_to_one(cc['host_core'][satellites_mask], cc['core_tag']) cc_prev[m_evolved_col(A, zeta, next=True)] = np.zeros_like( cc['infall_tree_node_mass']) if numSatellites != 0: # If there are satellites (not applicable for first step) m = cc[m_evolved_col(A, zeta)][satellites_mask] if useLocalHost: Mlocal = cc[m_evolved_col(A, zeta)][localhost_m21] M_A_zeta = (Mlocal == 0) * M + (Mlocal != 0) * Mlocal cc_prev[m_evolved_col( A, zeta, next=True)][satellites_mask] = SHMLM.m_evolved( m0=m, M0=M_A_zeta, step=steps[steps.index(step) + 1], step_prev=step, A=A, zeta=zeta) else: cc_prev[m_evolved_col( A, zeta, next=True)][satellites_mask] = SHMLM.m_evolved( m0=m, M0=M, step=steps[steps.index(step) + 1], step_prev=step, A=A, zeta=zeta) return cc
def create_core_catalog_mevolved(virialFlag=False, writeOutputFlag=True, useLocalHost=False, checkpointRestart=None): """ Appends mevolved to core catalog and saves output in HDF5. Works by computing mevolved for step+1 at each step and saving that in memory. """ if writeOutputFlag: print 'Reading data from {} and writing output to {}'.format( cc_data_dir, cc_output_dir) cc = {} cc_prev = {} if checkpointRestart is None: itersteps = steps else: itersteps = steps[steps.index(checkpointRestart):] for step in tqdm(itersteps): # Read in cc for step if checkpointRestart is None: cc = { v: gio.gio_read(fname_cc(step, 'input'), v)[0] for v in vars_cc } if virialFlag: # Convert all mass to virial cc['infall_tree_node_mass'] = SHMLM.m_vir( cc['infall_tree_node_mass'], step) else: cc = h5_read_dict(fname_cc(checkpointRestart, 'output'), 'coredata') satellites_mask = cc['central'] == 0 centrals_mask = cc['central'] == 1 # assert np.sum(satellites_mask) + np.sum(centrals_mask) == len(cc['infall_tree_node_mass']), 'central flag not 0 or 1.' numSatellites = np.sum(satellites_mask) # Verify there are no satellites at first step if step == steps[0]: assert numSatellites == 0, 'Satellites found at first step.' # Add column for m_evolved and initialize to 0 if checkpointRestart is None: for A in A_arr: for zeta in zeta_arr: cc[m_evolved_col(A, zeta)] = np.zeros_like( cc['infall_tree_node_mass']) # If there are satellites (not applicable for first step) if numSatellites != 0: if checkpointRestart is None: # Match satellites to prev step cores using core tag. vals, idx1, idx2 = np.intersect1d( cc['core_tag'][satellites_mask], cc_prev['core_tag'], return_indices=True) # assert len(vals) == len(cc['core_tag'][satellites_mask]), 'All cores from prev step did not carry over.' # assert len(cc['core_tag'][satellites_mask]) == len(np.unique(cc['core_tag'][satellites_mask])), 'Satellite core tags not unique for this time step.' # Initialize M array (corresponds with cc[satellites_mask]) to be host tree node mass of each satellite idx_m21 = many_to_one(cc['tree_node_index'][satellites_mask], cc['tree_node_index'][centrals_mask]) M, X, Y, Z = (cc[k][centrals_mask][idx_m21] for k in ['infall_tree_node_mass', 'x', 'y', 'z']) if checkpointRestart is None: for A in A_arr: for zeta in zeta_arr: # Set m_evolved of all satellites that have core tag match on prev step to next_m_evolved of prev step. # DOUBLE MASK ASSIGNMENT: cc['m_evolved'][satellites_mask][idx1] = cc_prev['next_m_evolved'][idx2] cc[m_evolved_col(A, zeta)][np.flatnonzero( satellites_mask)[idx1]] = cc_prev[m_evolved_col( A, zeta, next=True)][idx2] initMask = cc[m_evolved_col( A, zeta)][satellites_mask] == 0 minfall = cc['infall_tree_node_mass'][satellites_mask][ initMask] cc[m_evolved_col(A, zeta)][np.flatnonzero( satellites_mask)[initMask]] = SHMLM.m_evolved( m0=minfall, M0=M[initMask], step=step, step_prev=steps[steps.index(step) - 1], A=A, zeta=zeta, dtFactorFlag=True) if writeOutputFlag and (checkpointRestart is None): # Write output to disk h5_write_dict(fname_cc(step, 'output'), cc, 'coredata') #cc should be checkpointRestart step file if not None if checkpointRestart is not None: checkpointRestart = None # Compute m_evolved of satellites according to SHMLModel for NEXT time step and save as cc_prev['next_m_evolved'] in memory. if step != steps[-1]: cc_prev = {'core_tag': cc['core_tag'].copy()} if numSatellites != 0: localhost_m21 = many_to_one(cc['host_core'][satellites_mask], cc['core_tag']) for A in A_arr: for zeta in zeta_arr: cc_prev[m_evolved_col(A, zeta, next=True)] = np.zeros_like( cc['infall_tree_node_mass']) if numSatellites != 0: # If there are satellites (not applicable for first step) m = cc[m_evolved_col(A, zeta)][satellites_mask] if useLocalHost: Mlocal = cc[m_evolved_col(A, zeta)][localhost_m21] M_A_zeta = (Mlocal == 0) * M + (Mlocal != 0) * Mlocal cc_prev[m_evolved_col( A, zeta, next=True)][satellites_mask] = SHMLM.m_evolved( m0=m, M0=M_A_zeta, step=steps[steps.index(step) + 1], step_prev=step, A=A, zeta=zeta) else: cc_prev[m_evolved_col( A, zeta, next=True)][satellites_mask] = SHMLM.m_evolved( m0=m, M0=M, step=steps[steps.index(step) + 1], step_prev=step, A=A, zeta=zeta) return cc
def create_core_catalog_mevolved(virialFlag, writeOutputFlag=True, useLocalHost=False): """ Appends mevolved to core catalog and saves output in HDF5. Works by computing mevolved for step+1 at each step and saving that in memory. """ if writeOutputFlag: print 'Reading data from {} and writing output to {}'.format(cc_data_dir, cc_output_dir) cc = {} cc_prev = {} for step in tqdm(steps): # Read in cc for step cc = { v:gio.gio_read(fname_cc(step, 'input'), v)[0] for v in vars_cc } if virialFlag: # Convert all mass to virial cc['infall_mass'] = SHMLM.m_vir(cc['infall_mass'], step) satellites_mask = cc['central'] == 0 centrals_mask = cc['central'] == 1 # assert np.sum(satellites_mask) + np.sum(centrals_mask) == len(cc['infall_mass']), 'central flag not 0 or 1.' numSatellites = np.sum(satellites_mask) # Verify there are no satellites at first step if step == steps[0]: assert numSatellites == 0, 'Satellites found at first step.' # Add column for m_evolved and initialize to 0 for A in A_arr: for zeta in zeta_arr: cc[m_evolved_col(A, zeta)] = np.zeros_like(cc['infall_mass']) # If there are satellites (not applicable for first step) if numSatellites != 0: # Match satellites to prev step cores using core tag. vals, idx1, idx2 = np.intersect1d( cc['core_tag'][satellites_mask], cc_prev['core_tag'], return_indices=True ) # assert len(vals) == len(cc['core_tag'][satellites_mask]), 'All cores from prev step did not carry over.' # assert len(cc['core_tag'][satellites_mask]) == len(np.unique(cc['core_tag'][satellites_mask])), 'Satellite core tags not unique for this time step.' for A in A_arr: for zeta in zeta_arr: # Set m_evolved of all satellites that have core tag match on prev step to next_m_evolved of prev step. # DOUBLE MASK ASSIGNMENT: cc['m_evolved'][satellites_mask][idx1] = cc_prev['next_m_evolved'][idx2] cc[m_evolved_col(A, zeta)][ np.flatnonzero(satellites_mask)[idx1] ] = cc_prev[m_evolved_col(A, zeta, next=True)][idx2] # Initialize m array (corresponds with cc[satellites_mask]) to be either infall_mass (step after infall) or m_evolved (subsequent steps). m = (cc[m_evolved_col(A, zeta)][satellites_mask] == 0)*cc['infall_mass'][satellites_mask] + (cc[m_evolved_col(A, zeta)][satellites_mask] != 0)*cc[m_evolved_col(A, zeta)][satellites_mask] # Initialize m array (corresponds with cc[satellites_mask]) to be either virial_infall_mass (step after infall) or m_evolved (subsequent steps). #m = (cc[m_evolved_col(A, zeta)][satellites_mask] == 0)*SHMLM.m_vir(cc['infall_mass'][satellites_mask], step) + (cc[m_evolved_col(A, zeta)][satellites_mask] != 0)*cc[m_evolved_col(A, zeta)][satellites_mask] # Set m_evolved of satellites with m_evolved=0 to infall mass. cc[m_evolved_col(A, zeta)][satellites_mask] = m # Initialize M array (corresponds with cc[satellites_mask]) to be host tree node mass of each satellite M = cc['infall_mass'][centrals_mask][ many_to_one( cc['tree_node_index'][satellites_mask], cc['tree_node_index'][centrals_mask] ) ] # assert len(M) == len(m) == len(cc['m_evolved'][satellites_mask]), 'M, m, cc[satellites_mask] lengths not equal.' # wasInFragment: persistent flag that is False by default and becomes True when core is inside a fragment halo cc['wasInFragment'] = cc['fof_halo_tag']<0 if step != steps[0]: _, i1, i2 = np.intersect1d( cc['core_tag'], cc_prev['core_tag'], return_indices=True ) cc['wasInFragment'][i1] = np.logical_or( cc['wasInFragment'][i1], cc_prev['wasInFragment'][i2] ) if writeOutputFlag: # Write output to disk h5_write_dict( fname_cc(step, 'output'), cc, 'coredata' ) # Compute m_evolved of satellites according to SHMLModel for NEXT time step and save as cc_prev['next_m_evolved'] in memory. # Mass loss assumed to begin at step AFTER infall detected. if step != steps[-1]: cc_prev = { 'core_tag':cc['core_tag'].copy(), 'wasInFragment':cc['wasInFragment'].copy() } if numSatellites != 0: localhost_m21 = many_to_one( cc['host_core'][satellites_mask], cc['core_tag'] ) for A in A_arr: for zeta in zeta_arr: cc_prev[m_evolved_col(A, zeta, next=True)] = np.zeros_like(cc['infall_mass']) if numSatellites != 0: # If there are satellites (not applicable for first step) m = cc[m_evolved_col(A, zeta)][satellites_mask] if useLocalHost: Mlocal = cc[m_evolved_col(A, zeta)][localhost_m21] M_A_zeta = (Mlocal==0)*M + (Mlocal!=0)*Mlocal cc_prev[m_evolved_col(A, zeta, next=True)][satellites_mask] = SHMLM.m_evolved(m0=m, M0=M_A_zeta, step=steps[steps.index(step)+1], step_prev=step, A=A, zeta=zeta) else: cc_prev[m_evolved_col(A, zeta, next=True)]][satellites_mask] = SHMLM.m_evolved(m0=m, M0=M, step=steps[steps.index(step)+1], step_prev=step, A=23.7, zeta=0.36) return cc
def gio_read_dict(f, cols): """Read cols of GenericIO file f and return dictionary of numpy arrays.""" import genericio as gio return {k: gio.gio_read(f, k)[0] for k in cols}