def check_unique_core_tags(file1):
''' Check that core tags and infall tree nodes are unique
criteria: no repeating core tags or infall tree nodes
inputs: file1 = core catalog file
outputs: criteria (pass=0)
display: Uniqueness of core tag and infall tree node
'''
try:
core_tag = gio.gio_read(file1, "core_tag")
infall_tree_node_index = gio.gio_read(file1, "infall_tree_node_index")
core_tag_unique = _is_unique_array(core_tag)
tree_node_unique = _is_unique_array(infall_tree_node_index)
if core_tag_unique:
print("Core tag is unique")
else:
print("Core tag is not unique")
if tree_node_unique:
print("Infall tree node index is unique")
else:
print("Infall tree node index is not unique")
if core_tag_unique & tree_node_unique:
return 0
else:
return 1
except:
print("test failed")
def return_idx():
tag_new = gio.gio_read(file_new, 'fof_halo_tag')
tag_old = gio.gio_read(file_old, 'fof_halo_tag')
idx_new = np.argsort(tag_new)
idx_old = np.argsort(tag_old)
print((tag_new[idx_new] == tag_old[idx_old]).all())
return idx_new, idx_old
def check_positions(file1):
''' Check that the positions of the cores look sensible
criteria: _check_nan_in_positions test passess for all position variables
inputs: file1 = core catalog file
outputs: criteria (pass=0)
plot: 1d histogram of x,y,z values, 2d histogram of (x,y) values
display: output of _check_nan_in_positions test
'''
try:
test = 0
x = gio.gio_read(file1, 'x').flatten()
y = gio.gio_read(file1, 'y').flatten()
z = gio.gio_read(file1, 'z').flatten()
test += _check_nan_in_positions('x', x)
test += _check_nan_in_positions('y', y)
test += _check_nan_in_positions('z', z)
if vis:
plt.figure()
plt.hist(x, bins=100, alpha=0.4)
plt.hist(y, bins=100, alpha=0.4)
plt.hist(z, bins=100, alpha=0.4)
plt.xlabel('position (Mpc)')
plt.figure()
plt.hist2d(x, y, bins=100)
plt.xlabel('x (Mpc)')
plt.ylabel('y (Mpc)')
plt.show()
return test
except:
print("can't run position test")
def check_central(file1, step):
''' Check that all halos have a central core
criteria: a core with infall step = current step exists for all fof halos
inputs: file1 = core catalog file
step = time step associated with core catalog file
outputs: criteria (pass=0)
display: Maximum number of cores in a halo
Total number of fof halos in file
'''
try:
halo_tag = gio.gio_read(file1, 'fof_halo_tag')
infall_step = gio.gio_read(file1, 'infall_step')
unique_tags, tag_counts = np.unique(halo_tag, return_counts=True)
print("Maximum number of cores in a halo is " + str(max(tag_counts)))
print("Number of halos is " + str(len(unique_tags)))
unique_tags_central, tag_counts_central = np.unique(
halo_tag[infall_step == step], return_counts=True)
criteria = (np.sort(unique_tags) == np.sort(unique_tags_central)).all()
if criteria:
print("All halos have a central core")
return 0
else:
print("Some halos are missing a central core")
return 1
except:
print("test failed")
def check_mt_files(file1, file_mt, step):
''' Check that the number of halos in the merger tree file is consistent with the number of central cores
criteria: number of halos in MT file and number of central cores agree to within 10%
inputs: file1 = core catalog file
file_mt = merger tree file
step = current time step
outputs: criteria (pass=0)
display: Number of halos in merger tree file
Number of central cores in core file
option: plot_missing
if plot_missing, then print out the number of 'missing objects', check
how many of these have zero-masses, and plot the positions of the first
10,000 objects. (note at step=499 this shows boundary issues, at higher
redshifts this shows fragments)
'''
try:
plot_missing = True # useful to turn on if step = 499, so you can see if all missing objects are at mass resolution
if (step != 499):
print(
"Warning, merger tree files should only match properly at step 499 (at higher redshifts N_mergertree > N_centralcores due to fragmentation). As you are looking at a higher redshift, expect a ~5% level of missing cores with a range of masses (including 0 mass fragments)."
)
halo_tag = gio.gio_read(file_mt, 'fof_halo_tag')
halo_mass = gio.gio_read(file_mt, 'fof_halo_mass')
halo_tag_c = gio.gio_read(file1, 'fof_halo_tag')
infall_step = gio.gio_read(file1, 'infall_step')
halo_tag_c = halo_tag_c[infall_step == step]
print("Number of halos in merger tree file = " + str(len(halo_tag)))
print("Number of central cores in core file = " + str(len(halo_tag_c)))
diff_tot = (len(halo_tag) - len(halo_tag_c) +
0.0) / len(halo_tag_c) * 100.
criteria = (diff_tot < 10)
if vis & plot_missing:
print("Plotting missing cores - this may take a minute.")
import random
missing_cores = np.setdiff1d(halo_tag, halo_tag_c)
print("Number of missing objects = " + str(len(missing_cores)))
random.shuffle(missing_cores)
masses = []
for i in range(min(1000, len(missing_cores))):
#print(i)
idx = np.where(halo_tag == missing_cores[i])[0][0]
#print(idx)
masses.append(halo_mass[idx])
masses = np.array(masses)
if 0 in masses:
print("Fraction of zero-mass objects = " +
str(np.sum(masses == 0) / (len(masses) + 0.0)))
plt.figure()
plt.hist((masses[masses > 0] / 1.09e9), bins=100)
plt.xlabel('non-zero masses (log10)')
plt.show()
if criteria:
return 0
else:
return 1
except:
print("test_failed")
def read_mass_cdelta():
''' read and mask the SOD mass and concentration '''
mask12, fof_old = make_mask()
print("made mask")
sod_old = gio.gio_read(file_old, 'sod_halo_mass')[mask12]
sod_new = gio.gio_read(file_new, 'sod_halo_mass')[mask12]
cdelta_old = gio.gio_read(file_old, 'sod_halo_cdelta')[mask12]
cdelta_new = gio.gio_read(file_new, 'sod_halo_cdelta')[mask12]
return fof_old, sod_old, sod_new, cdelta_old, cdelta_new
def check_tags():
''' Double check ordering is the same from the tags '''
tag_old = gio.gio_read(file_old, 'fof_halo_tag')
tag_new = gio.gio_read(file_new, 'fof_halo_tag')
if (tag_old == tag_new).all():
print("ordering is the same")
else:
print("ordering has changed - double check this before continuing")
return
def check_core_evolution(file1, file2):
''' Check that core tags present in an earlier timestep are retained in the later timestep
criteria: less than 5% of the core tags present in the earlier timestep are missing in the later step
inputs: file1 = core catalog file
file2 = core catalog file at lower step number (higher redshift)
outputs: criteria (pass=0)
display: Percentage of cores present in earlier timestep missing in subsequent timestep
option: plot_missing
if plot_missing, then print out the number of 'missing objects', and plot the
positions of the first 10,000 objects alongside a core density map for comparison.
'''
try:
plot_missing = False
core_tag1 = gio.gio_read(file1, "core_tag")
core_tag2 = gio.gio_read(file2, "core_tag")
# file1 should be later in time (higher step number) than file2
missing_cores = np.setdiff1d(core_tag2, core_tag1, assume_unique=False)
perc_missing = float(len(missing_cores)) / len(core_tag2) * 100
criteria = (perc_missing < 5)
print(
"Percentage of cores present in earlier timestep missing in subsequent timestep = "
+ str(perc_missing))
if vis & plot_missing:
import random
random.shuffle(missing_cores)
print(
"Plotting positions of 10,000 missing cores - this may take a minute"
)
x = gio.gio_read(file2, 'x').flatten()
y = gio.gio_read(file2, 'y').flatten()
z = gio.gio_read(file2, 'z').flatten()
xx = []
yy = []
zz = []
for i in range(10000):
idx = np.where(core_tag2 == missing_cores[i])[0][0]
xx.append(x[idx])
yy.append(y[idx])
zz.append(z[idx])
xx = np.array(xx)
yy = np.array(yy)
zz = np.array(zz)
plt.hist2d(xx, yy, bins=100)
plt.title('location of missing cores')
plt.figure()
plt.hist2d(x, y, bins=100)
plt.title('location of all cores - for comparison')
plt.show()
if criteria:
return 0
else:
return 1
except:
print("test_failed")
def main():
arrx = gio.gio_read(file_name, 'x', rank, size)
arry = gio.gio_read(file_name, 'y', rank, size)
i = 0
for elem in arrx[:,0]:
if elem != 0:
i +=1
perc= np.around(i/float(arrx.shape[0]), decimals=3)
print i,"/",arrx.shape[0],"(",perc*100,"%)", " nonzero elements in rank ",rank
plt.scatter(arrx, arry)
plt.show()
def check_prop(var, idx_new, idx_old):
var_new = gio.gio_read(file_new, var)
var_old = gio.gio_read(file_old, var)
ident = (var_new[idx_new] == var_old[idx_old]).all()
if ident:
print("values are identical for variable ", var)
else:
print("values are not identical for variable ", var)
print(np.sum(var_new[idx_new] != var_old[idx_old]))
print(np.max(var_new[idx_new] - var_old[idx_old]))
print(np.min(var_new[idx_new] - var_old[idx_old]))
print(np.max((var_new[idx_new] - var_old[idx_old]) / var_old[idx_old]))
print(np.min((var_new[idx_new] - var_old[idx_old]) / var_old[idx_old]))
return
def check_infall_timesteps(file1):
''' Check the distribution of infall steps
criteria: minimum infall step > 0 (sanity check on time step assignment)
inputs: file1 = core catalog file
outputs: criteria (pass=0)
plots: histogram of infall time step
display: list of unique infall time steps
'''
try:
infall_step = gio.gio_read(file1, 'infall_step')
criteria = (np.min(infall_step) > 0)
print("List of infall steps:")
print(np.unique(infall_step))
if criteria:
print("Minimum infall step has positive value: " +
str(np.min(infall_step)))
if vis:
plt.figure()
plt.hist(infall_step, bins=100)
plt.xlabel('infall step')
plt.show()
return 0
else:
print("Minimum infall step not sensible: " +
str(np.min(infall_step)))
return 1
except:
print("can't run infall timestep test")
return 1
def check_infall_masses(file1):
''' Check how many particles make up the minimum infall mass, and plot the distribution
criteria: minimum infall mass > 0 (this is effectively a sanity check on mass assignment)
inputs: file1 = core catalog file
outputs: criteria (pass=0)
plots: histogram of log10(mass)
'''
try:
infall_mass = gio.gio_read(file1, 'infall_mass')
criteria = (np.min(infall_mass) > 0)
if criteria:
print("Minimum infall mass has positive value: " +
str(round(np.min(infall_mass) / 1.09086e9)) + " particles")
if vis:
plt.figure()
plt.hist(np.log10(infall_mass), bins=100)
plt.xlabel('log10(infall mass)')
plt.yscale('log')
plt.show()
return 0
else:
print("minimum infall mass not sensible: " +
str(np.min(infall_mass)))
return 1
except:
print("can't run infall mass test")
return 1
def read_in_lightcone(input_file,
vars=['x', 'y', 'z', 'vx', 'vy', 'vz', 'id']):
''' read in lightcone as a dictionary '''
lightcone_object = {}
for i in vars:
lightcone_object[i] = gio.gio_read(input_file, i)
return lightcone_object
def getData(
dataFile, rank, size, *args
): #Takes a data file and a list of attributes as input, and returns an array whose columns are the attributes from the file (e.g. mass, position, etc)
nObjects = gio.gio_read(dataFile, args[0], rank, size).shape[
0] #gets number of objects in file by checking the shape of the numpy array returned from the first arg .
data_array = np.empty(
shape=(nObjects, len(args))) #one row per object, one column per arg
names = []
formats = []
for index, arg in enumerate(args):
arr = gio.gio_read(dataFile, arg, rank, size).flatten()
data_array[:, index] = arr
names.append(arg)
formats.append(type(arr[0]))
dtype = dict(names=names, formats=formats)
new_array = np.core.records.fromarrays(data_array.transpose(), dtype)
return new_array
def read_centers():
mask12, fof_old = make_mask()
x_old = gio.gio_read(file_old, 'fof_halo_center_x')[mask12]
y_old = gio.gio_read(file_old, 'fof_halo_center_y')[mask12]
z_old = gio.gio_read(file_old, 'fof_halo_center_z')[mask12]
x_new = gio.gio_read(file_new, 'fof_halo_center_x')[mask12]
y_new = gio.gio_read(file_new, 'fof_halo_center_y')[mask12]
z_new = gio.gio_read(file_new, 'fof_halo_center_z')[mask12]
radius = gio.gio_read(file_old, 'sod_halo_radius')[mask12]
return fof_old, x_old, y_old, z_old, x_new, y_new, z_new, radius
def check_heirarchy(file1):
'''
Check that host core tag is within the same tree node index as the core tag.
Check that infall step of the host is higher than infall step of the core
'''
core_tag = gio.gio_read(file1, "core_tag")
host_core_tag = gio.gio_read(file1, "host_core")
vals, ar1, ar2 = np.intersect1d(core_tag,
host_core_tag,
assume_unique=False,
return_indices=True)
tni = gio.gio_read(file1, 'tree_node_index')
infall_step = gio.gio_read(file1, 'infall_step')
if (tni[ar1]
== tni[ar2]).all() & (infall_step[ar1] > infall_step[ar2]).all():
return
else:
print(np.sum(tni[ar1] != tni[ar2]),
np.sum(infall_step[ar1] <= infall_step[ar2]))
return 1
def norm_mass(file1):
'''
Checking that core abundance increases with host halo mass as a power law with index unity
'''
infall_mass = gio.gio_read(file1, 'infall_mass').flatten()
infall_step = gio.gio_read(file1, 'infall_step').flatten()
host_tag = gio.gio_read(file1, 'fof_halo_tag').flatten()
host_mass = infall_mass
infall_mass = infall_mass[host_mass >= 1.e12]
host_tag = host_tag[host_mass >= 1.e12]
infall_step = infall_step[host_mass >= 1.e12]
host_tag_u = host_tag[infall_step == 499]
host_mass = host_mass[host_mass >= 1.e12][infall_step == 499]
dict_values = dict(zip(host_tag_u, host_mass))
def apply_dict(i):
return dict_values[i]
infall_mass = gio.gio_read(file1, 'infall_mass').flatten()
infall_step = gio.gio_read(file1, 'infall_step').flatten()
host_tag = gio.gio_read(file1, 'fof_halo_tag').flatten()
mask = (infall_mass > 1.e12) & (infall_step != 499)
tags, counts = np.unique(host_tag[mask], return_counts=True)
mass_tot = np.array(map(apply_dict, tags))
mean_sats, edges, nums = bs(np.log10(mass_tot),
counts - 1.,
statistic='mean',
bins=100)
bin_mean = (edges[1:] - edges[:-1]) / 2. + edges[:-1]
idx_13 = np.where(
np.abs(bin_mean - 13.5) == np.min(np.abs(bin_mean - 13.5)))
idx_15 = np.where(
np.abs(bin_mean - 14.5) == np.min(np.abs(bin_mean - 14.5)))
diff_val = (np.log10(mean_sats[idx_15]) - np.log10(mean_sats[idx_13]))
criteria = (diff_val < 1.2) & (diff_val > 0.8)
print((np.log10(mean_sats[idx_15]) - np.log10(mean_sats[idx_13])))
if vis:
plt.figure()
plt.plot(bin_mean, mean_sats)
plt.ylabel('N_cores (>1.e12 mass)')
plt.yscale('log')
plt.xlabel('log10(host mass)')
plt.xlim([12.5, 15])
plt.show()
if criteria:
print(
"power law index of core abundance as a function of host halo mass has an index of nearly unity"
)
return 0
else:
print("core abundance as a function of host halo mass needs checking")
return 1
def read_outerrim(snapshot, redshift, file_number, path=""):
"""
Reads halo position, velocity and mass from OuterRim file
Args:
snapshot: OuterRim snapshot number
redshift: redshift of OuterRim snapshot
file_number: file number of OuterRim file to read
path: path of directory where OuterRim files are stored
Returns:
pos: array of comoving positions in units [Mpc/h]
vel: array of proper velocities in units [km/s]
mass: array of masses in units [Msun/h]
"""
halo_cat = path + "02_17_2016.OuterRim.%i.fofproperties#%i" % (snapshot,
file_number)
# read halo mass
mass = genericio.gio_read(halo_cat, 'fof_halo_mass')[0]
# read comoving positions
x = genericio.gio_read(halo_cat, 'fof_halo_center_x')[0]
y = genericio.gio_read(halo_cat, 'fof_halo_center_y')[0]
z = genericio.gio_read(halo_cat, 'fof_halo_center_z')[0]
pos = np.array([x, y, z]).transpose()
del x, y, z
# read comoving velocities
vx = genericio.gio_read(halo_cat, 'fof_halo_mean_vx')[0]
vy = genericio.gio_read(halo_cat, 'fof_halo_mean_vy')[0]
vz = genericio.gio_read(halo_cat, 'fof_halo_mean_vz')[0]
vel = np.array([vx, vy, vz]).transpose()
del vx, vy, vz
# convert velocities from comoving to proper
vel = vel / (1. + redshift)
return pos, vel, mass
def create_core_catalog_mevolved(virialFlag=False,
writeOutputFlag=True,
useLocalHost=False):
"""
Appends mevolved to core catalog and saves output in HDF5.
Works by computing mevolved for step+1 at each step and saving that in memory.
"""
if writeOutputFlag:
print 'Reading data from {} and writing output to {}'.format(
cc_data_dir, cc_output_dir)
cc = {}
cc_prev = {}
for step in tqdm(steps):
# Read in cc for step
cc = {v: gio.gio_read(fname_cc(step, 'input'), v)[0] for v in vars_cc}
if virialFlag:
# Convert all mass to virial
cc['infall_mass'] = SHMLM.m_vir(cc['infall_mass'], step)
satellites_mask = cc['central'] == 0
centrals_mask = cc['central'] == 1
# assert np.sum(satellites_mask) + np.sum(centrals_mask) == len(cc['infall_mass']), 'central flag not 0 or 1.'
numSatellites = np.sum(satellites_mask)
# Verify there are no satellites at first step
if step == steps[0]:
assert numSatellites == 0, 'Satellites found at first step.'
# Add column for m_evolved and initialize to 0
for A in A_arr:
for zeta in zeta_arr:
cc[m_evolved_col(A, zeta)] = np.zeros_like(cc['infall_mass'])
'''
#cc['mergedCoreTag'] = np.zeros_like(cc['core_tag'])
#cc['mergedCoreStep'] = np.zeros_like(cc['infall_step'])
cc['phaseSpaceMerged'] = np.zeros_like(cc['central'])
# wasInFragment: persistent flag that is False by default and becomes True when core is inside a fragment halo
cc['wasInFragment'] = cc['fof_halo_tag']<0
if step != steps[0]:
_, i1, i2 = np.intersect1d( cc['core_tag'], cc_prev['core_tag'], return_indices=True )
cc['wasInFragment'][i1] = np.logical_or( cc['wasInFragment'][i1], cc_prev['wasInFragment'][i2] )
#cc['mergedCoreTag'][i1] = cc_prev['mergedCoreTag'][i2]
#cc['mergedCoreStep'][i1] = cc_prev['mergedCoreStep'][i2]
cc['phaseSpaceMerged'][i1] = cc_prev['phaseSpaceMerged'][i2]
'''
# If there are satellites (not applicable for first step)
if numSatellites != 0:
# Match satellites to prev step cores using core tag.
vals, idx1, idx2 = np.intersect1d(cc['core_tag'][satellites_mask],
cc_prev['core_tag'],
return_indices=True)
# assert len(vals) == len(cc['core_tag'][satellites_mask]), 'All cores from prev step did not carry over.'
# assert len(cc['core_tag'][satellites_mask]) == len(np.unique(cc['core_tag'][satellites_mask])), 'Satellite core tags not unique for this time step.'
# Initialize M array (corresponds with cc[satellites_mask]) to be host tree node mass of each satellite
idx_m21 = many_to_one(cc['tree_node_index'][satellites_mask],
cc['tree_node_index'][centrals_mask])
M, X, Y, Z = (cc[k][centrals_mask][idx_m21]
for k in ['infall_mass', 'x', 'y', 'z'])
for A in A_arr:
for zeta in zeta_arr:
# Set m_evolved of all satellites that have core tag match on prev step to next_m_evolved of prev step.
# DOUBLE MASK ASSIGNMENT: cc['m_evolved'][satellites_mask][idx1] = cc_prev['next_m_evolved'][idx2]
cc[m_evolved_col(A, zeta)][np.flatnonzero(satellites_mask)
[idx1]] = cc_prev[m_evolved_col(
A, zeta, next=True)][idx2]
# Initialize m array (corresponds with cc[satellites_mask]) to be either infall_mass (step after infall) or m_evolved (subsequent steps).
initMask = cc[m_evolved_col(A, zeta)][satellites_mask] == 0
minfall = cc['infall_mass'][satellites_mask][initMask]
cc[m_evolved_col(A, zeta)][np.flatnonzero(
satellites_mask)[initMask]] = SHMLM.m_evolved(
m0=minfall,
M0=M[initMask],
step=step,
step_prev=steps[steps.index(step) - 1],
A=A,
zeta=zeta,
dtFactorFlag=True)
# m = (cc[m_evolved_col(A, zeta)][satellites_mask] == 0)*cc['infall_mass'][satellites_mask] + (cc[m_evolved_col(A, zeta)][satellites_mask] != 0)*cc[m_evolved_col(A, zeta)][satellites_mask]
# Set m_evolved of satellites with m_evolved=0 to infall mass.
# cc[m_evolved_col(A, zeta)][satellites_mask] = m
'''
x, y, z = cc['x'].copy(), cc['y'].copy(), cc['z'].copy()
x[satellites_mask] = SHMLM.periodic_bcs(cc['x'][satellites_mask], X, SHMLM.BOXSIZE)
y[satellites_mask] = SHMLM.periodic_bcs(cc['y'][satellites_mask], Y, SHMLM.BOXSIZE)
z[satellites_mask] = SHMLM.periodic_bcs(cc['z'][satellites_mask], Z, SHMLM.BOXSIZE)
nonMergeMask = np.flatnonzero(cc['phaseSpaceMerged']==0)
c1, c2 = np.array( list(combinations(nonMergeMask, 2)) ).T #c1, c2 are indices relative of cc
Delta_x = ((x[c1]-x[c2])**2 + (y[c1]-y[c2])**2 + (z[c1]-z[c2])**2)**0.5
Delta_v = ((cc['vx'][c1]-cc['vx'][c2])**2 + (cc['vy'][c1]-cc['vy'][c2])**2 + (cc['vz'][c1]-cc['vz'][c2])**2)**0.5
mass1bigger = cc['infall_mass'][c1] > cc['infall_mass'][c2]
massbiggeridx = np.where(mass1bigger, c1, c2)
masssmalleridx = np.where(mass1bigger, c2, c1)
sigma_x = 3*cc['radius'][massbiggeridx]
sigma_v = 3*cc['vel_disp'][massbiggeridx]
mergeFound = ((Delta_x/sigma_x + Delta_v/sigma_v) < 2)
cc['phaseSpaceMerged'][np.unique(masssmalleridx[mergeFound])] = 1
print "Step", step
print "Merged pairs found: {} out of {} pairs ({}%)".format( np.sum(mergeFound), len(mergeFound), np.sum(mergeFound)/len(mergeFound)*100. )
print "Total number of cores marked as merged:", np.sum(cc['phaseSpaceMerged']==1)
print "Number of central cores marked as merged:", np.sum(cc['phaseSpaceMerged'][centrals_mask]==1)
cc['phaseSpaceMerged'][centrals_mask] = 0
'''
if writeOutputFlag:
# Write output to disk
h5_write_dict(fname_cc(step, 'output'), cc, 'coredata')
# Compute m_evolved of satellites according to SHMLModel for NEXT time step and save as cc_prev['next_m_evolved'] in memory.
# Mass loss assumed to begin at step AFTER infall detected.
if step != steps[-1]:
# cc_prev = { k:cc[k].copy() for k in ['core_tag', 'wasInFragment', 'phaseSpaceMerged'] }
cc_prev = {'core_tag': cc['core_tag'].copy()}
if numSatellites != 0:
localhost_m21 = many_to_one(cc['host_core'][satellites_mask],
cc['core_tag'])
for A in A_arr:
for zeta in zeta_arr:
cc_prev[m_evolved_col(A, zeta, next=True)] = np.zeros_like(
cc['infall_mass'])
if numSatellites != 0: # If there are satellites (not applicable for first step)
m = cc[m_evolved_col(A, zeta)][satellites_mask]
if useLocalHost:
Mlocal = cc[m_evolved_col(A, zeta)][localhost_m21]
M_A_zeta = (Mlocal
== 0) * M + (Mlocal != 0) * Mlocal
cc_prev[m_evolved_col(
A, zeta,
next=True)][satellites_mask] = SHMLM.m_evolved(
m0=m,
M0=M_A_zeta,
step=steps[steps.index(step) + 1],
step_prev=step,
A=A,
zeta=zeta)
else:
cc_prev[m_evolved_col(
A, zeta,
next=True)][satellites_mask] = SHMLM.m_evolved(
m0=m,
M0=M,
step=steps[steps.index(step) + 1],
step_prev=step,
A=A,
zeta=zeta)
return cc
import sys
import numpy as np
import pandas as pd
sys.path.append("/home/pascal/projects/VizAly-Vis_IO/genericio/python")
import genericio as gio
## Input Section
input_file_name = "/bigData/Halos/b0168/m001-499.sodproperties"
query = "['fof_halo_count'] > 400000"
display_values = [
'fof_halo_tag', 'fof_halo_count', 'fof_halo_center_x', 'fof_halo_center_y',
'fof_halo_center_y'
]
##### Script Starts #####
num_vars = gio.gio_get_num_variables(input_file_name)
# Read in data for each variable
df = pd.DataFrame()
for i in range(num_vars):
var_name = gio.gio_get_variable(input_file_name, i)
data = gio.gio_read(input_file_name, var_name)
df.insert(i, var_name, data)
query_string = "df" + query
row_selection = eval(query_string)
col_selection = display_values
print(df.loc[row_selection, col_selection])
# permission.
#
# *****************************************************************************
#
# DISCLAIMER
# THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND. NEITHER THE
# UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR
# UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY,
# EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE
# ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS,
# PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE
# PRIVATELY OWNED RIGHTS.
#
# *****************************************************************************
import sys
import numpy as np
import genericio as gio
name = sys.argv[1]
gio.gio_inspect(name)
if gio.gio_has_variable(name, "x"):
x = gio.gio_read(name, "x")
y = gio.gio_read(name, "y")
z = gio.gio_read(name, "z")
print (np.column_stack((x, y, z)))
else:
pos = gio.gio_read(name, "pos")
print (pos)
# File
nroot = halodir + 'HaloCatalog/STEP' + str(istep)
files = glob.glob(nroot + '/*' + str(istep) + '*.fofproperties#' + ifile)
if (len(files) > 1):
print files
print 'STOP: More than one file for ifile ', ifile
sys.exit()
else:
infile = files[0]
gio.gio_inspect(infile)
print '----------------------------'
print 'Output: ', outfile
# Read the file
count = gio.gio_read(infile, "fof_halo_count")
tag = gio.gio_read(infile, "fof_halo_tag")
mass = gio.gio_read(infile, "fof_halo_mass")
xc = gio.gio_read(infile, "fof_halo_center_x")
yc = gio.gio_read(infile, "fof_halo_center_y")
zc = gio.gio_read(infile, "fof_halo_center_z")
xm = gio.gio_read(infile, "fof_halo_mean_x")
ym = gio.gio_read(infile, "fof_halo_mean_y")
zm = gio.gio_read(infile, "fof_halo_mean_z")
vx = gio.gio_read(infile, "fof_halo_mean_vx")
vy = gio.gio_read(infile, "fof_halo_mean_vy")
vz = gio.gio_read(infile, "fof_halo_mean_vz")
# Output
size = len(vz)
def dndmu(file1):
print file1
gio.gio_inspect(file1)
infall_mass = gio.gio_read(file1, 'infall_mass').flatten()
infall_step = gio.gio_read(file1, 'infall_step').flatten()
host_tag = gio.gio_read(file1, 'fof_halo_tag').flatten()
host_mass = infall_mass
infall_mass = infall_mass[host_mass >= 1.e12]
host_tag = host_tag[host_mass >= 1.e12]
infall_step = infall_step[host_mass >= 1.e12]
host_tag_u = host_tag[infall_step == 499]
host_mass = host_mass[host_mass >= 1.e12][infall_step == 499]
dict_values = dict(zip(host_tag_u, host_mass))
infall_mass = infall_mass[infall_step != 499]
host_tag = host_tag[infall_step != 499]
def apply_dict(i):
return dict_values[i]
print(host_tag[0])
print(dict_values[host_tag[0]])
print(apply_dict(host_tag[0]))
mass_tot = np.array(map(apply_dict, host_tag))
ratio_tot = infall_mass / mass_tot
print(len(mass_tot))
print(len(ratio_tot))
print(mass_tot[:100])
'''
print(len(host_tag_u))
ratio_tot=np.array([]).astype(float)
mass_tot=np.array([]).astype(float)
for i in range(len(host_tag_u)):
if i%1000==0:
print(i)
core_masses = infall_mass[(host_tag==host_tag_u[i])&(infall_step!=499)]
ratio = core_masses/host_mass[i]
mass = ratio*0.0 + host_mass[i]
ratio_tot = np.concatenate((ratio_tot,ratio))
mass_tot = np.concatenate((mass_tot,mass))
print(len(host_mass))
'''
bin12 = (mass_tot > 1.e12) & (mass_tot < 5.e12)
bin13 = (mass_tot > 1.e13) & (mass_tot < 5.e13)
bin14 = (mass_tot > 1.e14) & (mass_tot < 5.e14)
bin125 = (mass_tot > 5.e12) & (mass_tot < 1.e13)
bin135 = (mass_tot > 5.e13) & (mass_tot < 1.e14)
bin145 = (mass_tot > 5.e14) & (mass_tot < 1.e15)
print(np.sum(bin12), np.sum(bin125), np.sum(bin13), np.sum(bin135),
np.sum(bin14), np.sum(bin145))
bins = np.logspace(-2, 0, 100)
hist12, edges = np.histogram(
ratio_tot[bin12], normed=True,
bins=bins) #np.logspace(-1,0,100))#np.linspace(0.0,1.0,100))
hist13, edges = np.histogram(
ratio_tot[bin13], normed=True,
bins=bins) #np.logspace(-1,0,100))#linspace(0.0,1.0,100))
hist14, edges = np.histogram(
ratio_tot[bin14], normed=False,
bins=bins) #np.logspace(-1,0,100))#inspace(0.0,1.0,100))
hist125, edges = np.histogram(
ratio_tot[bin125], normed=True,
bins=bins) # np.logspace(-1,0,100))#np.linspace(0.0,1.0,100))
hist135, edges = np.histogram(
ratio_tot[bin135], normed=False,
bins=bins) #np.logspace(-1,0,100))#linspace(0.0,1.0,100))
hist145, edges = np.histogram(
ratio_tot[bin145], normed=True,
bins=bins) #np.logspace(-3,0,100))#inspace(0.0,1.0,100))
#plt.plot(edges[1:], hist12,label='bin 12')
#plt.plot(edges[1:], hist13,label = 'bin 13')
plt.plot(edges[1:], hist14, label='bin 14')
#plt.plot(edges[1:], hist125,label='bin 12.5')
plt.plot(edges[1:], hist135, label='bin 13.5')
#plt.plot(edges[1:], hist145,label= 'bin 14.5')
plt.yscale('log')
plt.xscale('log')
plt.legend()
plt.xlabel('mass ratio')
plt.ylabel('N_cores (normalized)')
plt.show()
bins = np.logspace(-0.5, 0, 100)
hist12, edges = np.histogram(
ratio_tot[bin12], normed=True,
bins=bins) #np.logspace(-1,0,100))#np.linspace(0.0,1.0,100))
hist13, edges = np.histogram(
ratio_tot[bin13], normed=True,
bins=bins) #np.logspace(-1,0,100))#linspace(0.0,1.0,100))
hist14, edges = np.histogram(
ratio_tot[bin14], normed=True,
bins=bins) #np.logspace(-1,0,100))#inspace(0.0,1.0,100))
hist125, edges = np.histogram(
ratio_tot[bin125], normed=True,
bins=bins) # np.logspace(-1,0,100))#np.linspace(0.0,1.0,100))
hist135, edges = np.histogram(
ratio_tot[bin135], normed=True,
bins=bins) #np.logspace(-1,0,100))#linspace(0.0,1.0,100))
hist145, edges = np.histogram(
ratio_tot[bin145], normed=True,
bins=bins) #np.logspace(-3,0,100))#inspace(0.0,1.0,100))
plt.plot(edges[1:], hist12, label='bin 12')
plt.plot(edges[1:], hist13, label='bin 13')
# plt.plot(edges[1:], hist14,label= 'bin 14')
plt.plot(edges[1:], hist125, label='bin 12.5')
plt.plot(edges[1:], hist135, label='bin 13.5')
#plt.plot(edges[1:], hist145,label= 'bin 14.5')
plt.yscale('log')
plt.xscale('log')
plt.legend()
plt.xlabel('mass ratio')
plt.ylabel('N_cores (normalized)')
plt.show()
# except:
# print('test_failed')
return
def central_flip(file1, file2, step):
''' Check for flipping of centrals from one step to the next
This test is in development, and may need to read in the accum files to see if there is truly missing substructures
'''
try:
x = gio.gio_read(file1, 'x').flatten()
y = gio.gio_read(file1, 'y').flatten()
z = gio.gio_read(file1, 'z').flatten()
halo_tag = gio.gio_read(file1, 'fof_halo_tag').flatten()
infall_step = gio.gio_read(file1, 'infall_step').flatten()
infall_mass = gio.gio_read(file1, 'infall_mass').flatten()
core_tag = gio.gio_read(file1, "core_tag").flatten()
core_tag2 = gio.gio_read(file2, "core_tag").flatten()
x2 = gio.gio_read(file2, 'x').flatten()
y2 = gio.gio_read(file2, 'y').flatten()
z2 = gio.gio_read(file2, 'z').flatten()
infall_mass2 = gio.gio_read(file2, 'infall_mass').flatten()
centrals = (infall_step == step)
fof_mass_mask = (
infall_mass[centrals] > 500. * 1.1e9
) # at least 500 particles (let's not bother with low mass halos for now)
nhalos = len(halo_tag[centrals][fof_mass_mask])
print("Number of halos above mass cut = " + str(nhalos))
count = 0
maxcount = min(nhalos, 5000)
distarr = []
for j in range(maxcount):
fof_halo_tag = halo_tag[centrals][fof_mass_mask][j]
mask_fof = (halo_tag == fof_halo_tag)
flip, dist = _halo_central_flip_light(
x[mask_fof], y[mask_fof], z[mask_fof], infall_step[mask_fof],
infall_mass[mask_fof], core_tag[mask_fof], step, fof_halo_tag)
# uncomment if you want for the worst cases to see comparison plots with the previous step
#flip,dist = _halo_central_flip_detailed(x[mask_fof],y[mask_fof],z[mask_fof],infall_step[mask_fof],infall_mass[mask_fof],core_tag[mask_fof],step,x2,y2,z2,core_tag2,infall_mass2,fof_halo_tag)
if flip:
count += 1
distarr.append(dist)
distarr = np.array(distarr)
np.savetxt('/media/luna1/prlarsen/dist1.txt', np.array(distarr))
print("Number of pairs closer than 30kpc = " +
str(np.sum(distarr < 0.03)) + " out of a possible " +
str(maxcount))
except:
print("test_failed")
return
import itertools as it
massbinsize = 0.5
step = 499
#step = 247
assert SHMLM_HM.step2z[step] == SHMLM_SV.step2z[step], "Redshifts of HM and SV don't match."
z = SHMLM_HM.step2z[step]
cc_HM = h5_read_dict('/home/isultan/projects/halomassloss/core_catalog_mevolved/output_ALCC_localhost_dtfactor_0.5/{}.corepropertiesextend.hdf5'.format(step), 'coredata')
cc_SV = h5_read_dict('/home/isultan/projects/halomassloss/core_catalog_mevolved/output_LJDS_localhost_dtfactor_0.5/m000p-{}.corepropertiesextend.hdf5'.format(step), 'coredata')
# mt_cores = gio_read_dict('/home/isultan/data/LJDS/MergerTrees_updated/m000p-{}.treenodes'.format(step), ['fof_halo_tag', 'fof_halo_mass', 'fof_halo_center_x', 'fof_halo_center_y', 'fof_halo_center_z'])
sh = gio_read_dict('/home/isultan/data/LJDS/subhalos/m000p-{}.subhaloproperties'.format(step), ['fof_halo_tag','subhalo_mean_x','subhalo_mean_y','subhalo_mean_z','subhalo_mean_vx', 'subhalo_mean_vy', 'subhalo_mean_vz', 'subhalo_count', 'subhalo_tag', 'subhalo_mass'])
mt = gio_read_dict('/home/isultan/data/LJDS/subhalos/m000p-{}.haloproperties'.format(step), ['fof_halo_tag', 'fof_halo_mass', 'fof_halo_center_x', 'fof_halo_center_y', 'fof_halo_center_z'])
cc_HM['infall_mass'] = gio.gio_read('/home/isultan/data/ALCC/CoreCatalog/{}.coreproperties'.format(step), 'infall_fof_halo_mass')[0] #cc_HM['infall_fof_halo_mass']
cc_SV['infall_mass'] = cc_SV['infall_fof_halo_mass']
# Subhalo-core matching: to look at the subhalo mass:core evolved mass ratio
def generate_cores_kdtree(M1, M2, cc, SHMLM, s1=False, disrupt=None, onlyFiltered=False, giveFragmentsFofMass=False, mt_cores=None):
# idx_filteredsatcores, M, X, Y, Z, nHalo = SHMLM.core_mask(cc, M1=10**8, M2=10**16, s1=s1, disrupt=disrupt, z=z)
idx_filteredsatcores, M, X, Y, Z, nHalo = SHMLM.core_mask(cc, M1, M2, s1=s1, disrupt=disrupt, z=z)
cc_filtered = { k:cc[k][idx_filteredsatcores].copy() for k in cc.keys() }
cc_filtered['M'] = M
if giveFragmentsFofMass:
fragmask = cc_filtered['fof_halo_tag']<0
frag_realfht = np.bitwise_and(cc_filtered['fof_halo_tag'][fragmask]*-1, 0xffffffffffff)
idx_m21 = many_to_one( frag_realfht, mt_cores['fof_halo_tag'] )
cc_filtered['M'][fragmask] = mt_cores['fof_halo_mass'][idx_m21]
cc_filtered['x'] = SHMLM.periodic_bcs(cc_filtered['x'], X, SHMLM.BOXSIZE)
def create_core_catalog_mevolved(virialFlag,
writeOutputFlag=True,
useLocalHost=False):
"""
Appends mevolved to core catalog and saves output in HDF5.
Works by computing mevolved for step+1 at each step and saving that in memory.
"""
if writeOutputFlag:
print 'Reading data from {} and writing output to {}'.format(
cc_data_dir, cc_output_dir)
global cc, coresMaskedArray, distance_upper_bound, pool_mergedCoreTag, coresKDTree, KDTree_idx
cc = {}
cc_prev = {}
for step in tqdm(steps):
# Read in cc for step
cc = {v: gio.gio_read(fname_cc(step, 'input'), v)[0] for v in vars_cc}
if virialFlag:
# Convert all mass to virial
cc['infall_mass'] = SHMLM.m_vir(cc['infall_mass'], step)
satellites_mask = cc['central'] == 0
centrals_mask = cc['central'] == 1
# assert np.sum(satellites_mask) + np.sum(centrals_mask) == len(cc['infall_mass']), 'central flag not 0 or 1.'
numSatellites = np.sum(satellites_mask)
# Verify there are no satellites at first step
if step == steps[0]:
assert numSatellites == 0, 'Satellites found at first step.'
# Add column for m_evolved and initialize to 0
for A in A_arr:
for zeta in zeta_arr:
cc[m_evolved_col(A, zeta)] = np.zeros_like(cc['infall_mass'])
cc['mergedCoreTag'] = np.zeros_like(cc['core_tag'])
cc['mergedCoreStep'] = np.zeros_like(cc['infall_step'])
# wasInFragment: persistent flag that is False by default and becomes True when core is inside a fragment halo
cc['wasInFragment'] = cc['fof_halo_tag'] < 0
if step != steps[0]:
_, i1, i2 = np.intersect1d(cc['core_tag'],
cc_prev['core_tag'],
return_indices=True)
cc['wasInFragment'][i1] = np.logical_or(
cc['wasInFragment'][i1], cc_prev['wasInFragment'][i2])
cc['mergedCoreTag'][i1] = cc_prev['mergedCoreTag'][i2]
cc['mergedCoreStep'][i1] = cc_prev['mergedCoreStep'][i2]
# If there are satellites (not applicable for first step)
if numSatellites != 0:
# Match satellites to prev step cores using core tag.
vals, idx1, idx2 = np.intersect1d(cc['core_tag'][satellites_mask],
cc_prev['core_tag'],
return_indices=True)
# assert len(vals) == len(cc['core_tag'][satellites_mask]), 'All cores from prev step did not carry over.'
# assert len(cc['core_tag'][satellites_mask]) == len(np.unique(cc['core_tag'][satellites_mask])), 'Satellite core tags not unique for this time step.'
for A in A_arr:
for zeta in zeta_arr:
# Set m_evolved of all satellites that have core tag match on prev step to next_m_evolved of prev step.
# DOUBLE MASK ASSIGNMENT: cc['m_evolved'][satellites_mask][idx1] = cc_prev['next_m_evolved'][idx2]
cc[m_evolved_col(A, zeta)][np.flatnonzero(satellites_mask)
[idx1]] = cc_prev[m_evolved_col(
A, zeta, next=True)][idx2]
# Initialize m array (corresponds with cc[satellites_mask]) to be either infall_mass (step after infall) or m_evolved (subsequent steps).
m = (cc[m_evolved_col(A, zeta)][satellites_mask]
== 0) * cc['infall_mass'][satellites_mask] + (
cc[m_evolved_col(A, zeta)][satellites_mask] !=
0) * cc[m_evolved_col(A, zeta)][satellites_mask]
# Set m_evolved of satellites with m_evolved=0 to infall mass.
cc[m_evolved_col(A, zeta)][satellites_mask] = m
# Initialize M array (corresponds with cc[satellites_mask]) to be host tree node mass of each satellite
idx_m21 = many_to_one(cc['tree_node_index'][satellites_mask],
cc['tree_node_index'][centrals_mask])
M, X, Y, Z = (cc[k][centrals_mask][idx_m21]
for k in ['infall_mass', 'x', 'y', 'z'])
KDTreemask = cc['mergedCoreTag'][satellites_mask] == 0
x = SHMLM.periodic_bcs(cc['x'][satellites_mask][KDTreemask],
X[KDTreemask], SHMLM.BOXSIZE)
y = SHMLM.periodic_bcs(cc['y'][satellites_mask][KDTreemask],
Y[KDTreemask], SHMLM.BOXSIZE)
z = SHMLM.periodic_bcs(cc['z'][satellites_mask][KDTreemask],
Z[KDTreemask], SHMLM.BOXSIZE)
# coresMaskedArray = np.vstack((x, y, z)).T
coresMaskedArray_Tree = np.vstack(
(np.r_[x,
cc['x'][centrals_mask]], np.r_[y,
cc['y'][centrals_mask]],
np.r_[z, cc['z'][centrals_mask]])).T
coresMaskedArray = coresMaskedArray_Tree
KDTree_idx = np.r_[np.flatnonzero(satellites_mask)[KDTreemask],
np.flatnonzero(centrals_mask)]
coresKDTree = spatial.cKDTree(coresMaskedArray_Tree)
# Set mergedCoreTag as central core of KDTreemask cores with no mergedCoreTag that are `merged`
'''
disruptMask = cc['merged'][satellites_mask][KDTreemask]|(cc['radius'][satellites_mask][KDTreemask] >= SHMLM.CORERADIUSCUT)
cc['mergedCoreTag'][ np.flatnonzero(satellites_mask)[KDTreemask] ] += disruptMask*cc['core_tag'][centrals_mask][idx_m21][KDTreemask]
cc['mergedCoreStep'][ np.flatnonzero(satellites_mask)[KDTreemask] ] += disruptMask*step
'''
### core merging detection
# rvir_KDTreemask = SHMLM.getRvir(cc[m_evolved_col(1.1, 0.068)][satellites_mask][KDTreemask], SHMLM.step2z[step]) #infall should be exact m_evolved in production
# rvir_KDTreemask = SHMLM.getRvir(cc['infall_mass'][satellites_mask][KDTreemask], SHMLM.step2z[step]) #infall should be exact m_evolved in production
# distance_upper_bound = SHMLM.COREVIRIALRADIUSFACTOR * rvir_KDTreemask
distance_upper_bound = SHMLM.COREVIRIALRADIUSFACTOR * SHMLM.getRvir(
cc['infall_mass'][KDTree_idx], SHMLM.step2z[step])
import ctypes
# pool_mergedCoreTag = { i:np.full(len(KDTree_idx), 0, dtype=np.int64) for i in range(1,pool_size+1) }
pool_mergedCoreTag = {
i: multiprocessing.RawArray(
ctypes.c_int64, np.zeros(len(KDTree_idx), dtype=np.int64))
for i in range(1, pool_size + 1)
}
# Search radius of 2*rvir around each core
from datetime import datetime
tstart = datetime.now()
work = np.arange(len(distance_upper_bound))
t1 = datetime.now() - tstart
print "work=", t1
tstart = datetime.now()
p = Pool(pool_size)
t2 = datetime.now() - tstart
print "Pool=", t2
tstart = datetime.now()
p.map(query, np.array_split(work, pool_size))
t3 = datetime.now() - tstart
print "mapp=", t3
tstart = datetime.now()
p.close()
t4 = datetime.now() - tstart
print "close=", t4
tstart = datetime.now()
p.join()
t5 = datetime.now() - tstart
print "join=", t5
tstart = datetime.now()
# mergedCoreTag = np.full_like(pool_mergedCoreTag[1], 0)
mergedCoreTag = np.zeros(len(KDTree_idx), dtype=np.int64)
# pool_mergedCoreTag = {k:np.array(pool_mergedCoreTag[k]) for k in pool_mergedCoreTag.keys()}
for i in range(1, pool_size + 1):
mCT = np.array(pool_mergedCoreTag[i])
newMergedMask = mCT != 0
# print np.sum(newMergedMask)
mergedCoreTag[newMergedMask] = mCT[newMergedMask]
t6 = datetime.now() - tstart
newMergedMask = mergedCoreTag != 0
cc['mergedCoreTag'][
KDTree_idx[newMergedMask]] = mergedCoreTag[newMergedMask]
cc['mergedCoreStep'][KDTree_idx[newMergedMask]] = step
print np.sum(cc['mergedCoreTag'] != 0)
t7 = datetime.now() - tstart
print t1, t2, t3, t4, t5, t6, t7
###
print 'Merged centrals: ', np.sum(
cc['mergedCoreTag'][centrals_mask] != 0)
cc['mergedCoreTag'][centrals_mask] = 0
cc['mergedCoreStep'][centrals_mask] = 0
if writeOutputFlag:
# Write output to disk
h5_write_dict(fname_cc(step, 'output'), cc, 'coredata')
# Compute m_evolved of satellites according to SHMLModel for NEXT time step and save as cc_prev['next_m_evolved'] in memory.
# Mass loss assumed to begin at step AFTER infall detected.
if step != steps[-1]:
# cc_prev = { 'core_tag':cc['core_tag'].copy(), 'wasInFragment':cc['wasInFragment'].copy() }
cc_prev = {
k: cc[k].copy()
for k in [
'core_tag', 'wasInFragment', 'mergedCoreTag',
'mergedCoreStep'
]
}
if numSatellites != 0:
localhost_m21 = many_to_one(cc['host_core'][satellites_mask],
cc['core_tag'])
for A in A_arr:
for zeta in zeta_arr:
cc_prev[m_evolved_col(A, zeta, next=True)] = np.zeros_like(
cc['infall_mass'])
if numSatellites != 0: # If there are satellites (not applicable for first step)
m = cc[m_evolved_col(A, zeta)][satellites_mask]
if useLocalHost:
Mlocal = cc[m_evolved_col(A, zeta)][localhost_m21]
M_A_zeta = (Mlocal
== 0) * M + (Mlocal != 0) * Mlocal
cc_prev[m_evolved_col(
A, zeta,
next=True)][satellites_mask] = SHMLM.m_evolved(
m0=m,
M0=M_A_zeta,
step=steps[steps.index(step) + 1],
step_prev=step,
A=A,
zeta=zeta)
else:
cc_prev[m_evolved_col(
A, zeta,
next=True)][satellites_mask] = SHMLM.m_evolved(
m0=m,
M0=M,
step=steps[steps.index(step) + 1],
step_prev=step,
A=A,
zeta=zeta)
return cc
def create_core_catalog_mevolved(writeOutputFlag=True, useLocalHost=False):
"""
Appends mevolved to core catalog and saves output in HDF5.
Works by computing mevolved for step+1 at each step and saving that in memory.
"""
if writeOutputFlag:
print 'Reading data from {} and writing output to {}'.format(
cc_data_dir, cc_output_dir)
cc = {}
cc_prev = {}
for step in tqdm(steps):
# Read in cc for step
cc = {
v: gio.gio_read(fname_cc(step, 'input'), v)[0]
for v in vars_cc(step)
}
satellites_mask = cc['central'] == 0
centrals_mask = cc['central'] == 1
# assert np.sum(satellites_mask) + np.sum(centrals_mask) == len(cc['infall_tree_node_mass']), 'central flag not 0 or 1.'
numSatellites = np.sum(satellites_mask)
# Verify there are no satellites at first step
if step == steps[0]:
assert numSatellites == 0, 'Satellites found at first step.'
# Add column for m_evolved and initialize to 0
cc[m_evolved_col(A, zeta)] = np.zeros_like(cc['infall_tree_node_mass'])
# If there are satellites (not applicable for first step)
if numSatellites != 0:
# Match satellites to prev step cores using core tag.
vals, idx1, idx2 = np.intersect1d(cc['core_tag'][satellites_mask],
cc_prev['core_tag'],
return_indices=True)
idx_m21 = many_to_one(cc['tree_node_index'][satellites_mask],
cc['tree_node_index'][centrals_mask])
M = cc['infall_tree_node_mass'][centrals_mask][idx_m21]
# Set m_evolved of all satellites that have core tag match on prev step to next_m_evolved of prev step.
cc[m_evolved_col(A, zeta)][np.flatnonzero(satellites_mask)
[idx1]] = cc_prev[m_evolved_col(
A, zeta, next=True)][idx2]
# Initialize m array (corresponds with cc[satellites_mask]) to be either infall_mass (step after infall) or m_evolved (subsequent steps).
initMask = cc[m_evolved_col(A, zeta)][satellites_mask] == 0
minfall = cc['infall_tree_node_mass'][satellites_mask][initMask]
cc[m_evolved_col(A, zeta)][np.flatnonzero(
satellites_mask)[initMask]] = SHMLM.m_evolved(
m0=minfall,
M0=M[initMask],
step=step,
step_prev=steps[steps.index(step) - 1],
A=A,
zeta=zeta,
dtFactorFlag=True)
if writeOutputFlag and (step == 499 or step == 247):
# Write output to disk
h5_write_dict(fname_cc(step, 'output'), cc, 'coredata')
# Compute m_evolved of satellites according to SHMLModel for NEXT time step and save as cc_prev['next_m_evolved'] in memory.
# Mass loss assumed to begin at step AFTER infall detected.
if step != steps[-1]:
cc_prev = {'core_tag': cc['core_tag'].copy()}
if numSatellites != 0:
localhost_m21 = many_to_one(cc['host_core'][satellites_mask],
cc['core_tag'])
cc_prev[m_evolved_col(A, zeta, next=True)] = np.zeros_like(
cc['infall_tree_node_mass'])
if numSatellites != 0: # If there are satellites (not applicable for first step)
m = cc[m_evolved_col(A, zeta)][satellites_mask]
if useLocalHost:
Mlocal = cc[m_evolved_col(A, zeta)][localhost_m21]
M_A_zeta = (Mlocal == 0) * M + (Mlocal != 0) * Mlocal
cc_prev[m_evolved_col(
A, zeta,
next=True)][satellites_mask] = SHMLM.m_evolved(
m0=m,
M0=M_A_zeta,
step=steps[steps.index(step) + 1],
step_prev=step,
A=A,
zeta=zeta)
else:
cc_prev[m_evolved_col(
A, zeta,
next=True)][satellites_mask] = SHMLM.m_evolved(
m0=m,
M0=M,
step=steps[steps.index(step) + 1],
step_prev=step,
A=A,
zeta=zeta)
return cc
def create_core_catalog_mevolved(virialFlag=False,
writeOutputFlag=True,
useLocalHost=False,
checkpointRestart=None):
"""
Appends mevolved to core catalog and saves output in HDF5.
Works by computing mevolved for step+1 at each step and saving that in memory.
"""
if writeOutputFlag:
print 'Reading data from {} and writing output to {}'.format(
cc_data_dir, cc_output_dir)
cc = {}
cc_prev = {}
if checkpointRestart is None:
itersteps = steps
else:
itersteps = steps[steps.index(checkpointRestart):]
for step in tqdm(itersteps):
# Read in cc for step
if checkpointRestart is None:
cc = {
v: gio.gio_read(fname_cc(step, 'input'), v)[0]
for v in vars_cc
}
if virialFlag:
# Convert all mass to virial
cc['infall_tree_node_mass'] = SHMLM.m_vir(
cc['infall_tree_node_mass'], step)
else:
cc = h5_read_dict(fname_cc(checkpointRestart, 'output'),
'coredata')
satellites_mask = cc['central'] == 0
centrals_mask = cc['central'] == 1
# assert np.sum(satellites_mask) + np.sum(centrals_mask) == len(cc['infall_tree_node_mass']), 'central flag not 0 or 1.'
numSatellites = np.sum(satellites_mask)
# Verify there are no satellites at first step
if step == steps[0]:
assert numSatellites == 0, 'Satellites found at first step.'
# Add column for m_evolved and initialize to 0
if checkpointRestart is None:
for A in A_arr:
for zeta in zeta_arr:
cc[m_evolved_col(A, zeta)] = np.zeros_like(
cc['infall_tree_node_mass'])
# If there are satellites (not applicable for first step)
if numSatellites != 0:
if checkpointRestart is None:
# Match satellites to prev step cores using core tag.
vals, idx1, idx2 = np.intersect1d(
cc['core_tag'][satellites_mask],
cc_prev['core_tag'],
return_indices=True)
# assert len(vals) == len(cc['core_tag'][satellites_mask]), 'All cores from prev step did not carry over.'
# assert len(cc['core_tag'][satellites_mask]) == len(np.unique(cc['core_tag'][satellites_mask])), 'Satellite core tags not unique for this time step.'
# Initialize M array (corresponds with cc[satellites_mask]) to be host tree node mass of each satellite
idx_m21 = many_to_one(cc['tree_node_index'][satellites_mask],
cc['tree_node_index'][centrals_mask])
M, X, Y, Z = (cc[k][centrals_mask][idx_m21]
for k in ['infall_tree_node_mass', 'x', 'y', 'z'])
if checkpointRestart is None:
for A in A_arr:
for zeta in zeta_arr:
# Set m_evolved of all satellites that have core tag match on prev step to next_m_evolved of prev step.
# DOUBLE MASK ASSIGNMENT: cc['m_evolved'][satellites_mask][idx1] = cc_prev['next_m_evolved'][idx2]
cc[m_evolved_col(A, zeta)][np.flatnonzero(
satellites_mask)[idx1]] = cc_prev[m_evolved_col(
A, zeta, next=True)][idx2]
initMask = cc[m_evolved_col(
A, zeta)][satellites_mask] == 0
minfall = cc['infall_tree_node_mass'][satellites_mask][
initMask]
cc[m_evolved_col(A, zeta)][np.flatnonzero(
satellites_mask)[initMask]] = SHMLM.m_evolved(
m0=minfall,
M0=M[initMask],
step=step,
step_prev=steps[steps.index(step) - 1],
A=A,
zeta=zeta,
dtFactorFlag=True)
if writeOutputFlag and (checkpointRestart is None):
# Write output to disk
h5_write_dict(fname_cc(step, 'output'), cc, 'coredata')
#cc should be checkpointRestart step file if not None
if checkpointRestart is not None:
checkpointRestart = None
# Compute m_evolved of satellites according to SHMLModel for NEXT time step and save as cc_prev['next_m_evolved'] in memory.
if step != steps[-1]:
cc_prev = {'core_tag': cc['core_tag'].copy()}
if numSatellites != 0:
localhost_m21 = many_to_one(cc['host_core'][satellites_mask],
cc['core_tag'])
for A in A_arr:
for zeta in zeta_arr:
cc_prev[m_evolved_col(A, zeta, next=True)] = np.zeros_like(
cc['infall_tree_node_mass'])
if numSatellites != 0: # If there are satellites (not applicable for first step)
m = cc[m_evolved_col(A, zeta)][satellites_mask]
if useLocalHost:
Mlocal = cc[m_evolved_col(A, zeta)][localhost_m21]
M_A_zeta = (Mlocal
== 0) * M + (Mlocal != 0) * Mlocal
cc_prev[m_evolved_col(
A, zeta,
next=True)][satellites_mask] = SHMLM.m_evolved(
m0=m,
M0=M_A_zeta,
step=steps[steps.index(step) + 1],
step_prev=step,
A=A,
zeta=zeta)
else:
cc_prev[m_evolved_col(
A, zeta,
next=True)][satellites_mask] = SHMLM.m_evolved(
m0=m,
M0=M,
step=steps[steps.index(step) + 1],
step_prev=step,
A=A,
zeta=zeta)
return cc
def create_core_catalog_mevolved(virialFlag, writeOutputFlag=True, useLocalHost=False):
"""
Appends mevolved to core catalog and saves output in HDF5.
Works by computing mevolved for step+1 at each step and saving that in memory.
"""
if writeOutputFlag:
print 'Reading data from {} and writing output to {}'.format(cc_data_dir, cc_output_dir)
cc = {}
cc_prev = {}
for step in tqdm(steps):
# Read in cc for step
cc = { v:gio.gio_read(fname_cc(step, 'input'), v)[0] for v in vars_cc }
if virialFlag:
# Convert all mass to virial
cc['infall_mass'] = SHMLM.m_vir(cc['infall_mass'], step)
satellites_mask = cc['central'] == 0
centrals_mask = cc['central'] == 1
# assert np.sum(satellites_mask) + np.sum(centrals_mask) == len(cc['infall_mass']), 'central flag not 0 or 1.'
numSatellites = np.sum(satellites_mask)
# Verify there are no satellites at first step
if step == steps[0]:
assert numSatellites == 0, 'Satellites found at first step.'
# Add column for m_evolved and initialize to 0
for A in A_arr:
for zeta in zeta_arr:
cc[m_evolved_col(A, zeta)] = np.zeros_like(cc['infall_mass'])
# If there are satellites (not applicable for first step)
if numSatellites != 0:
# Match satellites to prev step cores using core tag.
vals, idx1, idx2 = np.intersect1d( cc['core_tag'][satellites_mask], cc_prev['core_tag'], return_indices=True )
# assert len(vals) == len(cc['core_tag'][satellites_mask]), 'All cores from prev step did not carry over.'
# assert len(cc['core_tag'][satellites_mask]) == len(np.unique(cc['core_tag'][satellites_mask])), 'Satellite core tags not unique for this time step.'
for A in A_arr:
for zeta in zeta_arr:
# Set m_evolved of all satellites that have core tag match on prev step to next_m_evolved of prev step.
# DOUBLE MASK ASSIGNMENT: cc['m_evolved'][satellites_mask][idx1] = cc_prev['next_m_evolved'][idx2]
cc[m_evolved_col(A, zeta)][ np.flatnonzero(satellites_mask)[idx1] ] = cc_prev[m_evolved_col(A, zeta, next=True)][idx2]
# Initialize m array (corresponds with cc[satellites_mask]) to be either infall_mass (step after infall) or m_evolved (subsequent steps).
m = (cc[m_evolved_col(A, zeta)][satellites_mask] == 0)*cc['infall_mass'][satellites_mask] + (cc[m_evolved_col(A, zeta)][satellites_mask] != 0)*cc[m_evolved_col(A, zeta)][satellites_mask]
# Initialize m array (corresponds with cc[satellites_mask]) to be either virial_infall_mass (step after infall) or m_evolved (subsequent steps).
#m = (cc[m_evolved_col(A, zeta)][satellites_mask] == 0)*SHMLM.m_vir(cc['infall_mass'][satellites_mask], step) + (cc[m_evolved_col(A, zeta)][satellites_mask] != 0)*cc[m_evolved_col(A, zeta)][satellites_mask]
# Set m_evolved of satellites with m_evolved=0 to infall mass.
cc[m_evolved_col(A, zeta)][satellites_mask] = m
# Initialize M array (corresponds with cc[satellites_mask]) to be host tree node mass of each satellite
M = cc['infall_mass'][centrals_mask][ many_to_one( cc['tree_node_index'][satellites_mask], cc['tree_node_index'][centrals_mask] ) ]
# assert len(M) == len(m) == len(cc['m_evolved'][satellites_mask]), 'M, m, cc[satellites_mask] lengths not equal.'
# wasInFragment: persistent flag that is False by default and becomes True when core is inside a fragment halo
cc['wasInFragment'] = cc['fof_halo_tag']<0
if step != steps[0]:
_, i1, i2 = np.intersect1d( cc['core_tag'], cc_prev['core_tag'], return_indices=True )
cc['wasInFragment'][i1] = np.logical_or( cc['wasInFragment'][i1], cc_prev['wasInFragment'][i2] )
if writeOutputFlag:
# Write output to disk
h5_write_dict( fname_cc(step, 'output'), cc, 'coredata' )
# Compute m_evolved of satellites according to SHMLModel for NEXT time step and save as cc_prev['next_m_evolved'] in memory.
# Mass loss assumed to begin at step AFTER infall detected.
if step != steps[-1]:
cc_prev = { 'core_tag':cc['core_tag'].copy(), 'wasInFragment':cc['wasInFragment'].copy() }
if numSatellites != 0:
localhost_m21 = many_to_one( cc['host_core'][satellites_mask], cc['core_tag'] )
for A in A_arr:
for zeta in zeta_arr:
cc_prev[m_evolved_col(A, zeta, next=True)] = np.zeros_like(cc['infall_mass'])
if numSatellites != 0: # If there are satellites (not applicable for first step)
m = cc[m_evolved_col(A, zeta)][satellites_mask]
if useLocalHost:
Mlocal = cc[m_evolved_col(A, zeta)][localhost_m21]
M_A_zeta = (Mlocal==0)*M + (Mlocal!=0)*Mlocal
cc_prev[m_evolved_col(A, zeta, next=True)][satellites_mask] = SHMLM.m_evolved(m0=m, M0=M_A_zeta, step=steps[steps.index(step)+1], step_prev=step, A=A, zeta=zeta)
else:
cc_prev[m_evolved_col(A, zeta, next=True)]][satellites_mask] = SHMLM.m_evolved(m0=m, M0=M, step=steps[steps.index(step)+1], step_prev=step, A=23.7, zeta=0.36)
return cc
def gio_read_dict(f, cols):
"""Read cols of GenericIO file f and return dictionary of numpy arrays."""
import genericio as gio
return {k: gio.gio_read(f, k)[0] for k in cols}
