def save_fig(self, fig, name, subdir=None, fignum=True, verbose=True, close=False):
if (not name.endswith('.pdf')) and (not name.endswith('.png')):
name += '.pdf'
if subdir is not None:
fname = os.path.join(self.output_plots, subdir, name)
self.check_path(fname)
fname = zio.modify_exists(fname)
fig.savefig(fname)
if verbose:
print("Saved to '{}'".format(fname))
if fignum is not None:
if isinstance(fignum, str):
name = zio.modify_filename(name, prepend=fignum + "_")
fname = os.path.join(self.path_output_figs, name)
zio.check_path(fname)
fig.savefig(fname)
if verbose:
print("Saved to '{}'".format(fname))
# if close:
# plt.close('all')
return fname
def fname_append_snap(fname, snap):
if isinstance(snap, str):
app = "_snap-{}".format(snap)
else:
app = "_snap-{:03d}".format(snap)
fname = zio.modify_filename(fname, append=app)
return fname
def loadMergerEnvironments(run, loadsave=True, verbose=True, version=_VERSION):
"""
Load all subhalo environment data as a dictionary with keys from ``ENVIRON``.
NOTE: the 'env_in' dictionary was created using `_in_merger_environments()` (i.e. manually),
and might not be recreated appropriately by `_collectMergerEnvironments()`.
Arguments
---------
run <int> : illustris simulation run number, {1, 3}
loadsave <bool> : optional, load existing save if it exists, otherwise create new
verbose <bool> : optional, print verbose output
version <flt> : optional, version number to load (can only create current version!)
Returns
-------
env <dict> : all environment data for all subhalos, keys given by ``ENVIRON`` class
"""
if verbose: print(" - - Environments.loadMergerEnvironments()")
fname_out = _GET_MERGER_ENVIRONMENT_FILENAME(run, version=version)
fname_in = zio.modify_filename(fname_out, append='_in')
# Try to Load Existing Save File
# ------------------------------
if loadsave:
if verbose:
print((
" - - Attempting to load saved file from '{}' and '{}'".format(
fname_out, fname_in)))
if os.path.exists(fname_out):
env_out = zio.npzToDict(fname_out)
env_in = zio.npzToDict(fname_in)
if verbose: print(" - - - Loaded.")
else:
print((" - - - File '{}' or '{}' does not exist!".format(
fname_out, fname_in)))
loadsave = False
# Import environment data directly, and save
# ------------------------------------------
if not loadsave:
if verbose:
print((" - - Importing Merger Environments, version %s" %
(str(_VERSION))))
env_out, env_in = _collectMergerEnvironments(run,
verbose=verbose,
version=version)
zio.dictToNPZ(env_out, fname_out, verbose=True)
zio.dictToNPZ(env_in, fname_in, verbose=True)
return env_out, env_in
def _reorganize_files(core, raw_fnames, temp_fnames):
log = core.log
log.debug("details._reorganize_files()")
NUM_SNAPS = core.sets.NUM_SNAPS
snap_scales = core.cosmo.scales()
temps = [zio.modify_filename(tt, prepend='_') for tt in temp_fnames]
# Open new ASCII, Temp dets files
# Make sure path is okay
zio.check_path(temps[0])
# Open each temp file
# temp_files = [open(tfil, 'w') for tfil in temp_fnames]
temp_files = [open(tfil, 'w') for tfil in temps]
num_temp = len(temp_files)
num_raw = len(raw_fnames)
log.info("Organizing {:d} raw files into {:d} temp files".format(
num_raw, num_temp))
prec = _DEF_PRECISION
all_num_lines_in = 0
all_num_lines_out = 0
# Iterate over all Illustris Details Files
# ----------------------------------------
for ii, raw in enumerate(core.tqdm(raw_fnames, desc='Raw files')):
log.debug("File {}: '{}'".format(ii, raw))
lines = []
scales = []
# Load all lines and entry scale-factors from raw dets file
for dline in open(raw):
lines.append(dline)
# Extract scale-factor from line
detScale = DTYPE.SCALAR(dline.split()[1])
scales.append(detScale)
# Convert to array
lines = np.array(lines)
scales = np.array(scales)
num_lines_in = scales.size
# If file is empty, continue
if num_lines_in == 0:
log.debug("\tFile empty")
continue
log.debug("\tLoaded {}".format(num_lines_in))
# Round snapshot scales to desired precision
scales_round = np.around(snap_scales, -prec)
# Find snapshots following each entry (right-edge) or equal (include right: 'right=True')
# `-1` to put binaries into the snapshot UP-TO that scalefactor
# snap_nums = np.digitize(scales, scales_round, right=True) - 1
snap_nums = np.digitize(scales, scales_round, right=True)
# For each Snapshot, write appropriate lines
num_lines_out = 0
for snap in range(NUM_SNAPS):
inds = (snap_nums == snap)
num_lines_out_snap = np.count_nonzero(inds)
if num_lines_out_snap == 0:
continue
temp_files[snap].writelines(lines[inds])
# log.debug("\t\tWrote {} lines to snap {}".format(num_lines_out_snap, snap))
num_lines_out += num_lines_out_snap
if num_lines_out != num_lines_in:
log.error("File {}, '{}'".format(ii, raw))
log.raise_error("Wrote {} lines, loaded {} lines!".format(
num_lines_out, num_lines_in))
all_num_lines_in += num_lines_in
all_num_lines_out += num_lines_out
# Close out dets files
tot_size = 0.0
log.info("Closing files, checking sizes")
for ii, newdf in enumerate(temp_files):
newdf.close()
tot_size += os.path.getsize(newdf.name)
ave_size = tot_size / (1.0 * len(temp_files))
size_str = zio.bytes_string(tot_size)
ave_size_str = zio.bytes_string(ave_size)
log.info("Total temp size = '{}', average = '{}'".format(
size_str, ave_size_str))
log.info("Input lines = {:d}, Output lines = {:d}".format(
all_num_lines_in, all_num_lines_out))
if (all_num_lines_in != all_num_lines_out):
log.raise_error(
"input lines {}, does not match output lines {}!".format(
all_num_lines_in, all_num_lines_out))
log.info("Renaming temporary files...")
for ii, (aa, bb) in enumerate(zip(temps, temp_fnames)):
if ii == 0:
log.debug("'{}' ==> '{}'".format(aa, bb))
shutil.move(aa, bb)
return
def _in_merger_environments(run, verbose=True, version=_VERSION):
merger_snaps, snap_mergers, subh_ind_out, subh_ind_in = \
get_merger_and_subhalo_indices(run, verbose=verbose)
numMergers = len(merger_snaps)
# Get all subhalos for each snapshot (including duplicates and missing)
snap_subh_out = [subh_ind_out[smrg] for smrg in snap_mergers]
snap_subh_in = [subh_ind_in[smrg] for smrg in snap_mergers]
sampleSnap = 135
env_in = _initStorage(run,
sampleSnap,
snap_subh_out[sampleSnap],
numMergers,
verbose=verbose,
version=version)
beg = datetime.now()
pbar = zio.getProgressBar(numMergers)
pbar.start()
count = 0
numGood = 0
numBad = 0
# Iterate over each Snapshot
for snap, (merg, subh_in) in zmath.renumerate(
list(zip(snap_mergers, snap_subh_in))):
# Get indices of valid subhalos
inds_subh_in = np.where(subh_in >= 0)[0]
# Skip this snapshot if no valid subhalos
if inds_subh_in.size == 0 or len(merg) == 0: continue
# Select corresponding merger indices
inds_in = np.array(merg)[inds_subh_in]
# Get Data from Group Catalog
# ---------------------------
try:
gcat = Subhalo.importGroupCatalogData(
run, snap, subhalos=subh_in[inds_subh_in], verbose=False)
# Count bad, and skip to next snapshot on failure
except:
print(("gcat import snap {} failed. {} Mergers.".format(
snap, len(merg))))
numBad += len(merg)
count += len(merg)
pbar.update(count)
continue
# Extract desired data
for key in env_in[ENVIRON.GCAT_KEYS]:
env_in[key][inds_in, ...] = gcat[key][...]
# Load Each Merger-Subhalo file contents
# --------------------------------------
for ind_subh, merger in zip(inds_subh_in, inds_in):
count += 1
subhalo = subh_in[ind_subh]
# Store Subhalo number for each merger
env_in[ENVIRON.SUBH][merger] = subhalo
env_in[ENVIRON.SNAP][merger] = snap
# Set as good merger-environment
env_in[ENVIRON.STAT][merger] = 1
numGood += 1
# Update progessbar
pbar.update(count)
pbar.finish()
end = datetime.now()
if verbose:
print((" - - - Completed after %s" % (str(end - beg))))
print((" - - - Total %5d/%5d = %.4f" %
(count, numMergers, count / numMergers)))
print((" - - - Good %5d/%5d = %.4f" %
(numGood, numMergers, numGood / numMergers)))
print((" - - - Bad %5d/%5d = %.4f" %
(numBad, numMergers, numBad / numMergers)))
fname_out = _GET_MERGER_ENVIRONMENT_FILENAME(run, version=version)
fname_in = zio.modify_filename(fname_out, append='_in')
print("fname_out = '{}'".format(fname_out))
print("fname_in = '{}'".format(fname_in))
zio.dictToNPZ(env_in, fname_in, verbose=True)
return env_in
def main():
core = Core(
sets=dict(LOG_FILENAME="log_illbh-snapshots.log", RECREATE=True))
log = core.log
log.info("details.main()")
print(log.filename)
beg = datetime.now()
fname = core.paths.fname_bh_particles
exists = os.path.exists(fname)
recreate = core.sets.RECREATE
log.debug("File '{}' exists: {}".format(fname, exists))
if not recreate and exists:
log.info("Particle file exists: '{}'".format(fname))
return
log.warning("Loading BH particle data from snapshots")
fname_temp = zio.modify_filename(fname, prepend='_')
log.debug("Writing to temporary file '{}'".format(fname_temp))
# log.error("WARNING: running in TEMPORARY append mode!")
# with h5py.File(fname_temp, 'a') as out:
with h5py.File(fname_temp, 'r') as out:
all_ids = set()
for snap in core.tqdm(range(NUM_SNAPS), desc='Loading snapshots'):
log.debug("Loading snap {}".format(snap))
snap_str = '{:03d}'.format(snap)
group = out.create_group(snap_str)
try:
bhs = illpy.snapshot.loadSubset(core.paths.INPUT, snap,
PARTICLE.BH)
except Exception as err:
log.error("FAILED on snap {}!!!".format(snap))
continue
num_bhs = bhs['count']
log.info("Snap {} Loaded {} BHs".format(snap, num_bhs))
group.attrs['count'] = num_bhs
if num_bhs == 0:
continue
ids = bhs['ParticleIDs']
all_ids = all_ids.union(ids)
sort = np.argsort(ids)
keys = list(bhs.keys())
keys.pop(keys.index('count'))
for kk in keys:
group.create_dataset(kk, data=bhs[kk][:][sort])
'''
for snap in core.tqdm(range(NUM_SNAPS), desc='Loading snapshots'):
log.debug("Loading snap {}".format(snap))
snap_str = '{:03d}'.format(snap)
group = out[snap_str]
if 'ParticleIDs' not in group.keys():
log.error("Skipping snap {}".format(snap))
continue
ids = group['ParticleIDs']
num_bhs = ids.size
group.attrs['num'] = num_bhs
all_ids = all_ids.union(ids)
'''
all_ids = np.array(sorted(list(all_ids)))
first = NUM_SNAPS * np.ones_like(all_ids, dtype=np.uint32)
last = np.zeros_like(all_ids, dtype=np.uint32)
# Find the first and last snapshot that each BH is found in
for snap in core.tqdm(range(NUM_SNAPS), desc='Finding first/last'):
snap_str = '{:03d}'.format(snap)
try:
ids = out[snap_str]['ParticleIDs'][:]
except KeyError as err:
lvl = log.INFO if (snap in [53, 55]) else log.ERROR
log.log(
lvl,
"Failed to access `ParticleIDs` from snap {}".format(snap))
log.log(lvl, str(err))
continue
slots = np.searchsorted(all_ids, ids)
first[slots] = np.minimum(first[slots], snap)
last[slots] = np.maximum(last[slots], snap)
out.create_dataset('unique_ids', data=all_ids)
out.create_dataset('unique_first_snap', data=first)
out.create_dataset('unique_last_snap', data=last)
log.debug("Moving temporary to final file '{}' ==> '{}'".format(
fname_temp, fname))
shutil.move(fname_temp, fname)
size_str = zio.get_file_size(fname)
end = datetime.now()
log.info("Saved to '{}', size {}, after {}".format(fname, size_str,
end - beg))
return
def plotScattering(sample, snap, mbhb, log, plotNames):
"""Illustrate the scattering calculation for a single, sample system.
Performs calculation by calling the 'scattering()' method, just like in
"MBHBinaryEvolution.py".
Arguments
---------
sample : int
Target galaxy/merger number to examine (this is the number out of *all* binaries,
not just the valid ones [included in ``mbhb.evolution``]).
snap : int
Illustris snapshot number {1, 135}.
mbhb : `Binaries.MBHBinaries` object
log : `logging.Logger` object
plotNames : str
Base name of plots.
Returns
-------
plotNames : list of str
Filenames of the plots created.
"""
log.debug("plotScattering()")
PLOT_A10 = True # LC Occupancy
PLOT_A11 = True # Flux
PLOT_A15 = True # Flux vs. Separation
PLOT_A02 = False # Model Galaxy
PLOT_A08 = False # Dist Func
from . import GM_Figures
figNames = []
# Initialization
# --------------
radialRange = np.array(mbhb.sets.RADIAL_RANGE_MODELS) * PC
numSeps = mbhb.sets.PLOT_SCATTERING_SAMPLE_SEPS
# Convert from `sample` number, of all binaries to index for valid (``mbhb.evolution``) ones
val_inds = np.where(mbhb.valid)[0]
valid_sample = np.where(val_inds == sample)[0]
if valid_sample.size == 1:
valid_sample = valid_sample[0]
else:
raise ValueError("`sample` '{}' returned '{}' from `val_inds`".format(sample, valid_sample))
mstar = mbhb.galaxies.mstar
log.debug(" - Sample subhalo %d, Snapshot %d" % (sample, snap))
# Binary Properties (snapshot dependent)
# `evolution` class includes only valid binaries, so use `valid_sample`
# m1 = np.max(mbhb.evolution.masses[valid_sample, snap])
# m2 = np.min(mbhb.evolution.masses[valid_sample, snap])
m1 = np.max(mbhb.initMasses[sample])
m2 = np.min(mbhb.initMasses[sample])
# Galaxy properties (snapshot independent)
# `galaxies` class includes *all* binaries, so use `sample` itself
gals = mbhb.galaxies
eps = gals.eps[sample]
# functions of energy
rads = gals.rads
periods = gals.perOrb[sample]
dist_func = gals.dist_func[sample]
diffCoef = gals.diffCoef[sample]
j2Circs = gals.j2Circ[sample]
dnStarsAll = gals.dnStarsAll[sample]
ndensStars = gals.densStars[sample]/mstar
radHard = gals.rads_hard[sample]
numStarsAll = sp.integrate.cumtrapz(dnStarsAll[::-1], eps[::-1], initial=0.0)[::-1]
loss_cone = Loss_Cone_Explicit(mbhb.sets, log)
# Wrap the Scattering function (for convenience)
def evalScattering(binSep):
# retvals = scattering(m1, m2, binSep, rads, eps, periods, j2Circs, diffCoef, dist_func,
# [sample], radHard, mbhb.sets)
retvals = loss_cone.harden(
m1, m2, binSep, rads, eps, periods, j2Circs, diffCoef, dist_func,
[sample], radHard, mbhb.sets)
radLC, j2LC, enrLC, dnStarsFLC, numStarsFLC, numStarsSSLC, \
dfStarsFLC, dfStarsSSLC, fluxStarsFLC, fluxStarsSSLC, flux, dadt_lc = retvals
return dnStarsFLC, numStarsFLC, dfStarsFLC, fluxStarsFLC, dfStarsSSLC, fluxStarsSSLC
num_flc = np.zeros(numSeps)
flx_flc = np.zeros(numSeps)
flx_sslc = np.zeros(numSeps)
dadt_sslc = np.zeros(numSeps)
dadt_flc = np.zeros(numSeps)
subPlotNames = zio.modify_filename(plotNames, prepend='stellar_scattering/')
zio.check_path(subPlotNames)
# Iterate over range of binary separations, plot profiles for each
log.debug(" - Calculting scattering for %d binary separations" % (numSeps))
flx_seps = zmath.spacing(radialRange, scale='log', num=numSeps)
for ii, binSep in enumerate(tqdm.tqdm(flx_seps, desc="Calculating scattering")):
dnStarsFLC, numStarsFLC, dfStarsFLC, fluxStarsFLC, dfStarsSSLC, fluxStarsSSLC = \
evalScattering(binSep)
hard_sslc = dadt_scattering(m1+m2, binSep, np.max(fluxStarsSSLC), mstar)
hard_flc = dadt_scattering(m1+m2, binSep, np.max(fluxStarsFLC), mstar)
eps_rad = zmath.spline(gals.rads, gals.eps[sample], log=True, pos=True, extrap=True)
sepEner = eps_rad(binSep)
# Plot Fig A10 - Loss-Cone Occupancy
if PLOT_A10:
fig = GM_Figures.figa10_lc_occupancy(
gals.rads, eps, dnStarsAll, dnStarsFLC, numStarsAll, numStarsFLC,
binr=binSep, bine=sepEner)
fname1 = subPlotNames + "lcPipe_figa10-occ_%02d.png" % (ii)
zplot.saveFigure(fig, fname1, verbose=False) # , log=log)
plt.close(fig)
# Plot Fig A11 - loss-cone fluxes
if PLOT_A11:
fig = GM_Figures.figa11_lc_flux(
eps, gals.rads, dfStarsFLC, dfStarsSSLC, fluxStarsFLC, fluxStarsSSLC,
binr=binSep, bine=sepEner)
fname2 = subPlotNames + "lcPipe_figa11-flx_%02d.png" % (ii)
zplot.saveFigure(fig, fname2, verbose=False) # , log=log)
plt.close(fig)
# Store values to arrays
num_flc[ii] = np.max(numStarsFLC)
flx_flc[ii] = np.max(fluxStarsFLC)
flx_sslc[ii] = np.max(fluxStarsSSLC)
dadt_sslc[ii] = hard_sslc
dadt_flc[ii] = hard_flc
# pbar.update(ii)
# pbar.finish()
if PLOT_A10:
log.debug(" - - Saved FigA10 to (e.g.) '%s'" % (fname1))
if PLOT_A11:
log.debug(" - - Saved FigA11 to (e.g.) '%s'" % (fname2))
# Plot Figs A15 - Scattering vs. Separations
if PLOT_A15:
log.debug(" - Plotting Fig15")
fig = GM_Figures.figA15_flux_sep(flx_seps, flx_flc, flx_sslc, dadt_flc, dadt_sslc,
sample, snap)
fname = plotNames + "lcPipe_figa15-sep.png"
zplot.saveFigure(fig, fname, verbose=False, log=log)
plt.close(fig)
figNames.append(fname)
# Plot Figure A02 - Density/Mass Profiles
if PLOT_A02:
log.info(" - Plotting FigA02")
dens = [gals.densStars[sample], gals.densDM[sample]]
mass = [gals.massStars[sample], gals.massDM[sample], gals.massTot[sample]]
names = ['Stars', 'DM', 'Total']
vels = [gals.vdisp_stars[sample]]
fig = GM_Figures.figA02_ModelGalaxy(gals.rads, dens, mass, vels, names=names)
fname = plotNames + "lcPipe_figa02-dens.png"
zplot.saveFigure(fig, fname, verbose=False, log=log)
plt.close(fig)
figNames.append(fname)
# Plot Figure A08 - Reconstructed Number Density
if PLOT_A08:
log.debug(" - Plotting FigA08")
df_pot = zmath.spline(eps, dist_func, mono=True, pos=True, sort=True, log=True)
dfErrs = np.zeros(np.size(eps))
fig = GM_Figures.figA08_distFunc(eps, gals.rads, ndensStars, dist_func, dfErrs, df_pot, log,
sample=sample)
fname = plotNames + "lcPipe_figa08-distfunc.png"
zplot.saveFigure(fig, fname, verbose=False, log=log)
plt.close(fig)
figNames.append(fname)
return figNames
def saveFigure(fig,
fname,
verbose=True,
log=None,
level=logging.WARNING,
close=True,
**kwargs):
"""Save the given figure(s) to the given filename.
If ``fig`` is iterable, a multipage pdf is created. Otherwise a single file is made.
Does *not* make sure path exists.
Arguments
---------
fig <obj>([N]) : one or multiple ``matplotlib.figure.Figure`` objects.
fname <str> : filename to save to.
verbose <bool> : print verbose output to stdout
log <obj> : ``logging.Logger`` object to print output to
level <int> :
close <bool> : close figures after saving
**kwargs <dict> : additional arguments past to ``savefig()``.
"""
# CATCH WRONG ORDER OF ARGUMENTS
if type(fig) == str:
warnings.warn("FIRST ARGUMENT SHOULD BE `fig`!!")
temp = str(fig)
fig = fname
fname = temp
if log is not None: log.debug("Saving figure...")
if not np.iterable(fig): fig = [fig]
saved_names = []
# Save as multipage PDF
if fname.endswith('pdf') and np.size(fig) > 1:
from matplotlib.backends.backend_pdf import PdfPages
with PdfPages(fname) as pdf:
for ff in fig:
pdf.savefig(figure=ff, **kwargs)
if close: plt.close(ff)
# Make sure file now exists
if os.path.exists(fname):
saved_names.append(fname)
else:
raise RuntimeError(
"Figure '{}' did not save.".format(fname))
else:
# Save each figure to a different file
for ii, ff in enumerate(fig):
# On subsequent figures, append the number to the filename
if ii == 0:
usefname = str(fname)
else:
usefname = zio.modify_filename(fname, append='_%d' % (ii))
ff.savefig(usefname, **kwargs)
if close: plt.close(ff)
if os.path.exists(usefname):
saved_names.append(usefname)
else:
raise RuntimeError(
"Figure '{}' did not save.".format(usefname))
# No files saved or Some files were not saved
if not len(saved_names) or len(saved_names) != len(fig):
warn_str = "Error saving figures..."
if log is None:
warnings.warn(warn_str)
else:
log.warning(warn_str)
# Things look good.
else:
printStr = "Saved figure to '%s'" % (fname)
if log is not None:
log.log(level, printStr)
elif verbose:
print(printStr)
return
def save_fig(fig, fname, count):
save_name = zio.modify_filename(fname, append="_{}".format(count))
fig.savefig(fig, save_name)
print("Saved {} to '{}'".format(count, save_name))
return count + 1