def save_fig(self, fig, name, subdir=None, fignum=True, verbose=True, close=False):
if (not name.endswith('.pdf')) and (not name.endswith('.png')):
name += '.pdf'
if subdir is not None:
fname = os.path.join(self.output_plots, subdir, name)
self.check_path(fname)
fname = zio.modify_exists(fname)
fig.savefig(fname)
if verbose:
print("Saved to '{}'".format(fname))
if fignum is not None:
if isinstance(fignum, str):
name = zio.modify_filename(name, prepend=fignum + "_")
fname = os.path.join(self.path_output_figs, name)
zio.check_path(fname)
fig.savefig(fname)
if verbose:
print("Saved to '{}'".format(fname))
# if close:
# plt.close('all')
return fname
def _loadLogger(name, verbose=True, debug=False, run=None, rank=None, version=None, tofile=True):
"""Initialize a ``logging.Logger`` object for output messages.
All processes log to output files, and the root process also outputs to `stdout`. Constructs
the log name based on the `name` argument, which should be the `__file__` parameter from the
script which calls this method.
Arguments
---------
name : str
Base (file)name from which to construct the log's name, and log filename.
verbose : bool
Print 'verbose' (``logging.INFO``) output to stdout.
Overridden if ``debug == True``.
debug : bool
Print extremely verbose (``logging.DEBUG``) output to stdout.
Overrides `verbose` setting.
run : int or `None`
Illustris run number {1,3}. Added to log filename if provided.
rank : int or `None`
Rank of the current process for MPI runs. Added to the log filename if provided.
version : str or `None`
Current version of script being run. Added to the log filename if provided.
tofile : bool
Whether output should also be logged to a file.
Returns
-------
logger : ``logging.Logger`` object
Object for output logging.
"""
# Get logger and log-file names
logName, logFilename = _GET_LOG_NAMES(name, run=run, rank=rank, version=version)
# Make sure directory exists
zio.check_path(logFilename)
# Determine verbosity level
if debug:
strLvl = logging.DEBUG
elif verbose:
strLvl = logging.INFO
else:
strLvl = logging.WARNING
# Turn off file-logging
if not tofile: logFilename = None
fileLvl = logging.DEBUG
# Create logger
if rank == 0 or rank is None:
kw = dict(level_stream=strLvl)
else:
kw = dict(tostr=False)
# logger = zio.getLogger(logName, tofile=logFilename, fileLevel=fileLvl, strLevel=strLvl, **kw)
logger = zio.get_logger(logName, tofile=logFilename, level_file=fileLvl, **kw)
return logger
def __init__(self, sim_path, create=True):
# Make sure the path does NOT include a terminal path-separator
sim_path = os.path.dirname(os.path.join(sim_path, ''))
# If the 'output' folder is specified, move up a directory to the base-path
if sim_path.endswith(self._DNAME_OUTPUT):
sim_path = os.path.dirname(sim_path)
self.sim = sim_path
self.output = os.path.join(sim_path, self._DNAME_OUTPUT)
# self.analysis = os.path.join(sim_path, self._DNAME_ANALYSIS)
self.analysis = os.path.join(self.output, self._DNAME_ANALYSIS)
_check_paths = [self.output, self.analysis]
if create:
for cp in _check_paths:
zio.check_path(cp)
return
def _checkLog(log, run=None, debug=Settings.debug, verbose=Settings.verbose):
"""Create a default logging object if one is not given.
"""
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.rank
if not rank:
zio.check_path(_LOG_DIR)
comm.Barrier()
if log is None:
log = constants._loadLogger(
__file__, debug=debug, verbose=debug, run=run, rank=rank, version=__version__)
header = "\n%s\n%s\n%s" % (__file__, '='*len(__file__), str(datetime.now()))
log.debug(header)
if not rank:
print(("Log filename = ", log.filename))
return log
def save_fig(self,
fig,
fname,
path=None,
subdir=None,
snap_num=None,
modify=False,
verbose=False,
**kwargs):
fname = self.fname(fname,
path=path,
subdir=subdir,
snap_num=snap_num,
modify=modify)
kwargs.setdefault('transparent', True)
kwargs.setdefault('dpi', 100)
zio.check_path(fname)
fig.savefig(fname, **kwargs)
if verbose:
print("saved to '{}' size: {}".format(fname,
zio.get_file_size(fname)))
return fname
def test_filenames():
print("particle_hosts_test.filenames_test()")
for run in range(1, 4):
print("run = '{}'".format(run))
processed_dir = particle_hosts._get_path_processed(run)
print("processed_dir = '{}'".format(processed_dir))
zio.check_path(processed_dir)
assert_true(os.path.exists(processed_dir))
# make sure directory is writeable
test_fname = os.path.join(processed_dir, 'test_123_test.txt')
print("Trying to write to file '{}'".format(test_fname))
with open(test_fname, 'w') as test:
test.write('hello')
print("File '{}' Exists: {}".format(test_fname, str(os.path.exists(test_fname))))
assert_true(os.path.exists(test_fname))
print("Deleting file '{}'".format(test_fname))
os.remove(test_fname)
print("File '{}' Exists: {}".format(test_fname, str(os.path.exists(test_fname))))
assert_false(os.path.exists(test_fname))
# Offset table
print("loading path for: offset table")
path = particle_hosts._get_filename_offset_table(run, 135, version='1.0')
print(path)
# bh-hosts-snap table
print("loading path for: bh-hosts-snap table")
path = particle_hosts._get_filename_bh_hosts_snap_table(run, 135, version='1.0')
print(path)
# bh-hosts table
print("loading path for: bh-hosts table")
path = particle_hosts._get_filename_bh_hosts_table(run, 135, version='1.0')
print(path)
print("using 'run' 0 should fail")
assert_raises(KeyError, particle_hosts._get_path_processed, 0)
return
def _get_path_processed(run):
"""
_ILLUSTRIS_RUN_NAMES = {1: "L75n1820FP",
2: "L75n910FP",
3: "L75n455FP"}
_PROCESSED_DIR = "/n/home00/lkelley/illustris/data/{}/output/postprocessing/"
"""
from ..constants import _ILLUSTRIS_RUN_NAMES
# Make sure path ends in '/'
path = os.path.join(_PROCESSED_DIR.format(_ILLUSTRIS_RUN_NAMES[run]), '')
if zio.check_path(path) is None:
raise OSError("_get_path_processed(): Path failed '{}'".format(path))
return path
def _reorganize_files(core, raw_fnames, temp_fnames):
log = core.log
log.debug("details._reorganize_files()")
NUM_SNAPS = core.sets.NUM_SNAPS
snap_scales = core.cosmo.scales()
temps = [zio.modify_filename(tt, prepend='_') for tt in temp_fnames]
# Open new ASCII, Temp dets files
# Make sure path is okay
zio.check_path(temps[0])
# Open each temp file
# temp_files = [open(tfil, 'w') for tfil in temp_fnames]
temp_files = [open(tfil, 'w') for tfil in temps]
num_temp = len(temp_files)
num_raw = len(raw_fnames)
log.info("Organizing {:d} raw files into {:d} temp files".format(
num_raw, num_temp))
prec = _DEF_PRECISION
all_num_lines_in = 0
all_num_lines_out = 0
# Iterate over all Illustris Details Files
# ----------------------------------------
for ii, raw in enumerate(core.tqdm(raw_fnames, desc='Raw files')):
log.debug("File {}: '{}'".format(ii, raw))
lines = []
scales = []
# Load all lines and entry scale-factors from raw dets file
for dline in open(raw):
lines.append(dline)
# Extract scale-factor from line
detScale = DTYPE.SCALAR(dline.split()[1])
scales.append(detScale)
# Convert to array
lines = np.array(lines)
scales = np.array(scales)
num_lines_in = scales.size
# If file is empty, continue
if num_lines_in == 0:
log.debug("\tFile empty")
continue
log.debug("\tLoaded {}".format(num_lines_in))
# Round snapshot scales to desired precision
scales_round = np.around(snap_scales, -prec)
# Find snapshots following each entry (right-edge) or equal (include right: 'right=True')
# `-1` to put binaries into the snapshot UP-TO that scalefactor
# snap_nums = np.digitize(scales, scales_round, right=True) - 1
snap_nums = np.digitize(scales, scales_round, right=True)
# For each Snapshot, write appropriate lines
num_lines_out = 0
for snap in range(NUM_SNAPS):
inds = (snap_nums == snap)
num_lines_out_snap = np.count_nonzero(inds)
if num_lines_out_snap == 0:
continue
temp_files[snap].writelines(lines[inds])
# log.debug("\t\tWrote {} lines to snap {}".format(num_lines_out_snap, snap))
num_lines_out += num_lines_out_snap
if num_lines_out != num_lines_in:
log.error("File {}, '{}'".format(ii, raw))
log.raise_error("Wrote {} lines, loaded {} lines!".format(
num_lines_out, num_lines_in))
all_num_lines_in += num_lines_in
all_num_lines_out += num_lines_out
# Close out dets files
tot_size = 0.0
log.info("Closing files, checking sizes")
for ii, newdf in enumerate(temp_files):
newdf.close()
tot_size += os.path.getsize(newdf.name)
ave_size = tot_size / (1.0 * len(temp_files))
size_str = zio.bytes_string(tot_size)
ave_size_str = zio.bytes_string(ave_size)
log.info("Total temp size = '{}', average = '{}'".format(
size_str, ave_size_str))
log.info("Input lines = {:d}, Output lines = {:d}".format(
all_num_lines_in, all_num_lines_out))
if (all_num_lines_in != all_num_lines_out):
log.raise_error(
"input lines {}, does not match output lines {}!".format(
all_num_lines_in, all_num_lines_out))
log.info("Renaming temporary files...")
for ii, (aa, bb) in enumerate(zip(temps, temp_fnames)):
if ii == 0:
log.debug("'{}' ==> '{}'".format(aa, bb))
shutil.move(aa, bb)
return
def _runMaster(run, comm):
"""
Run master process which manages all of the secondary ``slave`` processes.
Details
-------
- Retrieves merger, snapshot and subhalo indices
- Iterates over snapshots and merger-subhalo pairs, distributing them to ``slave`` processes
which load each subhalo profile and writes them to individual-subhalo files.
- Loads most-bound particle ID numbers from group caalog for each snapshot and distributes
this to each slave-process as-well.
- Tracks how-many and which process (and subhalos) finish successfully
"""
from mpi4py import MPI
stat = MPI.Status()
# rank = comm.rank
size = comm.size
print(" - Initializing")
merger_snaps, snap_mergers, subh_ind_out, subh_ind_in = \
get_merger_and_subhalo_indices(run, verbose=True)
# Get all subhalos for each snapshot (including duplicates and missing)
snapSubh = [subh_ind_out[smrg] for smrg in snap_mergers]
# Get unique subhalos for each snapshot, discard duplicates
snapSubh_uni = [np.array(list(set(ssubh))) for ssubh in snapSubh]
# Discard missing matches ('-1')
snapSubh_uni = [ssubh[np.where(ssubh != -1)] for ssubh in snapSubh_uni]
numUni = [len(ssubh) for ssubh in snapSubh_uni]
numUniTot = np.sum(numUni)
numMSnaps = np.count_nonzero(numUni)
print(
(" - - %d Unique subhalos over %d Snapshots" % (numUniTot, numMSnaps)))
# Iterate over Snapshots and Subhalos
# ===================================
# distribute tasks to slave processes
count = 0
new = 0
exist = 0
fail = 0
times = np.zeros(numUniTot)
statFileName = _GET_ENVIRONMENTS_STATUS_FILENAME(run)
statFile = open(statFileName, 'w')
print((" - - Opened status file '%s'" % (statFileName)))
statFile.write('%s\n' % (str(datetime.now())))
beg = datetime.now()
for snap, subs in zmath.renumerate(snapSubh_uni):
if len(subs) <= 0: continue
# Create output directory (subhalo doesn't matter since only creating dir)
# don't let slave processes create it - makes conflicts
fname = _GET_MERGER_SUBHALO_FILENAME(run, snap, 0)
zio.check_path(fname)
# Get most bound particles for each subhalo in this snapshot
mostBound = Subhalo.importGroupCatalogData(run,
snap,
subhalos=subs,
fields=[SUBHALO.MOST_BOUND],
verbose=False)
# Go over each subhalo
for boundID, subhalo in zip(mostBound, subs):
# Write status to file
dur = (datetime.now() - beg)
statStr = 'Snap %3d %8d/%8d = %.4f in %s %8d new %8d exist %8d fail\n' % \
(snap, count, numUniTot, 1.0*count/numUniTot, str(dur), new, exist, fail)
statFile.write(statStr)
statFile.flush()
# Look for available slave process
data = comm.recv(source=MPI.ANY_SOURCE,
tag=MPI.ANY_TAG,
status=stat)
source = stat.Get_source()
tag = stat.Get_tag()
# Track number of completed profiles
if tag == _TAGS.DONE:
retStat, durat = data
times[count] = durat
count += 1
if retStat == _ENVSTAT.NEWF: new += 1
elif retStat == _ENVSTAT.EXST: exist += 1
else: fail += 1
# Distribute tasks
comm.send([snap, subhalo, boundID], dest=source, tag=_TAGS.START)
statFile.write('\n\nDone after %s' % (str(datetime.now() - beg)))
statFile.close()
# Close out all Processes
# =======================
numActive = size - 1
print((" - Exiting %d active processes" % (numActive)))
while (numActive > 0):
# Find available slave process
data = comm.recv(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=stat)
source = stat.Get_source()
tag = stat.Get_tag()
# If we're recieving exit confirmation, count it
if tag == _TAGS.EXIT: numActive -= 1
else:
# If a process just completed, count it
if tag == _TAGS.DONE:
times[count] = data[1]
count += 1
if data[0]: new += 1
# Send exit command
comm.send(None, dest=source, tag=_TAGS.EXIT)
print((" - - %d/%d = %.4f Completed tasks!" %
(count, numUniTot, 1.0 * count / numUniTot)))
print((" - - %d New Files" % (new)))
return
def main():
"""Create master and many slave processes to extract BH snapshot data.
"""
# Initialize MPI Parameters
# -------------------------
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.rank
size = comm.size
if (size <= 1): raise RuntimeError("Not setup for serial runs!")
if (rank == 0):
NAME = sys.argv[0]
print(("\n{:s}\n{:s}\n{:s}".format(NAME, '='*len(NAME), str(datetime.now()))))
zio.check_path(bh_constants._LOG_DIR)
# Make sure log-path is setup before continuing
comm.Barrier()
# Parse Arguments
# ---------------
args = _parseArguments()
run = args.run
verbose = args.verbose
# Load logger
logger = bh_constants._loadLogger(__file__, verbose=verbose, run=run,
rank=rank, version=_VERSION)
logger.info("run = %d " % (run))
logger.info("version = %.2f" % (_VERSION))
logger.info("MPI comm size = %d " % (size))
logger.info("Rank = %d " % (rank))
logger.info("")
logger.info("verbose = %s " % (str(verbose)))
logger.info("")
# Master Process
# --------------
if (rank == 0):
beg_all = datetime.now()
try:
logger.debug("Running Master")
_runMaster(run, comm, logger)
except Exception as err:
zio._mpiError(comm, log=logger, err=err)
end_all = datetime.now()
logger.debug("Done after '%s'" % (str(end_all-beg_all)))
logger.info("Merging snapshots")
loadBHSnapshotData(run, logger=logger, loadsave=False)
# Slave Processes
# ---------------
else:
try:
logger.debug("Running slave")
_runSlave(run, comm, logger)
except Exception as err:
zio._mpiError(comm, log=logger, err=err)
logger.debug("Done.")
return
def _runMaster(run, comm, logger):
"""Distribute snapshots and associated mrgs to individual slave tasks for loading.
Loads ``mergers`` and distributes them, based on snapshot, to slave processes run by the
``_runSlave`` method. A status file is created to track progress. Once all snapshots are
distributed, this method directs the termination of all of the slave processes.
Arguments
---------
run : int,
Ollustris simulation number {1, 3}.
comm : ``mpi4py.MPI.Intracomm`` object,
MPI intracommunicator object, `COMM_WORLD`.
logger : ``logging.Logger`` object,
Object for logging.
"""
from mpi4py import MPI
stat = MPI.Status()
rank = comm.rank
size = comm.size
logger.info("BHSnapshotData._runMaster()")
logger.debug("Rank %d/%d" % (rank, size))
# Make sure output directory exists
fname = _GET_BH_SINGLE_SNAPSHOT_FILENAME(run, 0)
zio.check_path(fname)
# Load BH Mergers
logger.info("Loading BH Mergers")
mrgs = mergers.load_fixed_mergers(run, loadsave=True, verbose=False)
numMergers = mrgs[MERGERS.NUM]
logger.debug("- Loaded %d mrgs" % (numMergers))
# Init status file
statFileName = bh_constants._GET_STATUS_FILENAME(__file__, run=run, version=_VERSION)
statFile = open(statFileName, 'w')
logger.debug("Opened status file '%s'" % (statFileName))
statFile.write('%s\n' % (str(datetime.now())))
beg = datetime.now()
num_pos = 0
num_neg = 0
num_new = 0
countDone = 0
count = 0
times = np.zeros(NUM_SNAPS-1)
# Iterate Over Snapshots
# ----------------------
# Go over snapshots in random order to get a better estimate of ETA/duration
snapList = np.arange(NUM_SNAPS-1)
np.random.shuffle(snapList)
logger.info("Iterating over snapshots")
pbar = zio.getProgressBar(NUM_SNAPS-1)
for snapNum in snapList:
logger.debug("- Snap %d, count %d, done %d" % (snapNum, count, countDone))
# Get Mergers occuring just after Snapshot `snapNum`
mrgs = mrgs[MERGERS.MAP_STOM][snapNum+1]
nums = len(mrgs)
targetIDs = mrgs[MERGERS.IDS][mrgs]
logger.debug("- %d Mergers from snapshot %d" % (nums, snapNum+1))
# Look for available slave process
data = comm.recv(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=stat)
src = stat.Get_source()
tag = stat.Get_tag()
logger.debug("- Received signal from %d" % (src))
# Track number of completed profiles
if (tag == MPI_TAGS.DONE):
durat, pos, neg, new = data
logger.debug("- - Done after %s, pos %d, neg %d, new %d" % (durat, pos, neg, new))
times[countDone] = durat
num_pos += pos
num_neg += neg
num_new += new
countDone += 1
# Distribute tasks
logger.debug("- Sending new task to %d" % (src))
comm.send([snapNum, mrgs, targetIDs, numMergers], dest=src, tag=MPI_TAGS.START)
logger.debug("- New task sent")
# Write status to file and log
dur = (datetime.now()-beg)
fracDone = 1.0*countDone/(NUM_SNAPS-1)
statStr = 'Snap %3d (rank %03d) %8d/%8d = %.4f in %s %8d pos %8d neg %3d new\n' % \
(snapNum, src, countDone, NUM_SNAPS-1, fracDone, str(dur), num_pos, num_neg, num_new)
statFile.write(statStr)
statFile.flush()
logger.debug(statStr)
count += 1
pbar.update(count)
statFile.write('\n\nDone after %s' % (str(datetime.now()-beg)))
statFile.close()
pbar.finish()
# Close out all Processes
# -----------------------
numActive = size-1
logger.info("Exiting %d active processes" % (numActive))
while(numActive > 0):
# Find available slave process
data = comm.recv(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=stat)
src = stat.Get_source()
tag = stat.Get_tag()
logger.debug("- Received signal from %d" % (src))
# If we're recieving exit confirmation, count it
if (tag == MPI_TAGS.EXIT): numActive -= 1
else:
# If a process just completed, count it
if (tag == MPI_TAGS.DONE):
durat, pos, neg, new = data
logger.debug("- - %d Done after %s, pos %d, neg %d, new %d" %
(src, durat, pos, neg, new))
times[countDone] = durat
countDone += 1
num_pos += pos
num_neg += neg
num_new += new
# Send exit command
logger.debug("Sending exit to %d. %d Active." % (src, numActive))
comm.send(None, dest=src, tag=MPI_TAGS.EXIT)
fracDone = 1.0*countDone/(NUM_SNAPS-1)
logger.debug("%d/%d = %.4f Completed tasks!" % (countDone, NUM_SNAPS-1, fracDone))
logger.debug("Average time %.4f +- %.4f" % (np.average(times), np.std(times)))
logger.info("Totals: pos = %5d neg = %5d new = %3d" % (num_pos, num_neg, num_new))
return
def main():
"""
Calculate distribution functions for all galaxies in parallel.
Runs the `_runMaster` on process 0 which communicates with, and distributes jobs to, each of the
`_runSlave` processes on all the other processors.
"""
# Initialize MPI Parameters
# -------------------------
from mpi4py import MPI
comm = MPI.COMM_WORLD
rank = comm.rank
size = comm.size
if size <= 1: raise RuntimeError("Not setup for serial runs!")
sets = settings.Settings()
mstar = sets.MSTAR * MSOL
if rank == 0:
NAME = sys.argv[0]
print("\n%s\n%s\n%s" % (NAME, '=' * len(NAME), str(datetime.now())))
zio.check_path(sets.GET_DIR_LOGS())
# Make sure log-path is setup before continuing
comm.Barrier()
# Parse Arguments
# ---------------
args = _parseArguments(sets)
run = args.run
verbose = args.verbose
smooth = args.smooth
relAcc = args.relAcc
intSteps = args.intSteps
# Load logger
log = _loadMPILogger(rank, verbose, sets)
if rank < 2: print("Log (Rank %d) filename '%s'" % (rank, log.filename))
# log runtime parameters
log.info("run = %d " % (run))
log.info("version = %s " % (__version__))
log.info("MPI comm size = %d " % (size))
log.info("Rank = %d " % (rank))
log.info("")
log.info("verbose = %s " % (str(verbose)))
log.info("smooth = %d " % (smooth))
log.info("relAcc = %e " % (relAcc))
log.info("intSteps = %d " % (intSteps))
log.info("")
# Master Process
# --------------
if rank == 0:
beg_all = datetime.now()
try:
log.info("Running Master")
eps, ndens, ndD1, ndD2, dist_funcs, dfErrs, recDens = _runMaster(
run, comm, log)
except Exception as err:
_mpiError(comm, log, err)
end_all = datetime.now()
log.debug("Done after '%s'" % (str(end_all - beg_all)))
fname = sets.GET_DIST_FUNC_FILENAME(run=run, vers=__version__)
zio.check_path(fname)
data = {}
data['run'] = run
data['eps'] = eps
data['ndens'] = ndens
data['ndD1'] = ndD1
data['ndD2'] = ndD2
data['distfuncs'] = dist_funcs
data['dferrs'] = dfErrs
data['recdens'] = recDens
data['version'] = __version__
zio.dictToNPZ(data, fname, verbose=True)
log.info("Saved data to '%s'" % (fname))
# Slave Processes
# ---------------
else:
try:
log.info("Running slave")
_runSlave(comm, smooth, relAcc, intSteps, mstar, log)
except Exception as err:
_mpiError(comm, log, err)
log.info("Done.")
return
def _load_bh_hosts_snap_table(run, snap, log, version=None, load_saved=True):
"""Load pre-existing, or manage the creation of the particle offset table.
Arguments
---------
run <int> : illustris simulation number {1, 3}
snap <int> : illustris snapshot number {1, 135}
load_saved <bool> : optional, load existing table
verbose <bool> : optional, print verbose output
Returns
-------
offsetTable <dict> : particle offset table, see `ParticleHosts` docs for more info.
"""
log.debug("particle_hosts._load_bh_hosts_snap_table()")
beg_all = datetime.now()
fname = _get_filename_bh_hosts_snap_table(run, snap)
_path = zio.check_path(fname)
if not os.path.isdir(_path):
log.raise_error("Error with path for '{}' (path: '{}')".format(
fname, _path))
# Load Existing Save
# ------------------
if load_saved:
# fname = FILENAME_BH_HOSTS_SNAP_TABLE(run, snap, version)
log.info("Loading from save '{}'".format(fname))
# Make sure path exists
if os.path.exists(fname):
host_table = h5py.File(fname, 'r')
else:
log.warning(
"File '{}' does not Exist. Reconstructing.".format(fname))
load_saved = False
# Reconstruct Hosts Table
# -----------------------
if not load_saved:
log.info("Constructing Offset Table for Snap {}".format(snap))
COSMO = Illustris_Cosmology_TOS()
if version is not None:
log.raise_error("Can only create version '{}'".format(_VERSION))
# Construct Offset Data
beg = datetime.now()
halo_nums, subh_nums, offsets = _construct_offset_table(run, snap, log)
log.after("Loaded {} entries".format(len(halo_nums)), beg, beg_all)
# Construct BH index Data
# Catch errors for bad snapshots
try:
bh_inds, bh_ids = _construct_bh_index_table(run, snap, log)
except:
# If this is a known bad snapshot, set values to None
if snap in COSMO.GET_BAD_SNAPS(run):
log.info("bad snapshot: run {}, snap {}".format(run, snap))
bh_inds = None
bh_ids = None
bh_halos = None
bh_subhs = None
# If this is not a known problem, still raise error
else:
log.error(
"this is not a known bad snapshot: run {}, snap {}".format(
run, snap))
raise
# On success, Find BH Subhalos
else:
bin_inds = np.digitize(bh_inds, offsets[:, PARTICLE.BH]).astype(
DTYPE.INDEX) - 1
if any(bin_inds < 0):
log.raise_error("Some bh_inds not matched!! '{}'".format(
str(bin_inds)))
bh_halos = halo_nums[bin_inds]
bh_subhs = subh_nums[bin_inds]
# Save To Dict
# ------------
log.info("Writing snapshot bh-host table to file '{}'".format(fname))
beg = datetime.now()
with h5py.File(fname, 'w') as host_table:
# Metadata
host_table.attrs[OFFTAB.RUN] = run
host_table.attrs[OFFTAB.SNAP] = snap
host_table.attrs[OFFTAB.VERSION] = _VERSION
host_table.attrs[OFFTAB.CREATED] = datetime.now().ctime()
host_table.attrs[OFFTAB.FILENAME] = fname
# BH Data
host_table.create_dataset(OFFTAB.BH_INDICES, data=bh_inds)
host_table.create_dataset(OFFTAB.BH_IDS, data=bh_ids)
host_table.create_dataset(OFFTAB.BH_HALOS, data=bh_halos)
host_table.create_dataset(OFFTAB.BH_SUBHALOS, data=bh_subhs)
log.after("Saved to '{}', size {}".format(fname,
zio.get_file_size(fname)),
beg,
beg_all,
lvl=log.WARNING)
host_table = h5py.File(fname, 'r')
return host_table
def _load_bh_hosts_table(run, log=None, load_saved=True, version=None):
"""Merge individual snapshot's blackhole hosts files into a single file.
Arguments
---------
run <int> : illustris simulation number {1, 3}
load_saved <bool> : optional, load existing save if possible
version <flt> : optional, target version number
Returns
-------
bh_hosts <dict> : table of hosts for all snapshots
"""
# log = check_log(log, run=run)
log.debug("particle_hosts._load_bh_hosts_table()")
beg_all = datetime.now()
fname_bh_hosts = _get_filename_bh_hosts_table(run)
_path = zio.check_path(fname_bh_hosts)
if not os.path.isdir(_path):
log.raise_error("Error with path for '{}' (path: '{}')".format(
fname_bh_hosts, _path))
# Load Existing Save
# ------------------
if load_saved:
log.info("Loading from save '{}'".format(fname_bh_hosts))
# Make sure path exists
if os.path.exists(fname_bh_hosts):
hosts_table = h5py.File(fname_bh_hosts, 'r')
else:
log.warning("File '{}' does not Exist. Reconstructing.".format(
fname_bh_hosts))
load_saved = False
# Reconstruct Hosts Table
# -----------------------
if not load_saved:
log.info("Constructing Hosts Table")
COSMO = Illustris_Cosmology_TOS()
if version is not None:
log.raise_error("Can only create version '{}'".format(_VERSION))
# Select the dict-keys for snapshot hosts to transfer
host_keys = [
OFFTAB.BH_IDS, OFFTAB.BH_INDICES, OFFTAB.BH_HALOS,
OFFTAB.BH_SUBHALOS
]
# Save To HDF5
# ------------
log.info("Writing bh-host table to file '{}'".format(fname_bh_hosts))
beg = datetime.now()
with h5py.File(fname_bh_hosts, 'w') as host_table:
# Metadata
host_table.attrs[OFFTAB.RUN] = run
host_table.attrs[OFFTAB.VERSION] = _VERSION
host_table.attrs[OFFTAB.CREATED] = datetime.now().ctime()
host_table.attrs[OFFTAB.FILENAME] = fname_bh_hosts
for snap in tqdm.trange(COSMO.NUM_SNAPS, desc="Loading snapshots"):
# Load Snapshot BH-Hosts
htab_snap = _load_bh_hosts_snap_table(run,
snap,
log,
load_saved=True)
# Extract and store target data
snap_str = "{:03d}".format(snap)
# Create a group for this snapshot
snap_group = host_table.create_group(snap_str)
# Transfer all parameters over
for hkey in host_keys:
snap_group.create_dataset(hkey, data=htab_snap[hkey][:])
log.after("Saved to '{}', size {}".format(
fname_bh_hosts, zio.get_file_size(fname_bh_hosts)),
beg,
beg_all,
lvl=log.WARNING)
host_table = h5py.File(fname_bh_hosts, 'r')
return hosts_table
def plotScattering(sample, snap, mbhb, log, plotNames):
"""Illustrate the scattering calculation for a single, sample system.
Performs calculation by calling the 'scattering()' method, just like in
"MBHBinaryEvolution.py".
Arguments
---------
sample : int
Target galaxy/merger number to examine (this is the number out of *all* binaries,
not just the valid ones [included in ``mbhb.evolution``]).
snap : int
Illustris snapshot number {1, 135}.
mbhb : `Binaries.MBHBinaries` object
log : `logging.Logger` object
plotNames : str
Base name of plots.
Returns
-------
plotNames : list of str
Filenames of the plots created.
"""
log.debug("plotScattering()")
PLOT_A10 = True # LC Occupancy
PLOT_A11 = True # Flux
PLOT_A15 = True # Flux vs. Separation
PLOT_A02 = False # Model Galaxy
PLOT_A08 = False # Dist Func
from . import GM_Figures
figNames = []
# Initialization
# --------------
radialRange = np.array(mbhb.sets.RADIAL_RANGE_MODELS) * PC
numSeps = mbhb.sets.PLOT_SCATTERING_SAMPLE_SEPS
# Convert from `sample` number, of all binaries to index for valid (``mbhb.evolution``) ones
val_inds = np.where(mbhb.valid)[0]
valid_sample = np.where(val_inds == sample)[0]
if valid_sample.size == 1:
valid_sample = valid_sample[0]
else:
raise ValueError("`sample` '{}' returned '{}' from `val_inds`".format(sample, valid_sample))
mstar = mbhb.galaxies.mstar
log.debug(" - Sample subhalo %d, Snapshot %d" % (sample, snap))
# Binary Properties (snapshot dependent)
# `evolution` class includes only valid binaries, so use `valid_sample`
# m1 = np.max(mbhb.evolution.masses[valid_sample, snap])
# m2 = np.min(mbhb.evolution.masses[valid_sample, snap])
m1 = np.max(mbhb.initMasses[sample])
m2 = np.min(mbhb.initMasses[sample])
# Galaxy properties (snapshot independent)
# `galaxies` class includes *all* binaries, so use `sample` itself
gals = mbhb.galaxies
eps = gals.eps[sample]
# functions of energy
rads = gals.rads
periods = gals.perOrb[sample]
dist_func = gals.dist_func[sample]
diffCoef = gals.diffCoef[sample]
j2Circs = gals.j2Circ[sample]
dnStarsAll = gals.dnStarsAll[sample]
ndensStars = gals.densStars[sample]/mstar
radHard = gals.rads_hard[sample]
numStarsAll = sp.integrate.cumtrapz(dnStarsAll[::-1], eps[::-1], initial=0.0)[::-1]
loss_cone = Loss_Cone_Explicit(mbhb.sets, log)
# Wrap the Scattering function (for convenience)
def evalScattering(binSep):
# retvals = scattering(m1, m2, binSep, rads, eps, periods, j2Circs, diffCoef, dist_func,
# [sample], radHard, mbhb.sets)
retvals = loss_cone.harden(
m1, m2, binSep, rads, eps, periods, j2Circs, diffCoef, dist_func,
[sample], radHard, mbhb.sets)
radLC, j2LC, enrLC, dnStarsFLC, numStarsFLC, numStarsSSLC, \
dfStarsFLC, dfStarsSSLC, fluxStarsFLC, fluxStarsSSLC, flux, dadt_lc = retvals
return dnStarsFLC, numStarsFLC, dfStarsFLC, fluxStarsFLC, dfStarsSSLC, fluxStarsSSLC
num_flc = np.zeros(numSeps)
flx_flc = np.zeros(numSeps)
flx_sslc = np.zeros(numSeps)
dadt_sslc = np.zeros(numSeps)
dadt_flc = np.zeros(numSeps)
subPlotNames = zio.modify_filename(plotNames, prepend='stellar_scattering/')
zio.check_path(subPlotNames)
# Iterate over range of binary separations, plot profiles for each
log.debug(" - Calculting scattering for %d binary separations" % (numSeps))
flx_seps = zmath.spacing(radialRange, scale='log', num=numSeps)
for ii, binSep in enumerate(tqdm.tqdm(flx_seps, desc="Calculating scattering")):
dnStarsFLC, numStarsFLC, dfStarsFLC, fluxStarsFLC, dfStarsSSLC, fluxStarsSSLC = \
evalScattering(binSep)
hard_sslc = dadt_scattering(m1+m2, binSep, np.max(fluxStarsSSLC), mstar)
hard_flc = dadt_scattering(m1+m2, binSep, np.max(fluxStarsFLC), mstar)
eps_rad = zmath.spline(gals.rads, gals.eps[sample], log=True, pos=True, extrap=True)
sepEner = eps_rad(binSep)
# Plot Fig A10 - Loss-Cone Occupancy
if PLOT_A10:
fig = GM_Figures.figa10_lc_occupancy(
gals.rads, eps, dnStarsAll, dnStarsFLC, numStarsAll, numStarsFLC,
binr=binSep, bine=sepEner)
fname1 = subPlotNames + "lcPipe_figa10-occ_%02d.png" % (ii)
zplot.saveFigure(fig, fname1, verbose=False) # , log=log)
plt.close(fig)
# Plot Fig A11 - loss-cone fluxes
if PLOT_A11:
fig = GM_Figures.figa11_lc_flux(
eps, gals.rads, dfStarsFLC, dfStarsSSLC, fluxStarsFLC, fluxStarsSSLC,
binr=binSep, bine=sepEner)
fname2 = subPlotNames + "lcPipe_figa11-flx_%02d.png" % (ii)
zplot.saveFigure(fig, fname2, verbose=False) # , log=log)
plt.close(fig)
# Store values to arrays
num_flc[ii] = np.max(numStarsFLC)
flx_flc[ii] = np.max(fluxStarsFLC)
flx_sslc[ii] = np.max(fluxStarsSSLC)
dadt_sslc[ii] = hard_sslc
dadt_flc[ii] = hard_flc
# pbar.update(ii)
# pbar.finish()
if PLOT_A10:
log.debug(" - - Saved FigA10 to (e.g.) '%s'" % (fname1))
if PLOT_A11:
log.debug(" - - Saved FigA11 to (e.g.) '%s'" % (fname2))
# Plot Figs A15 - Scattering vs. Separations
if PLOT_A15:
log.debug(" - Plotting Fig15")
fig = GM_Figures.figA15_flux_sep(flx_seps, flx_flc, flx_sslc, dadt_flc, dadt_sslc,
sample, snap)
fname = plotNames + "lcPipe_figa15-sep.png"
zplot.saveFigure(fig, fname, verbose=False, log=log)
plt.close(fig)
figNames.append(fname)
# Plot Figure A02 - Density/Mass Profiles
if PLOT_A02:
log.info(" - Plotting FigA02")
dens = [gals.densStars[sample], gals.densDM[sample]]
mass = [gals.massStars[sample], gals.massDM[sample], gals.massTot[sample]]
names = ['Stars', 'DM', 'Total']
vels = [gals.vdisp_stars[sample]]
fig = GM_Figures.figA02_ModelGalaxy(gals.rads, dens, mass, vels, names=names)
fname = plotNames + "lcPipe_figa02-dens.png"
zplot.saveFigure(fig, fname, verbose=False, log=log)
plt.close(fig)
figNames.append(fname)
# Plot Figure A08 - Reconstructed Number Density
if PLOT_A08:
log.debug(" - Plotting FigA08")
df_pot = zmath.spline(eps, dist_func, mono=True, pos=True, sort=True, log=True)
dfErrs = np.zeros(np.size(eps))
fig = GM_Figures.figA08_distFunc(eps, gals.rads, ndensStars, dist_func, dfErrs, df_pot, log,
sample=sample)
fname = plotNames + "lcPipe_figa08-distfunc.png"
zplot.saveFigure(fig, fname, verbose=False, log=log)
plt.close(fig)
figNames.append(fname)
return figNames
