def save_fig(self, fig, name, subdir=None, fignum=True, verbose=True, close=False): if (not name.endswith('.pdf')) and (not name.endswith('.png')): name += '.pdf' if subdir is not None: fname = os.path.join(self.output_plots, subdir, name) self.check_path(fname) fname = zio.modify_exists(fname) fig.savefig(fname) if verbose: print("Saved to '{}'".format(fname)) if fignum is not None: if isinstance(fignum, str): name = zio.modify_filename(name, prepend=fignum + "_") fname = os.path.join(self.path_output_figs, name) zio.check_path(fname) fig.savefig(fname) if verbose: print("Saved to '{}'".format(fname)) # if close: # plt.close('all') return fname
def _loadLogger(name, verbose=True, debug=False, run=None, rank=None, version=None, tofile=True): """Initialize a ``logging.Logger`` object for output messages. All processes log to output files, and the root process also outputs to `stdout`. Constructs the log name based on the `name` argument, which should be the `__file__` parameter from the script which calls this method. Arguments --------- name : str Base (file)name from which to construct the log's name, and log filename. verbose : bool Print 'verbose' (``logging.INFO``) output to stdout. Overridden if ``debug == True``. debug : bool Print extremely verbose (``logging.DEBUG``) output to stdout. Overrides `verbose` setting. run : int or `None` Illustris run number {1,3}. Added to log filename if provided. rank : int or `None` Rank of the current process for MPI runs. Added to the log filename if provided. version : str or `None` Current version of script being run. Added to the log filename if provided. tofile : bool Whether output should also be logged to a file. Returns ------- logger : ``logging.Logger`` object Object for output logging. """ # Get logger and log-file names logName, logFilename = _GET_LOG_NAMES(name, run=run, rank=rank, version=version) # Make sure directory exists zio.check_path(logFilename) # Determine verbosity level if debug: strLvl = logging.DEBUG elif verbose: strLvl = logging.INFO else: strLvl = logging.WARNING # Turn off file-logging if not tofile: logFilename = None fileLvl = logging.DEBUG # Create logger if rank == 0 or rank is None: kw = dict(level_stream=strLvl) else: kw = dict(tostr=False) # logger = zio.getLogger(logName, tofile=logFilename, fileLevel=fileLvl, strLevel=strLvl, **kw) logger = zio.get_logger(logName, tofile=logFilename, level_file=fileLvl, **kw) return logger
def __init__(self, sim_path, create=True): # Make sure the path does NOT include a terminal path-separator sim_path = os.path.dirname(os.path.join(sim_path, '')) # If the 'output' folder is specified, move up a directory to the base-path if sim_path.endswith(self._DNAME_OUTPUT): sim_path = os.path.dirname(sim_path) self.sim = sim_path self.output = os.path.join(sim_path, self._DNAME_OUTPUT) # self.analysis = os.path.join(sim_path, self._DNAME_ANALYSIS) self.analysis = os.path.join(self.output, self._DNAME_ANALYSIS) _check_paths = [self.output, self.analysis] if create: for cp in _check_paths: zio.check_path(cp) return
def _checkLog(log, run=None, debug=Settings.debug, verbose=Settings.verbose): """Create a default logging object if one is not given. """ from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.rank if not rank: zio.check_path(_LOG_DIR) comm.Barrier() if log is None: log = constants._loadLogger( __file__, debug=debug, verbose=debug, run=run, rank=rank, version=__version__) header = "\n%s\n%s\n%s" % (__file__, '='*len(__file__), str(datetime.now())) log.debug(header) if not rank: print(("Log filename = ", log.filename)) return log
def save_fig(self, fig, fname, path=None, subdir=None, snap_num=None, modify=False, verbose=False, **kwargs): fname = self.fname(fname, path=path, subdir=subdir, snap_num=snap_num, modify=modify) kwargs.setdefault('transparent', True) kwargs.setdefault('dpi', 100) zio.check_path(fname) fig.savefig(fname, **kwargs) if verbose: print("saved to '{}' size: {}".format(fname, zio.get_file_size(fname))) return fname
def test_filenames(): print("particle_hosts_test.filenames_test()") for run in range(1, 4): print("run = '{}'".format(run)) processed_dir = particle_hosts._get_path_processed(run) print("processed_dir = '{}'".format(processed_dir)) zio.check_path(processed_dir) assert_true(os.path.exists(processed_dir)) # make sure directory is writeable test_fname = os.path.join(processed_dir, 'test_123_test.txt') print("Trying to write to file '{}'".format(test_fname)) with open(test_fname, 'w') as test: test.write('hello') print("File '{}' Exists: {}".format(test_fname, str(os.path.exists(test_fname)))) assert_true(os.path.exists(test_fname)) print("Deleting file '{}'".format(test_fname)) os.remove(test_fname) print("File '{}' Exists: {}".format(test_fname, str(os.path.exists(test_fname)))) assert_false(os.path.exists(test_fname)) # Offset table print("loading path for: offset table") path = particle_hosts._get_filename_offset_table(run, 135, version='1.0') print(path) # bh-hosts-snap table print("loading path for: bh-hosts-snap table") path = particle_hosts._get_filename_bh_hosts_snap_table(run, 135, version='1.0') print(path) # bh-hosts table print("loading path for: bh-hosts table") path = particle_hosts._get_filename_bh_hosts_table(run, 135, version='1.0') print(path) print("using 'run' 0 should fail") assert_raises(KeyError, particle_hosts._get_path_processed, 0) return
def _get_path_processed(run): """ _ILLUSTRIS_RUN_NAMES = {1: "L75n1820FP", 2: "L75n910FP", 3: "L75n455FP"} _PROCESSED_DIR = "/n/home00/lkelley/illustris/data/{}/output/postprocessing/" """ from ..constants import _ILLUSTRIS_RUN_NAMES # Make sure path ends in '/' path = os.path.join(_PROCESSED_DIR.format(_ILLUSTRIS_RUN_NAMES[run]), '') if zio.check_path(path) is None: raise OSError("_get_path_processed(): Path failed '{}'".format(path)) return path
def _reorganize_files(core, raw_fnames, temp_fnames): log = core.log log.debug("details._reorganize_files()") NUM_SNAPS = core.sets.NUM_SNAPS snap_scales = core.cosmo.scales() temps = [zio.modify_filename(tt, prepend='_') for tt in temp_fnames] # Open new ASCII, Temp dets files # Make sure path is okay zio.check_path(temps[0]) # Open each temp file # temp_files = [open(tfil, 'w') for tfil in temp_fnames] temp_files = [open(tfil, 'w') for tfil in temps] num_temp = len(temp_files) num_raw = len(raw_fnames) log.info("Organizing {:d} raw files into {:d} temp files".format( num_raw, num_temp)) prec = _DEF_PRECISION all_num_lines_in = 0 all_num_lines_out = 0 # Iterate over all Illustris Details Files # ---------------------------------------- for ii, raw in enumerate(core.tqdm(raw_fnames, desc='Raw files')): log.debug("File {}: '{}'".format(ii, raw)) lines = [] scales = [] # Load all lines and entry scale-factors from raw dets file for dline in open(raw): lines.append(dline) # Extract scale-factor from line detScale = DTYPE.SCALAR(dline.split()[1]) scales.append(detScale) # Convert to array lines = np.array(lines) scales = np.array(scales) num_lines_in = scales.size # If file is empty, continue if num_lines_in == 0: log.debug("\tFile empty") continue log.debug("\tLoaded {}".format(num_lines_in)) # Round snapshot scales to desired precision scales_round = np.around(snap_scales, -prec) # Find snapshots following each entry (right-edge) or equal (include right: 'right=True') # `-1` to put binaries into the snapshot UP-TO that scalefactor # snap_nums = np.digitize(scales, scales_round, right=True) - 1 snap_nums = np.digitize(scales, scales_round, right=True) # For each Snapshot, write appropriate lines num_lines_out = 0 for snap in range(NUM_SNAPS): inds = (snap_nums == snap) num_lines_out_snap = np.count_nonzero(inds) if num_lines_out_snap == 0: continue temp_files[snap].writelines(lines[inds]) # log.debug("\t\tWrote {} lines to snap {}".format(num_lines_out_snap, snap)) num_lines_out += num_lines_out_snap if num_lines_out != num_lines_in: log.error("File {}, '{}'".format(ii, raw)) log.raise_error("Wrote {} lines, loaded {} lines!".format( num_lines_out, num_lines_in)) all_num_lines_in += num_lines_in all_num_lines_out += num_lines_out # Close out dets files tot_size = 0.0 log.info("Closing files, checking sizes") for ii, newdf in enumerate(temp_files): newdf.close() tot_size += os.path.getsize(newdf.name) ave_size = tot_size / (1.0 * len(temp_files)) size_str = zio.bytes_string(tot_size) ave_size_str = zio.bytes_string(ave_size) log.info("Total temp size = '{}', average = '{}'".format( size_str, ave_size_str)) log.info("Input lines = {:d}, Output lines = {:d}".format( all_num_lines_in, all_num_lines_out)) if (all_num_lines_in != all_num_lines_out): log.raise_error( "input lines {}, does not match output lines {}!".format( all_num_lines_in, all_num_lines_out)) log.info("Renaming temporary files...") for ii, (aa, bb) in enumerate(zip(temps, temp_fnames)): if ii == 0: log.debug("'{}' ==> '{}'".format(aa, bb)) shutil.move(aa, bb) return
def _runMaster(run, comm): """ Run master process which manages all of the secondary ``slave`` processes. Details ------- - Retrieves merger, snapshot and subhalo indices - Iterates over snapshots and merger-subhalo pairs, distributing them to ``slave`` processes which load each subhalo profile and writes them to individual-subhalo files. - Loads most-bound particle ID numbers from group caalog for each snapshot and distributes this to each slave-process as-well. - Tracks how-many and which process (and subhalos) finish successfully """ from mpi4py import MPI stat = MPI.Status() # rank = comm.rank size = comm.size print(" - Initializing") merger_snaps, snap_mergers, subh_ind_out, subh_ind_in = \ get_merger_and_subhalo_indices(run, verbose=True) # Get all subhalos for each snapshot (including duplicates and missing) snapSubh = [subh_ind_out[smrg] for smrg in snap_mergers] # Get unique subhalos for each snapshot, discard duplicates snapSubh_uni = [np.array(list(set(ssubh))) for ssubh in snapSubh] # Discard missing matches ('-1') snapSubh_uni = [ssubh[np.where(ssubh != -1)] for ssubh in snapSubh_uni] numUni = [len(ssubh) for ssubh in snapSubh_uni] numUniTot = np.sum(numUni) numMSnaps = np.count_nonzero(numUni) print( (" - - %d Unique subhalos over %d Snapshots" % (numUniTot, numMSnaps))) # Iterate over Snapshots and Subhalos # =================================== # distribute tasks to slave processes count = 0 new = 0 exist = 0 fail = 0 times = np.zeros(numUniTot) statFileName = _GET_ENVIRONMENTS_STATUS_FILENAME(run) statFile = open(statFileName, 'w') print((" - - Opened status file '%s'" % (statFileName))) statFile.write('%s\n' % (str(datetime.now()))) beg = datetime.now() for snap, subs in zmath.renumerate(snapSubh_uni): if len(subs) <= 0: continue # Create output directory (subhalo doesn't matter since only creating dir) # don't let slave processes create it - makes conflicts fname = _GET_MERGER_SUBHALO_FILENAME(run, snap, 0) zio.check_path(fname) # Get most bound particles for each subhalo in this snapshot mostBound = Subhalo.importGroupCatalogData(run, snap, subhalos=subs, fields=[SUBHALO.MOST_BOUND], verbose=False) # Go over each subhalo for boundID, subhalo in zip(mostBound, subs): # Write status to file dur = (datetime.now() - beg) statStr = 'Snap %3d %8d/%8d = %.4f in %s %8d new %8d exist %8d fail\n' % \ (snap, count, numUniTot, 1.0*count/numUniTot, str(dur), new, exist, fail) statFile.write(statStr) statFile.flush() # Look for available slave process data = comm.recv(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=stat) source = stat.Get_source() tag = stat.Get_tag() # Track number of completed profiles if tag == _TAGS.DONE: retStat, durat = data times[count] = durat count += 1 if retStat == _ENVSTAT.NEWF: new += 1 elif retStat == _ENVSTAT.EXST: exist += 1 else: fail += 1 # Distribute tasks comm.send([snap, subhalo, boundID], dest=source, tag=_TAGS.START) statFile.write('\n\nDone after %s' % (str(datetime.now() - beg))) statFile.close() # Close out all Processes # ======================= numActive = size - 1 print((" - Exiting %d active processes" % (numActive))) while (numActive > 0): # Find available slave process data = comm.recv(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=stat) source = stat.Get_source() tag = stat.Get_tag() # If we're recieving exit confirmation, count it if tag == _TAGS.EXIT: numActive -= 1 else: # If a process just completed, count it if tag == _TAGS.DONE: times[count] = data[1] count += 1 if data[0]: new += 1 # Send exit command comm.send(None, dest=source, tag=_TAGS.EXIT) print((" - - %d/%d = %.4f Completed tasks!" % (count, numUniTot, 1.0 * count / numUniTot))) print((" - - %d New Files" % (new))) return
def main(): """Create master and many slave processes to extract BH snapshot data. """ # Initialize MPI Parameters # ------------------------- from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.rank size = comm.size if (size <= 1): raise RuntimeError("Not setup for serial runs!") if (rank == 0): NAME = sys.argv[0] print(("\n{:s}\n{:s}\n{:s}".format(NAME, '='*len(NAME), str(datetime.now())))) zio.check_path(bh_constants._LOG_DIR) # Make sure log-path is setup before continuing comm.Barrier() # Parse Arguments # --------------- args = _parseArguments() run = args.run verbose = args.verbose # Load logger logger = bh_constants._loadLogger(__file__, verbose=verbose, run=run, rank=rank, version=_VERSION) logger.info("run = %d " % (run)) logger.info("version = %.2f" % (_VERSION)) logger.info("MPI comm size = %d " % (size)) logger.info("Rank = %d " % (rank)) logger.info("") logger.info("verbose = %s " % (str(verbose))) logger.info("") # Master Process # -------------- if (rank == 0): beg_all = datetime.now() try: logger.debug("Running Master") _runMaster(run, comm, logger) except Exception as err: zio._mpiError(comm, log=logger, err=err) end_all = datetime.now() logger.debug("Done after '%s'" % (str(end_all-beg_all))) logger.info("Merging snapshots") loadBHSnapshotData(run, logger=logger, loadsave=False) # Slave Processes # --------------- else: try: logger.debug("Running slave") _runSlave(run, comm, logger) except Exception as err: zio._mpiError(comm, log=logger, err=err) logger.debug("Done.") return
def _runMaster(run, comm, logger): """Distribute snapshots and associated mrgs to individual slave tasks for loading. Loads ``mergers`` and distributes them, based on snapshot, to slave processes run by the ``_runSlave`` method. A status file is created to track progress. Once all snapshots are distributed, this method directs the termination of all of the slave processes. Arguments --------- run : int, Ollustris simulation number {1, 3}. comm : ``mpi4py.MPI.Intracomm`` object, MPI intracommunicator object, `COMM_WORLD`. logger : ``logging.Logger`` object, Object for logging. """ from mpi4py import MPI stat = MPI.Status() rank = comm.rank size = comm.size logger.info("BHSnapshotData._runMaster()") logger.debug("Rank %d/%d" % (rank, size)) # Make sure output directory exists fname = _GET_BH_SINGLE_SNAPSHOT_FILENAME(run, 0) zio.check_path(fname) # Load BH Mergers logger.info("Loading BH Mergers") mrgs = mergers.load_fixed_mergers(run, loadsave=True, verbose=False) numMergers = mrgs[MERGERS.NUM] logger.debug("- Loaded %d mrgs" % (numMergers)) # Init status file statFileName = bh_constants._GET_STATUS_FILENAME(__file__, run=run, version=_VERSION) statFile = open(statFileName, 'w') logger.debug("Opened status file '%s'" % (statFileName)) statFile.write('%s\n' % (str(datetime.now()))) beg = datetime.now() num_pos = 0 num_neg = 0 num_new = 0 countDone = 0 count = 0 times = np.zeros(NUM_SNAPS-1) # Iterate Over Snapshots # ---------------------- # Go over snapshots in random order to get a better estimate of ETA/duration snapList = np.arange(NUM_SNAPS-1) np.random.shuffle(snapList) logger.info("Iterating over snapshots") pbar = zio.getProgressBar(NUM_SNAPS-1) for snapNum in snapList: logger.debug("- Snap %d, count %d, done %d" % (snapNum, count, countDone)) # Get Mergers occuring just after Snapshot `snapNum` mrgs = mrgs[MERGERS.MAP_STOM][snapNum+1] nums = len(mrgs) targetIDs = mrgs[MERGERS.IDS][mrgs] logger.debug("- %d Mergers from snapshot %d" % (nums, snapNum+1)) # Look for available slave process data = comm.recv(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=stat) src = stat.Get_source() tag = stat.Get_tag() logger.debug("- Received signal from %d" % (src)) # Track number of completed profiles if (tag == MPI_TAGS.DONE): durat, pos, neg, new = data logger.debug("- - Done after %s, pos %d, neg %d, new %d" % (durat, pos, neg, new)) times[countDone] = durat num_pos += pos num_neg += neg num_new += new countDone += 1 # Distribute tasks logger.debug("- Sending new task to %d" % (src)) comm.send([snapNum, mrgs, targetIDs, numMergers], dest=src, tag=MPI_TAGS.START) logger.debug("- New task sent") # Write status to file and log dur = (datetime.now()-beg) fracDone = 1.0*countDone/(NUM_SNAPS-1) statStr = 'Snap %3d (rank %03d) %8d/%8d = %.4f in %s %8d pos %8d neg %3d new\n' % \ (snapNum, src, countDone, NUM_SNAPS-1, fracDone, str(dur), num_pos, num_neg, num_new) statFile.write(statStr) statFile.flush() logger.debug(statStr) count += 1 pbar.update(count) statFile.write('\n\nDone after %s' % (str(datetime.now()-beg))) statFile.close() pbar.finish() # Close out all Processes # ----------------------- numActive = size-1 logger.info("Exiting %d active processes" % (numActive)) while(numActive > 0): # Find available slave process data = comm.recv(source=MPI.ANY_SOURCE, tag=MPI.ANY_TAG, status=stat) src = stat.Get_source() tag = stat.Get_tag() logger.debug("- Received signal from %d" % (src)) # If we're recieving exit confirmation, count it if (tag == MPI_TAGS.EXIT): numActive -= 1 else: # If a process just completed, count it if (tag == MPI_TAGS.DONE): durat, pos, neg, new = data logger.debug("- - %d Done after %s, pos %d, neg %d, new %d" % (src, durat, pos, neg, new)) times[countDone] = durat countDone += 1 num_pos += pos num_neg += neg num_new += new # Send exit command logger.debug("Sending exit to %d. %d Active." % (src, numActive)) comm.send(None, dest=src, tag=MPI_TAGS.EXIT) fracDone = 1.0*countDone/(NUM_SNAPS-1) logger.debug("%d/%d = %.4f Completed tasks!" % (countDone, NUM_SNAPS-1, fracDone)) logger.debug("Average time %.4f +- %.4f" % (np.average(times), np.std(times))) logger.info("Totals: pos = %5d neg = %5d new = %3d" % (num_pos, num_neg, num_new)) return
def main(): """ Calculate distribution functions for all galaxies in parallel. Runs the `_runMaster` on process 0 which communicates with, and distributes jobs to, each of the `_runSlave` processes on all the other processors. """ # Initialize MPI Parameters # ------------------------- from mpi4py import MPI comm = MPI.COMM_WORLD rank = comm.rank size = comm.size if size <= 1: raise RuntimeError("Not setup for serial runs!") sets = settings.Settings() mstar = sets.MSTAR * MSOL if rank == 0: NAME = sys.argv[0] print("\n%s\n%s\n%s" % (NAME, '=' * len(NAME), str(datetime.now()))) zio.check_path(sets.GET_DIR_LOGS()) # Make sure log-path is setup before continuing comm.Barrier() # Parse Arguments # --------------- args = _parseArguments(sets) run = args.run verbose = args.verbose smooth = args.smooth relAcc = args.relAcc intSteps = args.intSteps # Load logger log = _loadMPILogger(rank, verbose, sets) if rank < 2: print("Log (Rank %d) filename '%s'" % (rank, log.filename)) # log runtime parameters log.info("run = %d " % (run)) log.info("version = %s " % (__version__)) log.info("MPI comm size = %d " % (size)) log.info("Rank = %d " % (rank)) log.info("") log.info("verbose = %s " % (str(verbose))) log.info("smooth = %d " % (smooth)) log.info("relAcc = %e " % (relAcc)) log.info("intSteps = %d " % (intSteps)) log.info("") # Master Process # -------------- if rank == 0: beg_all = datetime.now() try: log.info("Running Master") eps, ndens, ndD1, ndD2, dist_funcs, dfErrs, recDens = _runMaster( run, comm, log) except Exception as err: _mpiError(comm, log, err) end_all = datetime.now() log.debug("Done after '%s'" % (str(end_all - beg_all))) fname = sets.GET_DIST_FUNC_FILENAME(run=run, vers=__version__) zio.check_path(fname) data = {} data['run'] = run data['eps'] = eps data['ndens'] = ndens data['ndD1'] = ndD1 data['ndD2'] = ndD2 data['distfuncs'] = dist_funcs data['dferrs'] = dfErrs data['recdens'] = recDens data['version'] = __version__ zio.dictToNPZ(data, fname, verbose=True) log.info("Saved data to '%s'" % (fname)) # Slave Processes # --------------- else: try: log.info("Running slave") _runSlave(comm, smooth, relAcc, intSteps, mstar, log) except Exception as err: _mpiError(comm, log, err) log.info("Done.") return
def _load_bh_hosts_snap_table(run, snap, log, version=None, load_saved=True): """Load pre-existing, or manage the creation of the particle offset table. Arguments --------- run <int> : illustris simulation number {1, 3} snap <int> : illustris snapshot number {1, 135} load_saved <bool> : optional, load existing table verbose <bool> : optional, print verbose output Returns ------- offsetTable <dict> : particle offset table, see `ParticleHosts` docs for more info. """ log.debug("particle_hosts._load_bh_hosts_snap_table()") beg_all = datetime.now() fname = _get_filename_bh_hosts_snap_table(run, snap) _path = zio.check_path(fname) if not os.path.isdir(_path): log.raise_error("Error with path for '{}' (path: '{}')".format( fname, _path)) # Load Existing Save # ------------------ if load_saved: # fname = FILENAME_BH_HOSTS_SNAP_TABLE(run, snap, version) log.info("Loading from save '{}'".format(fname)) # Make sure path exists if os.path.exists(fname): host_table = h5py.File(fname, 'r') else: log.warning( "File '{}' does not Exist. Reconstructing.".format(fname)) load_saved = False # Reconstruct Hosts Table # ----------------------- if not load_saved: log.info("Constructing Offset Table for Snap {}".format(snap)) COSMO = Illustris_Cosmology_TOS() if version is not None: log.raise_error("Can only create version '{}'".format(_VERSION)) # Construct Offset Data beg = datetime.now() halo_nums, subh_nums, offsets = _construct_offset_table(run, snap, log) log.after("Loaded {} entries".format(len(halo_nums)), beg, beg_all) # Construct BH index Data # Catch errors for bad snapshots try: bh_inds, bh_ids = _construct_bh_index_table(run, snap, log) except: # If this is a known bad snapshot, set values to None if snap in COSMO.GET_BAD_SNAPS(run): log.info("bad snapshot: run {}, snap {}".format(run, snap)) bh_inds = None bh_ids = None bh_halos = None bh_subhs = None # If this is not a known problem, still raise error else: log.error( "this is not a known bad snapshot: run {}, snap {}".format( run, snap)) raise # On success, Find BH Subhalos else: bin_inds = np.digitize(bh_inds, offsets[:, PARTICLE.BH]).astype( DTYPE.INDEX) - 1 if any(bin_inds < 0): log.raise_error("Some bh_inds not matched!! '{}'".format( str(bin_inds))) bh_halos = halo_nums[bin_inds] bh_subhs = subh_nums[bin_inds] # Save To Dict # ------------ log.info("Writing snapshot bh-host table to file '{}'".format(fname)) beg = datetime.now() with h5py.File(fname, 'w') as host_table: # Metadata host_table.attrs[OFFTAB.RUN] = run host_table.attrs[OFFTAB.SNAP] = snap host_table.attrs[OFFTAB.VERSION] = _VERSION host_table.attrs[OFFTAB.CREATED] = datetime.now().ctime() host_table.attrs[OFFTAB.FILENAME] = fname # BH Data host_table.create_dataset(OFFTAB.BH_INDICES, data=bh_inds) host_table.create_dataset(OFFTAB.BH_IDS, data=bh_ids) host_table.create_dataset(OFFTAB.BH_HALOS, data=bh_halos) host_table.create_dataset(OFFTAB.BH_SUBHALOS, data=bh_subhs) log.after("Saved to '{}', size {}".format(fname, zio.get_file_size(fname)), beg, beg_all, lvl=log.WARNING) host_table = h5py.File(fname, 'r') return host_table
def _load_bh_hosts_table(run, log=None, load_saved=True, version=None): """Merge individual snapshot's blackhole hosts files into a single file. Arguments --------- run <int> : illustris simulation number {1, 3} load_saved <bool> : optional, load existing save if possible version <flt> : optional, target version number Returns ------- bh_hosts <dict> : table of hosts for all snapshots """ # log = check_log(log, run=run) log.debug("particle_hosts._load_bh_hosts_table()") beg_all = datetime.now() fname_bh_hosts = _get_filename_bh_hosts_table(run) _path = zio.check_path(fname_bh_hosts) if not os.path.isdir(_path): log.raise_error("Error with path for '{}' (path: '{}')".format( fname_bh_hosts, _path)) # Load Existing Save # ------------------ if load_saved: log.info("Loading from save '{}'".format(fname_bh_hosts)) # Make sure path exists if os.path.exists(fname_bh_hosts): hosts_table = h5py.File(fname_bh_hosts, 'r') else: log.warning("File '{}' does not Exist. Reconstructing.".format( fname_bh_hosts)) load_saved = False # Reconstruct Hosts Table # ----------------------- if not load_saved: log.info("Constructing Hosts Table") COSMO = Illustris_Cosmology_TOS() if version is not None: log.raise_error("Can only create version '{}'".format(_VERSION)) # Select the dict-keys for snapshot hosts to transfer host_keys = [ OFFTAB.BH_IDS, OFFTAB.BH_INDICES, OFFTAB.BH_HALOS, OFFTAB.BH_SUBHALOS ] # Save To HDF5 # ------------ log.info("Writing bh-host table to file '{}'".format(fname_bh_hosts)) beg = datetime.now() with h5py.File(fname_bh_hosts, 'w') as host_table: # Metadata host_table.attrs[OFFTAB.RUN] = run host_table.attrs[OFFTAB.VERSION] = _VERSION host_table.attrs[OFFTAB.CREATED] = datetime.now().ctime() host_table.attrs[OFFTAB.FILENAME] = fname_bh_hosts for snap in tqdm.trange(COSMO.NUM_SNAPS, desc="Loading snapshots"): # Load Snapshot BH-Hosts htab_snap = _load_bh_hosts_snap_table(run, snap, log, load_saved=True) # Extract and store target data snap_str = "{:03d}".format(snap) # Create a group for this snapshot snap_group = host_table.create_group(snap_str) # Transfer all parameters over for hkey in host_keys: snap_group.create_dataset(hkey, data=htab_snap[hkey][:]) log.after("Saved to '{}', size {}".format( fname_bh_hosts, zio.get_file_size(fname_bh_hosts)), beg, beg_all, lvl=log.WARNING) host_table = h5py.File(fname_bh_hosts, 'r') return hosts_table
def plotScattering(sample, snap, mbhb, log, plotNames): """Illustrate the scattering calculation for a single, sample system. Performs calculation by calling the 'scattering()' method, just like in "MBHBinaryEvolution.py". Arguments --------- sample : int Target galaxy/merger number to examine (this is the number out of *all* binaries, not just the valid ones [included in ``mbhb.evolution``]). snap : int Illustris snapshot number {1, 135}. mbhb : `Binaries.MBHBinaries` object log : `logging.Logger` object plotNames : str Base name of plots. Returns ------- plotNames : list of str Filenames of the plots created. """ log.debug("plotScattering()") PLOT_A10 = True # LC Occupancy PLOT_A11 = True # Flux PLOT_A15 = True # Flux vs. Separation PLOT_A02 = False # Model Galaxy PLOT_A08 = False # Dist Func from . import GM_Figures figNames = [] # Initialization # -------------- radialRange = np.array(mbhb.sets.RADIAL_RANGE_MODELS) * PC numSeps = mbhb.sets.PLOT_SCATTERING_SAMPLE_SEPS # Convert from `sample` number, of all binaries to index for valid (``mbhb.evolution``) ones val_inds = np.where(mbhb.valid)[0] valid_sample = np.where(val_inds == sample)[0] if valid_sample.size == 1: valid_sample = valid_sample[0] else: raise ValueError("`sample` '{}' returned '{}' from `val_inds`".format(sample, valid_sample)) mstar = mbhb.galaxies.mstar log.debug(" - Sample subhalo %d, Snapshot %d" % (sample, snap)) # Binary Properties (snapshot dependent) # `evolution` class includes only valid binaries, so use `valid_sample` # m1 = np.max(mbhb.evolution.masses[valid_sample, snap]) # m2 = np.min(mbhb.evolution.masses[valid_sample, snap]) m1 = np.max(mbhb.initMasses[sample]) m2 = np.min(mbhb.initMasses[sample]) # Galaxy properties (snapshot independent) # `galaxies` class includes *all* binaries, so use `sample` itself gals = mbhb.galaxies eps = gals.eps[sample] # functions of energy rads = gals.rads periods = gals.perOrb[sample] dist_func = gals.dist_func[sample] diffCoef = gals.diffCoef[sample] j2Circs = gals.j2Circ[sample] dnStarsAll = gals.dnStarsAll[sample] ndensStars = gals.densStars[sample]/mstar radHard = gals.rads_hard[sample] numStarsAll = sp.integrate.cumtrapz(dnStarsAll[::-1], eps[::-1], initial=0.0)[::-1] loss_cone = Loss_Cone_Explicit(mbhb.sets, log) # Wrap the Scattering function (for convenience) def evalScattering(binSep): # retvals = scattering(m1, m2, binSep, rads, eps, periods, j2Circs, diffCoef, dist_func, # [sample], radHard, mbhb.sets) retvals = loss_cone.harden( m1, m2, binSep, rads, eps, periods, j2Circs, diffCoef, dist_func, [sample], radHard, mbhb.sets) radLC, j2LC, enrLC, dnStarsFLC, numStarsFLC, numStarsSSLC, \ dfStarsFLC, dfStarsSSLC, fluxStarsFLC, fluxStarsSSLC, flux, dadt_lc = retvals return dnStarsFLC, numStarsFLC, dfStarsFLC, fluxStarsFLC, dfStarsSSLC, fluxStarsSSLC num_flc = np.zeros(numSeps) flx_flc = np.zeros(numSeps) flx_sslc = np.zeros(numSeps) dadt_sslc = np.zeros(numSeps) dadt_flc = np.zeros(numSeps) subPlotNames = zio.modify_filename(plotNames, prepend='stellar_scattering/') zio.check_path(subPlotNames) # Iterate over range of binary separations, plot profiles for each log.debug(" - Calculting scattering for %d binary separations" % (numSeps)) flx_seps = zmath.spacing(radialRange, scale='log', num=numSeps) for ii, binSep in enumerate(tqdm.tqdm(flx_seps, desc="Calculating scattering")): dnStarsFLC, numStarsFLC, dfStarsFLC, fluxStarsFLC, dfStarsSSLC, fluxStarsSSLC = \ evalScattering(binSep) hard_sslc = dadt_scattering(m1+m2, binSep, np.max(fluxStarsSSLC), mstar) hard_flc = dadt_scattering(m1+m2, binSep, np.max(fluxStarsFLC), mstar) eps_rad = zmath.spline(gals.rads, gals.eps[sample], log=True, pos=True, extrap=True) sepEner = eps_rad(binSep) # Plot Fig A10 - Loss-Cone Occupancy if PLOT_A10: fig = GM_Figures.figa10_lc_occupancy( gals.rads, eps, dnStarsAll, dnStarsFLC, numStarsAll, numStarsFLC, binr=binSep, bine=sepEner) fname1 = subPlotNames + "lcPipe_figa10-occ_%02d.png" % (ii) zplot.saveFigure(fig, fname1, verbose=False) # , log=log) plt.close(fig) # Plot Fig A11 - loss-cone fluxes if PLOT_A11: fig = GM_Figures.figa11_lc_flux( eps, gals.rads, dfStarsFLC, dfStarsSSLC, fluxStarsFLC, fluxStarsSSLC, binr=binSep, bine=sepEner) fname2 = subPlotNames + "lcPipe_figa11-flx_%02d.png" % (ii) zplot.saveFigure(fig, fname2, verbose=False) # , log=log) plt.close(fig) # Store values to arrays num_flc[ii] = np.max(numStarsFLC) flx_flc[ii] = np.max(fluxStarsFLC) flx_sslc[ii] = np.max(fluxStarsSSLC) dadt_sslc[ii] = hard_sslc dadt_flc[ii] = hard_flc # pbar.update(ii) # pbar.finish() if PLOT_A10: log.debug(" - - Saved FigA10 to (e.g.) '%s'" % (fname1)) if PLOT_A11: log.debug(" - - Saved FigA11 to (e.g.) '%s'" % (fname2)) # Plot Figs A15 - Scattering vs. Separations if PLOT_A15: log.debug(" - Plotting Fig15") fig = GM_Figures.figA15_flux_sep(flx_seps, flx_flc, flx_sslc, dadt_flc, dadt_sslc, sample, snap) fname = plotNames + "lcPipe_figa15-sep.png" zplot.saveFigure(fig, fname, verbose=False, log=log) plt.close(fig) figNames.append(fname) # Plot Figure A02 - Density/Mass Profiles if PLOT_A02: log.info(" - Plotting FigA02") dens = [gals.densStars[sample], gals.densDM[sample]] mass = [gals.massStars[sample], gals.massDM[sample], gals.massTot[sample]] names = ['Stars', 'DM', 'Total'] vels = [gals.vdisp_stars[sample]] fig = GM_Figures.figA02_ModelGalaxy(gals.rads, dens, mass, vels, names=names) fname = plotNames + "lcPipe_figa02-dens.png" zplot.saveFigure(fig, fname, verbose=False, log=log) plt.close(fig) figNames.append(fname) # Plot Figure A08 - Reconstructed Number Density if PLOT_A08: log.debug(" - Plotting FigA08") df_pot = zmath.spline(eps, dist_func, mono=True, pos=True, sort=True, log=True) dfErrs = np.zeros(np.size(eps)) fig = GM_Figures.figA08_distFunc(eps, gals.rads, ndensStars, dist_func, dfErrs, df_pot, log, sample=sample) fname = plotNames + "lcPipe_figa08-distfunc.png" zplot.saveFigure(fig, fname, verbose=False, log=log) plt.close(fig) figNames.append(fname) return figNames