def _generate_timestep_pairs_from_sims(self, sim1, sim2):
assert sim1 != sim2, "Can't link simulation to itself"
logger.info("Match timesteps of %r to %r", sim1, sim2)
ts1s = sim1.timesteps
ts2s = sim2.timesteps
pairs = []
for ts1 in ts1s:
ts2 = self._get_best_timestep_matching(ts2s, ts1)
pairing_is_mutual = (self._get_best_timestep_matching(ts1s,
ts2) == ts1)
if pairing_is_mutual:
logger.info("Pairing timesteps: %r and %r", ts1, ts2)
pairs += [(ts1, ts2)]
else:
logger.warning("No pairing found for timestep %r", ts1)
return pairs
def create_db_objects_from_catalog(self, cat, finder_offset_to_halos_1,
finder_offset_to_halos_2, same_d_id):
items = []
missing_db_object = 0
for i, possibilities in enumerate(cat):
h1 = finder_offset_to_halos_1.get(i, None)
for cat_i, weight in possibilities:
h2 = finder_offset_to_halos_2.get(cat_i, None)
if h1 is not None and h2 is not None:
items.append(
core.halo_data.HaloLink(h1, h2, same_d_id, weight))
else:
missing_db_object += 1
if missing_db_object > 0:
logger.warning(
"%d link(s) could not be identified because the halo objects do not exist in the DB",
missing_db_object)
return items
def need_crosslink_ts(self, ts1, ts2, object_typecode=0):
num_sources = ts1.halos.count()
num_targets = ts2.halos.count()
if num_targets == 0:
logger.warning("Will not link: no halos in target timestep %r",
ts2)
return False
if num_sources == 0:
logger.warning("Will not link: no halos in source timestep %r",
ts1)
return False
halo_source = sqlalchemy.orm.aliased(core.halo.SimulationObjectBase,
name="halo_source")
halo_target = sqlalchemy.orm.aliased(core.halo.SimulationObjectBase,
name="halo_target")
same_d_id = self._get_linkname_dictionaryitem().id
exists = self.session.query(core.halo_data.HaloLink).join(halo_source, core.halo_data.HaloLink.halo_from). \
join(halo_target, core.halo_data.HaloLink.halo_to). \
filter(halo_source.timestep_id == ts1.id, halo_target.timestep_id == ts2.id,
halo_source.object_typecode == object_typecode,
halo_target.object_typecode == object_typecode,
core.halo_data.HaloLink.relation_id == same_d_id).count() > 0
self.session.commit()
if exists:
logger.warning(
"Will not link: links already exist between %r and %r", ts1,
ts2)
return False
return True
def scan_for_BHs(files, session):
for timestep in parallel_tasks.distributed(files):
logger.info("Processing %s", timestep)
try:
timestep_particle_data = timestep.load()
except:
logger.warning("File not found - continuing")
continue
if len(timestep_particle_data.star) < 1:
logger.warning("No stars - continuing")
continue
timestep_particle_data.physical_units()
logger.info(
"Gathering existing BH halo information from database for step %r",
timestep)
bhobjs = timestep.bhs.all()
existing_bh_nums = [x.halo_number for x in bhobjs]
logger.info("...found %d existing BHs", len(existing_bh_nums))
logger.info("Gathering BH info from simulation for step %r", timestep)
bh_iord_this_timestep = timestep_particle_data.star['iord'][np.where(
timestep_particle_data.star['tform'] < 0)[0]]
bh_mass_this_timestep = timestep_particle_data.star['mass'][np.where(
timestep_particle_data.star['tform'] < 0)[0]]
logger.info("Found %d black holes for %r", len(bh_iord_this_timestep),
timestep)
logger.info(
"Updating BH trackdata and BH objects using on-disk information from %r",
timestep)
add_missing_trackdata_and_BH_objects(timestep, bh_iord_this_timestep,
existing_bh_nums, session)
session.expire_all()
logger.info("Calculating halo associations for BHs in timestep %r",
timestep)
bh_cen_halos, bh_halos = bh_halo_assign(timestep_particle_data)
# re-order our information so that links refer to BHs in descending order of mass
bh_order_by_mass = np.argsort(bh_mass_this_timestep)[::-1]
bh_iord_this_timestep = bh_iord_this_timestep[bh_order_by_mass]
if bh_halos is not None:
bh_halos = bh_halos[bh_order_by_mass]
if bh_cen_halos is not None:
bh_cen_halos = bh_cen_halos[bh_order_by_mass]
logger.info("Freeing the timestep particle data")
with check_deleted(timestep_particle_data):
del (timestep_particle_data)
if bh_halos is not None:
assign_bh_to_halos(bh_halos, bh_iord_this_timestep, timestep, "BH")
if bh_cen_halos is not None:
assign_bh_to_halos(bh_cen_halos, bh_iord_this_timestep, timestep,
"BH_central", "host_halo")
def generate_halolinks(session, fname, pairs):
for ts1, ts2 in parallel_tasks.distributed(pairs):
bh_log = None
if BlackHolesLog.can_load(ts2.filename):
bh_log = BlackHolesLog(ts2.filename)
elif ShortenedOrbitLog.can_load(ts2.filename):
bh_log = ShortenedOrbitLog(ts2.filename)
if bh_log is None:
logger.error("Warning! No orbit file found!")
links = []
mergers_links = []
bh_map = {}
logger.info("Gathering BH tracking information for steps %r and %r",
ts1, ts2)
with parallel_tasks.ExclusiveLock("bh"):
dict_obj = db.core.get_or_create_dictionary_item(
session, "tracker")
dict_obj_next = db.core.get_or_create_dictionary_item(
session, "BH_merger_next")
dict_obj_prev = db.core.get_or_create_dictionary_item(
session, "BH_merger_prev")
track_links_n, idf_n, idt_n = db.tracking.get_tracker_links(
session, dict_obj_next)
bh_objects_1, nums1, id1 = get_bh_objs_numbers_and_dbids(ts1)
bh_objects_2, nums2, id2 = get_bh_objs_numbers_and_dbids(ts2)
tracker_links, idf, idt = db.tracking.get_tracker_links(
session, dict_obj)
idf_n = np.array(idf_n)
idt_n = np.array(idt_n)
if len(nums1) == 0 or len(nums2) == 0:
logger.info("No BHs found in either step %r or %r... moving on",
ts1, ts2)
continue
logger.info("Generating BH tracker links between steps %r and %r", ts1,
ts2)
o1 = np.where(np.in1d(nums1, nums2))[0]
o2 = np.where(np.in1d(nums2, nums1))[0]
if len(o1) == 0 or len(o2) == 0:
continue
with session.no_autoflush:
for ii, jj in zip(o1, o2):
if nums1[ii] != nums2[jj]:
raise RuntimeError("BH iords are mismatched")
exists = np.where((idf == id1[ii]) & (idt == id2[jj]))[0]
if len(exists) == 0:
links.append(
tangos.core.halo_data.HaloLink(bh_objects_1[ii],
bh_objects_2[jj],
dict_obj, 1.0))
links.append(
tangos.core.halo_data.HaloLink(bh_objects_2[jj],
bh_objects_1[ii],
dict_obj, 1.0))
logger.info("Generated %d tracker links between steps %r and %r",
len(links), ts1, ts2)
logger.info("Generating BH Merger information for steps %r and %r",
ts1, ts2)
for l in open(fname[0]):
l_split = l.split()
t = float(l_split[6])
bh_dest_id = int(l_split[0])
bh_src_id = int(l_split[1])
ratio = float(l_split[4])
# ratios in merger file are ambiguous (since major progenitor may be "source" rather than "destination")
# re-establish using the log file:
try:
ratio = bh_log.determine_merger_ratio(bh_src_id, bh_dest_id)
except (ValueError, AttributeError) as e:
logger.debug(
"Could not calculate merger ratio for %d->%d from the BH log; assuming the .BHmergers-asserted value is accurate",
bh_src_id, bh_dest_id)
if t > ts1.time_gyr and t <= ts2.time_gyr:
bh_map[bh_src_id] = (bh_dest_id, ratio)
resolve_multiple_mergers(bh_map)
logger.info("Gathering BH merger links for steps %r and %r", ts1, ts2)
with session.no_autoflush:
for src, (dest, ratio) in bh_map.items():
if src not in nums1 or dest not in nums2:
logger.warning(
"Can't link BH %r -> %r; missing BH objects in database",
src, dest)
continue
bh_src_before = bh_objects_1[nums1.index(src)]
bh_dest_after = bh_objects_2[nums2.index(dest)]
if ((idf_n == bh_src_before.id) &
(idt_n == bh_dest_after.id)).sum() == 0:
mergers_links.append(
tangos.core.halo_data.HaloLink(bh_src_before,
bh_dest_after,
dict_obj_next, 1.0))
mergers_links.append(
tangos.core.halo_data.HaloLink(bh_dest_after,
bh_src_before,
dict_obj_prev, ratio))
logger.info("Generated %d BH merger links for steps %r and %r",
len(mergers_links), ts1, ts2)
with parallel_tasks.ExclusiveLock("bh"):
logger.info("Committing total %d BH links for steps %r and %r",
len(mergers_links) + len(links), ts1, ts2)
session.add_all(links)
session.add_all(mergers_links)
session.commit()
logger.info("Finished committing BH links for steps %r and %r",
ts1, ts2)
def assign_bh_to_halos(bh_halo_assignment,
bh_iord,
timestep,
linkname,
hostname=None):
session = Session.object_session(timestep)
linkname_dict_id = tangos.core.dictionary.get_or_create_dictionary_item(
session, linkname)
if hostname is not None:
host_dict_id = tangos.core.dictionary.get_or_create_dictionary_item(
session, hostname)
else:
host_dict_id = None
logger.info("Gathering %s links for step %r", linkname, timestep)
links, link_id_from, link_id_to = db.tracking.get_tracker_links(
session, linkname_dict_id)
halos = timestep.halos.filter_by(object_typecode=0).all()
halo_nums = [h.halo_number for h in halos]
halo_catind = [h.finder_offset for h in halos]
halo_ids = np.array([h.id for h in halos])
logger.info("Gathering bh halo information for %r", timestep)
with parallel_tasks.lock.SharedLock("bh"):
bh_database_object, existing_bh_nums, bhobj_ids = get_bh_objs_numbers_and_dbids(
timestep)
bh_links = []
with session.no_autoflush:
for bhi, haloi in zip(bh_iord, bh_halo_assignment):
haloi = int(haloi)
bhi = int(bhi)
if haloi not in halo_catind:
logger.warning(
"Skipping BH in halo %d as no corresponding halo found in the database",
haloi)
continue
if bhi not in existing_bh_nums:
logger.warning("Can't find the database object for BH %d", bhi)
print(bhi)
print(existing_bh_nums)
continue
bh_index_in_list = existing_bh_nums.index(bhi)
halo_index_in_list = halo_catind.index(haloi)
bh_obj = bh_database_object[bh_index_in_list]
halo_obj = halos[halo_index_in_list]
num_existing_links = (
(link_id_from == halo_ids[halo_index_in_list]) &
(link_id_to == bhobj_ids[bh_index_in_list])).sum()
if num_existing_links == 0:
bh_links.append(
tangos.core.halo_data.HaloLink(halo_obj, bh_obj,
linkname_dict_id))
if host_dict_id is not None:
bh_links.append(
tangos.core.halo_data.HaloLink(bh_obj, halo_obj,
host_dict_id))
logger.info("Committing %d %s links for step %r...", len(bh_links),
linkname, timestep)
with parallel_tasks.ExclusiveLock("bh"):
session.add_all(bh_links)
session.commit()
logger.info("...done")