def run_calculation_loop(self):
parallel_tasks.database.synchronize_creator_object()
pair_list = self._generate_timestep_pairs()
if len(pair_list) == 0:
logger.error("No timesteps found to link")
return
pair_list = parallel_tasks.distributed(pair_list)
object_type = core.halo.Halo.object_typecode_from_tag(self.args.type_)
for s_x, s in pair_list:
logger.info("Linking %r and %r", s_x, s)
if self.args.force or self.need_crosslink_ts(s_x, s, object_type):
self.crosslink_ts(s_x,
s,
0,
self.args.hmax,
self.args.dmonly,
object_typecode=object_type)
def timelink_bh(sims, session):
query = db.sim_query_from_name_list(sims, session)
for sim in query.all():
pairs = list(zip(sim.timesteps[:-1], sim.timesteps[1:]))
fname = glob.glob(db.config.base + "/" + sim.basename + "/*.BHmergers")
if len(fname) == 0:
logger.error("No merger file for " + sim.basename)
return
elif len(fname) > 1:
logger.error("Can't work out which is the merger file for " +
sim.basename)
logger.error("Found: %s", fname)
return
with session.no_autoflush:
generate_halolinks(session, fname, pairs)
def crosslink_ts(self,
ts1,
ts2,
halo_min=0,
halo_max=None,
dmonly=False,
threshold=config.default_linking_threshold,
object_typecode=0):
"""Link the halos of two timesteps together
:type ts1 tangos.core.TimeStep
:type ts2 tangos.core.TimeStep"""
logger.info("Gathering halo information for %r and %r", ts1, ts2)
halos1 = self.make_finder_id_to_halo_map(ts1, object_typecode)
halos2 = self.make_finder_id_to_halo_map(ts2, object_typecode)
with parallel_tasks.ExclusiveLock("create_db_objects_from_catalog"):
same_d_id = core.dictionary.get_or_create_dictionary_item(
self.session, "ptcls_in_common")
self.session.commit()
output_handler_1 = ts1.simulation.get_output_handler()
output_handler_2 = ts2.simulation.get_output_handler()
if type(output_handler_1).match_objects != type(
output_handler_2).match_objects:
logger.error(
"Timesteps %r and %r cannot be crosslinked; they are using incompatible file readers",
ts1, ts2)
return
# keep the files alive throughout (so they are not garbage-collected after the first match_objects):
snap1 = ts1.load()
snap2 = ts2.load()
try:
cat = output_handler_1.match_objects(
ts1.extension,
ts2.extension,
halo_min,
halo_max,
dmonly,
threshold,
core.halo.Halo.object_typetag_from_code(object_typecode),
output_handler_for_ts2=output_handler_2)
back_cat = output_handler_2.match_objects(
ts2.extension,
ts1.extension,
halo_min,
halo_max,
dmonly,
threshold,
core.halo.Halo.object_typetag_from_code(object_typecode),
output_handler_for_ts2=output_handler_1)
except Exception as e:
if isinstance(e, KeyboardInterrupt):
raise
logger.exception(
"Exception during attempt to crosslink timesteps %r and %r",
ts1, ts2)
return
with self.session.no_autoflush:
logger.info("Gathering links for %r and %r", ts1, ts2)
items = self.create_db_objects_from_catalog(
cat, halos1, halos2, same_d_id)
logger.info("Identified %d links between %r and %r", len(items),
ts1, ts2)
items_back = self.create_db_objects_from_catalog(
back_cat, halos2, halos1, same_d_id)
logger.info("Identified %d links between %r and %r",
len(items_back), ts2, ts1)
with parallel_tasks.ExclusiveLock("create_db_objects_from_catalog"):
logger.info("Preparing to commit links for %r and %r", ts1, ts2)
self.session.add_all(items)
self.session.add_all(items_back)
self.session.commit()
logger.info("Finished committing total of %d links for %r and %r",
len(items) + len(items_back), ts1, ts2)
def generate_halolinks(session, fname, pairs):
for ts1, ts2 in parallel_tasks.distributed(pairs):
bh_log = None
if BlackHolesLog.can_load(ts2.filename):
bh_log = BlackHolesLog(ts2.filename)
elif ShortenedOrbitLog.can_load(ts2.filename):
bh_log = ShortenedOrbitLog(ts2.filename)
if bh_log is None:
logger.error("Warning! No orbit file found!")
links = []
mergers_links = []
bh_map = {}
logger.info("Gathering BH tracking information for steps %r and %r",
ts1, ts2)
with parallel_tasks.ExclusiveLock("bh"):
dict_obj = db.core.get_or_create_dictionary_item(
session, "tracker")
dict_obj_next = db.core.get_or_create_dictionary_item(
session, "BH_merger_next")
dict_obj_prev = db.core.get_or_create_dictionary_item(
session, "BH_merger_prev")
track_links_n, idf_n, idt_n = db.tracking.get_tracker_links(
session, dict_obj_next)
bh_objects_1, nums1, id1 = get_bh_objs_numbers_and_dbids(ts1)
bh_objects_2, nums2, id2 = get_bh_objs_numbers_and_dbids(ts2)
tracker_links, idf, idt = db.tracking.get_tracker_links(
session, dict_obj)
idf_n = np.array(idf_n)
idt_n = np.array(idt_n)
if len(nums1) == 0 or len(nums2) == 0:
logger.info("No BHs found in either step %r or %r... moving on",
ts1, ts2)
continue
logger.info("Generating BH tracker links between steps %r and %r", ts1,
ts2)
o1 = np.where(np.in1d(nums1, nums2))[0]
o2 = np.where(np.in1d(nums2, nums1))[0]
if len(o1) == 0 or len(o2) == 0:
continue
with session.no_autoflush:
for ii, jj in zip(o1, o2):
if nums1[ii] != nums2[jj]:
raise RuntimeError("BH iords are mismatched")
exists = np.where((idf == id1[ii]) & (idt == id2[jj]))[0]
if len(exists) == 0:
links.append(
tangos.core.halo_data.HaloLink(bh_objects_1[ii],
bh_objects_2[jj],
dict_obj, 1.0))
links.append(
tangos.core.halo_data.HaloLink(bh_objects_2[jj],
bh_objects_1[ii],
dict_obj, 1.0))
logger.info("Generated %d tracker links between steps %r and %r",
len(links), ts1, ts2)
logger.info("Generating BH Merger information for steps %r and %r",
ts1, ts2)
for l in open(fname[0]):
l_split = l.split()
t = float(l_split[6])
bh_dest_id = int(l_split[0])
bh_src_id = int(l_split[1])
ratio = float(l_split[4])
# ratios in merger file are ambiguous (since major progenitor may be "source" rather than "destination")
# re-establish using the log file:
try:
ratio = bh_log.determine_merger_ratio(bh_src_id, bh_dest_id)
except (ValueError, AttributeError) as e:
logger.debug(
"Could not calculate merger ratio for %d->%d from the BH log; assuming the .BHmergers-asserted value is accurate",
bh_src_id, bh_dest_id)
if t > ts1.time_gyr and t <= ts2.time_gyr:
bh_map[bh_src_id] = (bh_dest_id, ratio)
resolve_multiple_mergers(bh_map)
logger.info("Gathering BH merger links for steps %r and %r", ts1, ts2)
with session.no_autoflush:
for src, (dest, ratio) in bh_map.items():
if src not in nums1 or dest not in nums2:
logger.warning(
"Can't link BH %r -> %r; missing BH objects in database",
src, dest)
continue
bh_src_before = bh_objects_1[nums1.index(src)]
bh_dest_after = bh_objects_2[nums2.index(dest)]
if ((idf_n == bh_src_before.id) &
(idt_n == bh_dest_after.id)).sum() == 0:
mergers_links.append(
tangos.core.halo_data.HaloLink(bh_src_before,
bh_dest_after,
dict_obj_next, 1.0))
mergers_links.append(
tangos.core.halo_data.HaloLink(bh_dest_after,
bh_src_before,
dict_obj_prev, ratio))
logger.info("Generated %d BH merger links for steps %r and %r",
len(mergers_links), ts1, ts2)
with parallel_tasks.ExclusiveLock("bh"):
logger.info("Committing total %d BH links for steps %r and %r",
len(mergers_links) + len(links), ts1, ts2)
session.add_all(links)
session.add_all(mergers_links)
session.commit()
logger.info("Finished committing BH links for steps %r and %r",
ts1, ts2)