def run_calculation_loop(self): parallel_tasks.database.synchronize_creator_object() pair_list = self._generate_timestep_pairs() if len(pair_list) == 0: logger.error("No timesteps found to link") return pair_list = parallel_tasks.distributed(pair_list) object_type = core.halo.Halo.object_typecode_from_tag(self.args.type_) for s_x, s in pair_list: logger.info("Linking %r and %r", s_x, s) if self.args.force or self.need_crosslink_ts(s_x, s, object_type): self.crosslink_ts(s_x, s, 0, self.args.hmax, self.args.dmonly, object_typecode=object_type)
def timelink_bh(sims, session): query = db.sim_query_from_name_list(sims, session) for sim in query.all(): pairs = list(zip(sim.timesteps[:-1], sim.timesteps[1:])) fname = glob.glob(db.config.base + "/" + sim.basename + "/*.BHmergers") if len(fname) == 0: logger.error("No merger file for " + sim.basename) return elif len(fname) > 1: logger.error("Can't work out which is the merger file for " + sim.basename) logger.error("Found: %s", fname) return with session.no_autoflush: generate_halolinks(session, fname, pairs)
def crosslink_ts(self, ts1, ts2, halo_min=0, halo_max=None, dmonly=False, threshold=config.default_linking_threshold, object_typecode=0): """Link the halos of two timesteps together :type ts1 tangos.core.TimeStep :type ts2 tangos.core.TimeStep""" logger.info("Gathering halo information for %r and %r", ts1, ts2) halos1 = self.make_finder_id_to_halo_map(ts1, object_typecode) halos2 = self.make_finder_id_to_halo_map(ts2, object_typecode) with parallel_tasks.ExclusiveLock("create_db_objects_from_catalog"): same_d_id = core.dictionary.get_or_create_dictionary_item( self.session, "ptcls_in_common") self.session.commit() output_handler_1 = ts1.simulation.get_output_handler() output_handler_2 = ts2.simulation.get_output_handler() if type(output_handler_1).match_objects != type( output_handler_2).match_objects: logger.error( "Timesteps %r and %r cannot be crosslinked; they are using incompatible file readers", ts1, ts2) return # keep the files alive throughout (so they are not garbage-collected after the first match_objects): snap1 = ts1.load() snap2 = ts2.load() try: cat = output_handler_1.match_objects( ts1.extension, ts2.extension, halo_min, halo_max, dmonly, threshold, core.halo.Halo.object_typetag_from_code(object_typecode), output_handler_for_ts2=output_handler_2) back_cat = output_handler_2.match_objects( ts2.extension, ts1.extension, halo_min, halo_max, dmonly, threshold, core.halo.Halo.object_typetag_from_code(object_typecode), output_handler_for_ts2=output_handler_1) except Exception as e: if isinstance(e, KeyboardInterrupt): raise logger.exception( "Exception during attempt to crosslink timesteps %r and %r", ts1, ts2) return with self.session.no_autoflush: logger.info("Gathering links for %r and %r", ts1, ts2) items = self.create_db_objects_from_catalog( cat, halos1, halos2, same_d_id) logger.info("Identified %d links between %r and %r", len(items), ts1, ts2) items_back = self.create_db_objects_from_catalog( back_cat, halos2, halos1, same_d_id) logger.info("Identified %d links between %r and %r", len(items_back), ts2, ts1) with parallel_tasks.ExclusiveLock("create_db_objects_from_catalog"): logger.info("Preparing to commit links for %r and %r", ts1, ts2) self.session.add_all(items) self.session.add_all(items_back) self.session.commit() logger.info("Finished committing total of %d links for %r and %r", len(items) + len(items_back), ts1, ts2)
def generate_halolinks(session, fname, pairs): for ts1, ts2 in parallel_tasks.distributed(pairs): bh_log = None if BlackHolesLog.can_load(ts2.filename): bh_log = BlackHolesLog(ts2.filename) elif ShortenedOrbitLog.can_load(ts2.filename): bh_log = ShortenedOrbitLog(ts2.filename) if bh_log is None: logger.error("Warning! No orbit file found!") links = [] mergers_links = [] bh_map = {} logger.info("Gathering BH tracking information for steps %r and %r", ts1, ts2) with parallel_tasks.ExclusiveLock("bh"): dict_obj = db.core.get_or_create_dictionary_item( session, "tracker") dict_obj_next = db.core.get_or_create_dictionary_item( session, "BH_merger_next") dict_obj_prev = db.core.get_or_create_dictionary_item( session, "BH_merger_prev") track_links_n, idf_n, idt_n = db.tracking.get_tracker_links( session, dict_obj_next) bh_objects_1, nums1, id1 = get_bh_objs_numbers_and_dbids(ts1) bh_objects_2, nums2, id2 = get_bh_objs_numbers_and_dbids(ts2) tracker_links, idf, idt = db.tracking.get_tracker_links( session, dict_obj) idf_n = np.array(idf_n) idt_n = np.array(idt_n) if len(nums1) == 0 or len(nums2) == 0: logger.info("No BHs found in either step %r or %r... moving on", ts1, ts2) continue logger.info("Generating BH tracker links between steps %r and %r", ts1, ts2) o1 = np.where(np.in1d(nums1, nums2))[0] o2 = np.where(np.in1d(nums2, nums1))[0] if len(o1) == 0 or len(o2) == 0: continue with session.no_autoflush: for ii, jj in zip(o1, o2): if nums1[ii] != nums2[jj]: raise RuntimeError("BH iords are mismatched") exists = np.where((idf == id1[ii]) & (idt == id2[jj]))[0] if len(exists) == 0: links.append( tangos.core.halo_data.HaloLink(bh_objects_1[ii], bh_objects_2[jj], dict_obj, 1.0)) links.append( tangos.core.halo_data.HaloLink(bh_objects_2[jj], bh_objects_1[ii], dict_obj, 1.0)) logger.info("Generated %d tracker links between steps %r and %r", len(links), ts1, ts2) logger.info("Generating BH Merger information for steps %r and %r", ts1, ts2) for l in open(fname[0]): l_split = l.split() t = float(l_split[6]) bh_dest_id = int(l_split[0]) bh_src_id = int(l_split[1]) ratio = float(l_split[4]) # ratios in merger file are ambiguous (since major progenitor may be "source" rather than "destination") # re-establish using the log file: try: ratio = bh_log.determine_merger_ratio(bh_src_id, bh_dest_id) except (ValueError, AttributeError) as e: logger.debug( "Could not calculate merger ratio for %d->%d from the BH log; assuming the .BHmergers-asserted value is accurate", bh_src_id, bh_dest_id) if t > ts1.time_gyr and t <= ts2.time_gyr: bh_map[bh_src_id] = (bh_dest_id, ratio) resolve_multiple_mergers(bh_map) logger.info("Gathering BH merger links for steps %r and %r", ts1, ts2) with session.no_autoflush: for src, (dest, ratio) in bh_map.items(): if src not in nums1 or dest not in nums2: logger.warning( "Can't link BH %r -> %r; missing BH objects in database", src, dest) continue bh_src_before = bh_objects_1[nums1.index(src)] bh_dest_after = bh_objects_2[nums2.index(dest)] if ((idf_n == bh_src_before.id) & (idt_n == bh_dest_after.id)).sum() == 0: mergers_links.append( tangos.core.halo_data.HaloLink(bh_src_before, bh_dest_after, dict_obj_next, 1.0)) mergers_links.append( tangos.core.halo_data.HaloLink(bh_dest_after, bh_src_before, dict_obj_prev, ratio)) logger.info("Generated %d BH merger links for steps %r and %r", len(mergers_links), ts1, ts2) with parallel_tasks.ExclusiveLock("bh"): logger.info("Committing total %d BH links for steps %r and %r", len(mergers_links) + len(links), ts1, ts2) session.add_all(links) session.add_all(mergers_links) session.commit() logger.info("Finished committing BH links for steps %r and %r", ts1, ts2)