def test_comparison_operators(self): self.assertEqual(Box(LonLat.fromDegrees(45, 45)), LonLat.fromDegrees(45, 45)) self.assertEqual(Box.fromDegrees(90, -45, 180, 45), Box(NormalizedAngleInterval.fromDegrees(90, 180), AngleInterval.fromDegrees(-45, 45))) self.assertNotEqual(Box(LonLat.fromDegrees(45, 45)), LonLat.fromDegrees(45, 90)) self.assertNotEqual(Box.fromDegrees(90, -45, 180, 45), Box.fromDegrees(90, -45, 180, 90))
def testConstruction(self): a1 = NormalizedAngle(1.0) a2 = NormalizedAngle.fromRadians(1.0) a3 = NormalizedAngle.fromDegrees(57.29577951308232) self.assertEqual(a1, a2) self.assertEqual(a1.asRadians(), 1.0) self.assertEqual(a1, a3) self.assertEqual(a1.asDegrees(), 57.29577951308232) self.assertEqual(NormalizedAngle.between(NormalizedAngle(0), NormalizedAngle(1)), NormalizedAngle(1)) a = NormalizedAngle.center(NormalizedAngle(0), NormalizedAngle(1)) self.assertAlmostEqual(a.asRadians(), 0.5, places=15) a = NormalizedAngle(LonLat.fromDegrees(45, 0), LonLat.fromDegrees(90, 0)) self.assertAlmostEqual(a.asDegrees(), 45.0, places=13) a = NormalizedAngle(UnitVector3d.Y(), UnitVector3d.Z()) self.assertAlmostEqual(a.asDegrees(), 90.0, places=13)
def test_center_and_dimensions(self): b = Box.fromDegrees(-90, -45, 90, 45) self.assertEqual(b.getCenter(), LonLat.fromDegrees(0, 0)) self.assertEqual(b.getWidth(), Angle.fromDegrees(180)) self.assertEqual(b.getHeight(), Angle.fromDegrees(90)) self.assertEqual(b.getLon().getA(), NormalizedAngle.fromDegrees(-90)) self.assertEqual(b.getLat().getB(), Angle.fromDegrees(45))
def testConstruction(self): a1 = NormalizedAngle(1.0) a2 = NormalizedAngle.fromRadians(1.0) a3 = NormalizedAngle.fromDegrees(57.29577951308232) self.assertEqual(a1, a2) self.assertEqual(a1.asRadians(), 1.0) self.assertEqual(a1, a3) self.assertEqual(a1.asDegrees(), 57.29577951308232) self.assertEqual( NormalizedAngle.between(NormalizedAngle(0), NormalizedAngle(1)), NormalizedAngle(1)) a = NormalizedAngle.center(NormalizedAngle(0), NormalizedAngle(1)) self.assertAlmostEqual(a.asRadians(), 0.5, places=15) a = NormalizedAngle(LonLat.fromDegrees(45, 0), LonLat.fromDegrees(90, 0)) self.assertAlmostEqual(a.asDegrees(), 45.0, places=13) a = NormalizedAngle(UnitVector3d.Y(), UnitVector3d.Z()) self.assertAlmostEqual(a.asDegrees(), 90.0, places=13)
def testConstruction(self): v = Vector3d(1, 1, 1) u = UnitVector3d.orthogonalTo(v) self.assertAlmostEqual(u.dot(v), 0.0, places=15) a = UnitVector3d(1, 1, 1) self.assertEqual(a, UnitVector3d(Vector3d(1, 1, 1))) self.assertAlmostEqual(a.x(), math.sqrt(3.0) / 3.0, places=15) self.assertAlmostEqual(a.y(), math.sqrt(3.0) / 3.0, places=15) self.assertAlmostEqual(a.z(), math.sqrt(3.0) / 3.0, places=15) b = UnitVector3d(Angle.fromDegrees(45), Angle.fromDegrees(45)) self.assertEqual(b, UnitVector3d(LonLat.fromDegrees(45, 45))) self.assertAlmostEqual(b.x(), 0.5, places=15) self.assertAlmostEqual(b.y(), 0.5, places=15) self.assertAlmostEqual(b.z(), 0.5 * math.sqrt(2.0), places=15) c = UnitVector3d.northFrom(b) d = UnitVector3d(LonLat.fromDegrees(225, 45)) self.assertAlmostEqual(c.x(), d.x(), places=15) self.assertAlmostEqual(c.y(), d.y(), places=15) self.assertAlmostEqual(c.z(), d.z(), places=15)
def testConstruction(self): v = Vector3d(1, 1, 1) u = UnitVector3d.orthogonalTo(v) self.assertAlmostEqual(u.dot(v), 0.0, places=15) u = UnitVector3d(1, 1, 1) self.assertEqual(u, UnitVector3d(Vector3d(1, 1, 1))) self.assertAlmostEqual(u.x(), math.sqrt(3.0) / 3.0, places=15) self.assertAlmostEqual(u.y(), math.sqrt(3.0) / 3.0, places=15) self.assertAlmostEqual(u.z(), math.sqrt(3.0) / 3.0, places=15) u = UnitVector3d(Angle.fromDegrees(45), Angle.fromDegrees(45)) self.assertEqual(u, UnitVector3d(LonLat.fromDegrees(45, 45))) self.assertAlmostEqual(u.x(), 0.5, places=15) self.assertAlmostEqual(u.y(), 0.5, places=15) self.assertAlmostEqual(u.z(), 0.5 * math.sqrt(2.0), places=15) u = UnitVector3d.northFrom(u.asVector3d()) w = UnitVector3d(LonLat.fromDegrees(225, 45)) self.assertAlmostEqual(u.x(), w.x(), places=15) self.assertAlmostEqual(u.y(), w.y(), places=15) self.assertAlmostEqual(u.z(), w.z(), places=15)
def test_relationships(self): b1 = Box.fromDegrees(90, 0, 180, 45) p = LonLat.fromDegrees(135, 10) self.assertTrue(p in b1) self.assertTrue(b1.contains(p)) b2 = Box.fromDegrees(135, 15, 135, 30) self.assertTrue(b1.contains(b2)) self.assertTrue(b2.isWithin(b1)) b3 = Box.fromDegrees(0, -45, 90, 0) u = UnitVector3d(1, 1, -1) self.assertTrue(b1.intersects(b3)) self.assertTrue(u in b3) self.assertTrue(b3.contains(u)) b4 = Box.fromDegrees(200, 10, 300, 20) self.assertTrue(b1.isDisjointFrom(b4)) r = b1.relate(LonLat.fromDegrees(135, 10)) self.assertEqual(r, CONTAINS) r = b4.relate(b1) self.assertEqual(r, DISJOINT)
def test_construction(self): b = Box(Box.allLongitudes(), Box.allLatitudes()) self.assertTrue(b.isFull()) b = Box.fromDegrees(-90, -45, 90, 45) self.assertEqual(b, Box(b.getLon(), b.getLat())) a = Box.fromRadians(-0.5 * math.pi, -0.25 * math.pi, 0.5 * math.pi, 0.25 * math.pi) b = Box(LonLat.fromRadians(-0.5 * math.pi, -0.25 * math.pi), LonLat.fromRadians(0.5 * math.pi, 0.25 * math.pi)) c = Box(LonLat.fromRadians(0, 0), Angle(0.5 * math.pi), Angle(0.25 * math.pi)) d = c.clone() self.assertEqual(a, b) self.assertEqual(b, c) self.assertEqual(c, d) self.assertNotEqual(id(c), id(d)) b = Box() self.assertTrue(b.isEmpty()) self.assertTrue(Box.empty().isEmpty()) self.assertTrue(Box.full().isFull())
def _region_to_columns(self, region: Optional[Region], record: Dict[str, Any]) -> None: if region is None: return # Get spatial parameters from the bounding circle. circle = region.getBoundingCircle() center = LonLat(circle.getCenter()) record["s_ra"] = center.getLon().asDegrees() record["s_dec"] = center.getLat().asDegrees() record["s_fov"] = circle.getOpeningAngle().asDegrees() * 2 if isinstance(region, ConvexPolygon): poly = ["POLYGON ICRS"] for vertex in region.getVertices(): lon_lat = LonLat(vertex) poly += [ f"{lon_lat.getLon().asDegrees():.6f}", f"{lon_lat.getLat().asDegrees():.6f}", ] record["s_region"] = " ".join(poly) else: _LOG.warning(f"Unexpected region type: {type(region)}")
def testConstruction(self): p = LonLat.fromDegrees(45, 45) self.assertEqual(p, LonLat(NormalizedAngle.fromDegrees(45), Angle.fromDegrees(45))) u = UnitVector3d(p) q = LonLat(u) self.assertAlmostEqual( p.getLon().asRadians(), q.getLon().asRadians(), places=13) self.assertAlmostEqual( p.getLat().asRadians(), q.getLat().asRadians(), places=13) self.assertAlmostEqual(p.getLon().asRadians(), LonLat.latitudeOf(u).asRadians(), places=13) self.assertAlmostEqual(p.getLon().asRadians(), LonLat.longitudeOf(u).asRadians(), places=13)
def testConstruction(self): p = LonLat.fromDegrees(45, 45) self.assertEqual(p, LonLat(NormalizedAngle.fromDegrees(45), Angle.fromDegrees(45))) u = UnitVector3d(p) q = LonLat(u) self.assertAlmostEqual(p.getLon().asRadians(), q.getLon().asRadians(), places=13) self.assertAlmostEqual(p.getLat().asRadians(), q.getLat().asRadians(), places=13) self.assertAlmostEqual(p.getLon().asRadians(), LonLat.latitudeOf(u).asRadians(), places=13) self.assertAlmostEqual(p.getLon().asRadians(), LonLat.longitudeOf(u).asRadians(), places=13)
def test_expanding_and_clipping(self): a = Box.fromDegrees(0, 0, 10, 10) b = (a.expandedTo(LonLat.fromDegrees(20, 20)).expandedTo( Box.fromDegrees(0, 0, 30, 10)).clippedTo( Box.fromDegrees(10, 10, 15, 15)).clippedTo(LonLat.fromDegrees(11, 11))) a.expandTo(LonLat.fromDegrees(20, 20)) a.expandTo(Box.fromDegrees(0, 0, 30, 10)) a.clipTo(Box.fromDegrees(10, 10, 15, 15)) a.clipTo(LonLat.fromDegrees(11, 11)) self.assertEqual(a, b) self.assertEqual(a, LonLat.fromDegrees(11, 11)) a.clipTo(LonLat.fromDegrees(0, 0)) self.assertTrue(a.isEmpty())
def _filterDiaObjects(self, latest_objects, region): """Filter out objects from a catalog which are outside region. Parameters ---------- latest_objects : `afw.table.BaseCatalog` Catalog containing DiaObject records region : `sphgem.Region` Returns ------- Filtered `afw.table.BaseCatalog` containing only records contained in the region. """ mask = numpy.ndarray(len(latest_objects), dtype=bool) for i, obj in enumerate(latest_objects): # TODO: For now we use PPDB units (degrees), will have to be changed # in case we adopt afw units. lonLat = LonLat.fromRadians(obj['coord_ra'].asRadians(), obj['coord_dec'].asRadians()) dir_obj = UnitVector3d(lonLat) mask[i] = region.contains(dir_obj) return latest_objects.subset(mask)
def testString(self): self.assertEqual(str(LonLat.fromRadians(1, 1)), "[1, 1]") self.assertEqual(repr(LonLat.fromRadians(1, 1)), "LonLat.fromRadians(1.0, 1.0)")
def testComparisonOperators(self): self.assertEqual(LonLat.fromDegrees(45, 45), LonLat.fromDegrees(45, 45)) self.assertNotEqual(LonLat.fromDegrees(0, 0), LonLat.fromDegrees(45, 45))
def run(self): """Run whole shebang. """ if self.args.config: self.config.load(self.args.config) if self.args.dump_config: self.config.saveToStream(sys.stdout) return 0 # instantiate db interface db = Ppdb(self.config) if self.config.divide > 1: # check that we have reasonable MPI setup if self.config.mp_mode == "mpi": comm = MPI.COMM_WORLD num_proc = comm.Get_size() rank = comm.Get_rank() node = MPI.Get_processor_name() _LOG.info(COLOR_YELLOW + "MPI job rank=%d size=%d, node %s" + COLOR_RESET, rank, num_proc, node) num_tiles = self.config.divide**2 if num_proc != num_tiles: raise ValueError(f"Number of MPI processes ({num_proc}) " f"does not match number of tiles ({num_tiles})") if rank != 0: # run simple loop for all non-master processes return self.run_mpi_tile_loop(db, comm) # Initialize starting values from database visits table last_visit = db.lastVisit() if last_visit is not None: start_visit_id = last_visit.visitId + 1 start_time = last_visit.visitTime + timedelta(seconds=self.config.interval) else: start_visit_id = self.config.start_visit_id start_time = self.config.start_time_dt if self.config.divide > 1: _LOG.info("Will divide FOV into %dx%d regions", self.config.divide, self.config.divide) _LOG.info("Max. number of ranges for pixelator: %d", self.config.htm_max_ranges) # read sources file _LOG.info("Start loading variable sources from %r", self.config.sources_file) var_sources = numpy.load(self.config.sources_file) _LOG.info("Finished loading variable sources, count = %s", len(var_sources)) # diaObjectId for last new DIA object, for variable sources we use their # index as objectId, for transients we want to use ID outside that range if last_visit is not None and last_visit.lastObjectId is not None: self.lastObjectId = max(self.lastObjectId, last_visit.lastObjectId) if self.lastObjectId < len(var_sources): _LOG.error('next object id is too low: %s', self.lastObjectId) return 1 _LOG.debug("lastObjectId: %s", self.lastObjectId) # diaSourceId for last DIA source stored in database if last_visit is not None and last_visit.lastSourceId is not None: self.lastSourceId = max(self.lastSourceId, last_visit.lastSourceId) _LOG.info("lastSourceId: %s", self.lastSourceId) # loop over visits visitTimes = _visitTimes(start_time, self.config.interval, self.args.num_visits) for visit_id, dt in enumerate(visitTimes, start_visit_id): if visit_id % 1000 == 0: _LOG.info(COLOR_YELLOW + "+++ Start daily activities" + COLOR_RESET) db.dailyJob() _LOG.info(COLOR_YELLOW + "+++ Done with daily activities" + COLOR_RESET) _LOG.info(COLOR_GREEN + "+++ Start processing visit %s at %s" + COLOR_RESET, visit_id, dt) loop_timer = timer.Timer().start() with timer.Timer("DIA"): # point telescope in random southern direction pointing_xyz = generators.rand_sphere_xyz(1, -1)[0] pointing_v = UnitVector3d(pointing_xyz[0], pointing_xyz[1], pointing_xyz[2]) ra = LonLat.longitudeOf(pointing_v).asDegrees() decl = LonLat.latitudeOf(pointing_v).asDegrees() # sphgeom.Circle opening angle is actually a half of opening angle region = Circle(pointing_v, Angle(self.config.FOV_rad/2)) _LOG.info("Pointing ra, decl = %s, %s; xyz = %s", ra, decl, pointing_xyz) # Simulating difference image analysis dia = DIA.DIA(pointing_xyz, self.config.FOV_rad, var_sources, self.config.false_per_visit + self.config.transient_per_visit) sources, indices = dia.makeSources() _LOG.info("DIA generated %s sources", len(sources)) # assign IDs to transients for i in range(len(sources)): if indices[i] < 0: self.lastObjectId += 1 indices[i] = self.lastObjectId # print current database row counts, this takes long time # so only do it once in a while modu = 200 if visit_id <= 10000 else 1000 if visit_id % modu == 0: counts = db.tableRowCount() for tbl, count in sorted(counts.items()): _LOG.info('%s row count: %s', tbl, count) # numpy seems to do some multi-threaded stuff which "leaks" CPU cycles to the code below # and it gets counted as resource usage in timers, add a short delay here so that threads # finish and don't influence our timers below. time.sleep(0.1) if self.config.divide == 1: # do it in-process with timer.Timer("VisitProcessing"): self.visit(db, visit_id, dt, region, sources, indices) else: if self.config.mp_mode == "fork": tiles = geom.make_square_tiles( self.config.FOV_rad, self.config.divide, self.config.divide, pointing_v) with timer.Timer("VisitProcessing"): # spawn subprocesses to handle individual tiles children = [] for ix, iy, region in tiles: # make sure lastSourceId is unique in in each process self.lastSourceId += len(sources) tile = (ix, iy) pid = os.fork() if pid == 0: # child self.visit(db, visit_id, dt, region, sources, indices, tile) # stop here sys.exit(0) else: _LOG.debug("Forked process %d for tile %s", pid, tile) children.append(pid) # wait until all children finish for pid in children: try: pid, status = os.waitpid(pid, 0) if status != 0: _LOG.warning(COLOR_RED + "Child process PID=%s failed: %s" + COLOR_RESET, pid, status) except OSError as exc: _LOG.warning(COLOR_RED + "wait failed for PID=%s: %s" + COLOR_RESET, pid, exc) elif self.config.mp_mode == "mpi": tiles = geom.make_square_tiles( self.config.FOV_rad, self.config.divide, self.config.divide, pointing_v, False) _LOG.info("Split FOV into %d tiles for MPI", len(tiles)) # spawn subprocesses to handle individual tiles, special # care needed for self.lastSourceId because it's # propagated back from (0, 0) lastSourceId = self.lastSourceId tile_data = [] for ix, iy, region in tiles: lastSourceId += len(sources) tile = (ix, iy) tile_data += [(visit_id, dt, region, sources, indices, tile, lastSourceId)] # make sure lastSourceId is unique in in each process with timer.Timer("VisitProcessing"): _LOG.info("Scatter sources to %d tile processes", len(tile_data)) self.run_mpi_tile(db, MPI.COMM_WORLD, tile_data) self.lastSourceId = lastSourceId if not self.args.no_update: # store last visit info db.saveVisit(visit_id, dt) _LOG.info(COLOR_BLUE + "--- Finished processing visit %s, time: %s" + COLOR_RESET, visit_id, loop_timer) # stop MPI slaves if self.config.divide > 1 and self.config.mp_mode == "mpi": _LOG.info("Stopping MPI tile processes") tile_data = [None] * self.config.divide**2 self.run_mpi_tile(db, MPI.COMM_WORLD, tile_data) return 0
def run(self) -> Optional[int]: """Run whole shebang. """ # load configurations if self.args.app_config: self.config.load(self.args.app_config) if self.args.backend == "sql": self.dbconfig = ApdbSqlConfig() if self.args.config: self.dbconfig.load(self.args.config) elif self.args.backend == "cassandra": self.dbconfig = ApdbCassandraConfig() if self.args.config: self.dbconfig.load(self.args.config) if self.args.dump_config: self.config.saveToStream(sys.stdout) self.dbconfig.saveToStream(sys.stdout) return 0 # instantiate db interface db: Apdb if self.args.backend == "sql": db = ApdbSql(config=self.dbconfig) elif self.args.backend == "cassandra": db = ApdbCassandra(config=self.dbconfig) visitInfoStore = VisitInfoStore(self.args.visits_file) num_tiles = 1 if self.config.divide != 1: tiles = geom.make_tiles(self.config.FOV_rad, self.config.divide) num_tiles = len(tiles) # check that we have reasonable MPI setup if self.config.mp_mode == "mpi": comm = MPI.COMM_WORLD num_proc = comm.Get_size() rank = comm.Get_rank() node = MPI.Get_processor_name() _LOG.info( COLOR_YELLOW + "MPI job rank=%d size=%d, node %s" + COLOR_RESET, rank, num_proc, node) if num_proc != num_tiles: raise ValueError( f"Number of MPI processes ({num_proc}) " f"does not match number of tiles ({num_tiles})") if rank != 0: # run simple loop for all non-master processes self.run_mpi_tile_loop(db, comm) return None # Initialize starting values from database visits table last_visit = visitInfoStore.lastVisit() if last_visit is not None: start_visit_id = last_visit.visitId + 1 nsec = last_visit.visitTime.nsecs( DateTime.TAI) + self.config.interval * 1_000_000_000 start_time = DateTime(nsec, DateTime.TAI) else: start_visit_id = self.config.start_visit_id start_time = self.config.start_time_dt if self.config.divide > 1: _LOG.info("Will divide FOV into %d regions", num_tiles) src_read_period = self.config.src_read_period src_read_visits = round(self.config.src_read_period * self.config.src_read_duty_cycle) _LOG.info("Will read sources for %d visits out of %d", src_read_visits, src_read_period) # read sources file _LOG.info("Start loading variable sources from %r", self.config.sources_file) var_sources = numpy.load(self.config.sources_file) _LOG.info("Finished loading variable sources, count = %s", len(var_sources)) # diaObjectId for last new DIA object, for variable sources we use their # index as objectId, for transients we want to use ID outside that range if last_visit is not None and last_visit.lastObjectId is not None: self.lastObjectId = max(self.lastObjectId, last_visit.lastObjectId) if self.lastObjectId < len(var_sources): _LOG.error('next object id is too low: %s', self.lastObjectId) return 1 _LOG.debug("lastObjectId: %s", self.lastObjectId) # diaSourceId for last DIA source stored in database if last_visit is not None and last_visit.lastSourceId is not None: self.lastSourceId = max(self.lastSourceId, last_visit.lastSourceId) _LOG.info("lastSourceId: %s", self.lastSourceId) # loop over visits visitTimes = _visitTimes(start_time, self.config.interval, self.args.num_visits) for visit_id, dt in enumerate(visitTimes, start_visit_id): if visit_id % 1000 == 0: _LOG.info(COLOR_YELLOW + "+++ Start daily activities" + COLOR_RESET) db.dailyJob() _LOG.info(COLOR_YELLOW + "+++ Done with daily activities" + COLOR_RESET) _LOG.info( COLOR_GREEN + "+++ Start processing visit %s at %s" + COLOR_RESET, visit_id, dt) loop_timer = timer.Timer().start() with timer.Timer("DIA"): # point telescope in random southern direction pointing_xyz = generators.rand_sphere_xyz(1, -1)[0] pointing_v = UnitVector3d(pointing_xyz[0], pointing_xyz[1], pointing_xyz[2]) ra = LonLat.longitudeOf(pointing_v).asDegrees() decl = LonLat.latitudeOf(pointing_v).asDegrees() # sphgeom.Circle opening angle is actually a half of opening angle region = Circle(pointing_v, Angle(self.config.FOV_rad / 2)) _LOG.info("Pointing ra, decl = %s, %s; xyz = %s", ra, decl, pointing_xyz) # Simulating difference image analysis dia = DIA.DIA( pointing_xyz, self.config.FOV_rad, var_sources, self.config.false_per_visit + self.config.transient_per_visit) sources, indices = dia.makeSources() _LOG.info("DIA generated %s sources", len(sources)) # assign IDs to transients for i in range(len(sources)): if indices[i] < 0: self.lastObjectId += 1 indices[i] = self.lastObjectId # print current database row counts, this takes long time # so only do it once in a while modu = 200 if visit_id <= 10000 else 1000 if visit_id % modu == 0: if hasattr(db, "tableRowCount"): counts = db.tableRowCount() # type: ignore for tbl, count in sorted(counts.items()): _LOG.info('%s row count: %s', tbl, count) # numpy seems to do some multi-threaded stuff which "leaks" CPU cycles to the code below # and it gets counted as resource usage in timers, add a short delay here so that threads # finish and don't influence our timers below. time.sleep(0.1) if self.config.divide == 1: # do it in-process with timer.Timer("VisitProcessing"): self.visit(db, visit_id, dt, region, sources, indices) else: if self.config.mp_mode == "fork": tiles = geom.make_tiles(self.config.FOV_rad, self.config.divide, pointing_v) with timer.Timer("VisitProcessing"): # spawn subprocesses to handle individual tiles children = [] for ix, iy, region in tiles: # make sure lastSourceId is unique in in each process self.lastSourceId += len(sources) tile = (ix, iy) pid = os.fork() if pid == 0: # child self.visit(db, visit_id, dt, region, sources, indices, tile) # stop here sys.exit(0) else: _LOG.debug("Forked process %d for tile %s", pid, tile) children.append(pid) # wait until all children finish for pid in children: try: pid, status = os.waitpid(pid, 0) if status != 0: _LOG.warning( COLOR_RED + "Child process PID=%s failed: %s" + COLOR_RESET, pid, status) except OSError as exc: _LOG.warning( COLOR_RED + "wait failed for PID=%s: %s" + COLOR_RESET, pid, exc) elif self.config.mp_mode == "mpi": tiles = geom.make_tiles(self.config.FOV_rad, self.config.divide, pointing_v) _LOG.info("Split FOV into %d tiles for MPI", len(tiles)) # spawn subprocesses to handle individual tiles, special # care needed for self.lastSourceId because it's # propagated back from (0, 0) lastSourceId = self.lastSourceId tile_data = [] for ix, iy, region in tiles: lastSourceId += len(sources) tile = (ix, iy) tile_data += [(visit_id, dt, region, sources, indices, tile, lastSourceId)] # make sure lastSourceId is unique in in each process with timer.Timer("VisitProcessing"): _LOG.info("Scatter sources to %d tile processes", len(tile_data)) self.run_mpi_tile(db, MPI.COMM_WORLD, tile_data) self.lastSourceId = lastSourceId if not self.args.no_update: # store last visit info visitInfoStore.saveVisit(visit_id, dt, self.lastObjectId, self.lastSourceId) _LOG.info( COLOR_BLUE + "--- Finished processing visit %s, time: %s" + COLOR_RESET, visit_id, loop_timer) # stop MPI slaves if num_tiles > 1 and self.config.mp_mode == "mpi": _LOG.info("Stopping MPI tile processes") tile_data_stop = [None] * self.config.divide**2 self.run_mpi_tile(db, MPI.COMM_WORLD, tile_data_stop) return 0
def test_dilation_and_erosion(self): a = Box.fromRadians(0.5, -0.5, 1.5, 0.5) b = a.dilatedBy(Angle(0.5), Angle(0.5)).erodedBy(Angle(1), Angle(1)) a.dilateBy(Angle(0.5), Angle(0.5)).erodeBy(Angle(1), Angle(1)) self.assertEqual(a, b) self.assertEqual(a, LonLat.fromRadians(1, 0))
def testPickle(self): p = LonLat.fromRadians(2, 1) q = pickle.loads(pickle.dumps(p)) self.assertEqual(p, q)
def testSkyMapDimensions(self): """Test involving only skymap dimensions, no joins to instrument""" registry = self.registry # need a bunch of dimensions and datasets for test, we want # "abstract_filter" in the test so also have to add physical_filter # dimensions registry.addDimensionEntry("instrument", dict(instrument="DummyCam")) registry.addDimensionEntry("physical_filter", dict(instrument="DummyCam", physical_filter="dummy_r", abstract_filter="r")) registry.addDimensionEntry("physical_filter", dict(instrument="DummyCam", physical_filter="dummy_i", abstract_filter="i")) registry.addDimensionEntry("skymap", dict(skymap="DummyMap", hash="sha!".encode("utf8"))) for tract in range(10): registry.addDimensionEntry("tract", dict(skymap="DummyMap", tract=tract)) for patch in range(10): registry.addDimensionEntry("patch", dict(skymap="DummyMap", tract=tract, patch=patch, cell_x=0, cell_y=0, region=Box(LonLat(NormalizedAngle(0.), Angle(0.))))) # dataset types collection = "test" run = registry.makeRun(collection=collection) storageClass = StorageClass("testDataset") registry.storageClasses.registerStorageClass(storageClass) calexpType = DatasetType(name="deepCoadd_calexp", dimensions=registry.dimensions.extract(("skymap", "tract", "patch", "abstract_filter")), storageClass=storageClass) registry.registerDatasetType(calexpType) mergeType = DatasetType(name="deepCoadd_mergeDet", dimensions=registry.dimensions.extract(("skymap", "tract", "patch")), storageClass=storageClass) registry.registerDatasetType(mergeType) measType = DatasetType(name="deepCoadd_meas", dimensions=registry.dimensions.extract(("skymap", "tract", "patch", "abstract_filter")), storageClass=storageClass) registry.registerDatasetType(measType) dimensions = registry.dimensions.empty.union(calexpType.dimensions, mergeType.dimensions, measType.dimensions, implied=True) # add pre-existing datasets for tract in (1, 3, 5): for patch in (2, 4, 6, 7): dataId = dict(skymap="DummyMap", tract=tract, patch=patch) registry.addDataset(mergeType, dataId=dataId, run=run) for aFilter in ("i", "r"): dataId = dict(skymap="DummyMap", tract=tract, patch=patch, abstract_filter=aFilter) registry.addDataset(calexpType, dataId=dataId, run=run) # with empty expression builder = DataIdQueryBuilder.fromDimensions(registry, dimensions) builder.requireDataset(calexpType, collections=[collection]) builder.requireDataset(mergeType, collections=[collection]) rows = list(builder.execute()) self.assertEqual(len(rows), 3*4*2) # 4 tracts x 4 patches x 2 filters for dataId in rows: self.assertCountEqual(dataId.keys(), ("skymap", "tract", "patch", "abstract_filter")) self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5)) self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7)) self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r")) # limit to 2 tracts and 2 patches builder = DataIdQueryBuilder.fromDimensions(registry, dimensions) builder.requireDataset(calexpType, collections=[collection]) builder.requireDataset(mergeType, collections=[collection]) builder.whereParsedExpression("tract IN (1, 5) AND patch.patch IN (2, 7)") rows = list(builder.execute()) self.assertEqual(len(rows), 2*2*2) # 4 tracts x 4 patches x 2 filters self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 5)) self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 7)) self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r")) # limit to single filter builder = DataIdQueryBuilder.fromDimensions(registry, dimensions) builder.requireDataset(calexpType, collections=[collection]) builder.requireDataset(mergeType, collections=[collection]) builder.whereParsedExpression("abstract_filter = 'i'") rows = list(builder.execute()) self.assertEqual(len(rows), 3*4*1) # 4 tracts x 4 patches x 2 filters self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5)) self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7)) self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i",)) # expression excludes everything, specifying non-existing skymap is # not a fatal error, it's operator error builder = DataIdQueryBuilder.fromDimensions(registry, dimensions) builder.requireDataset(calexpType, collections=[collection]) builder.requireDataset(mergeType, collections=[collection]) builder.whereParsedExpression("skymap = 'Mars'") rows = list(builder.execute()) self.assertEqual(len(rows), 0)
def testString(self): p = LonLat.fromRadians(1, 1) self.assertEqual(str(p), '[1.0, 1.0]') self.assertEqual(repr(p), 'LonLat.fromRadians(1.0, 1.0)') self.assertEqual(p, eval(repr(p), dict(LonLat=LonLat)))
def in_region(obj: Any) -> bool: lonLat = LonLat.fromDegrees(obj['ra'], obj['decl']) dir_obj = UnitVector3d(lonLat) return region.contains(dir_obj)