def test_dilation_and_erosion(self): a = Angle(math.pi / 2) c = Circle(UnitVector3d.X()) d = c.dilatedBy(a).erodedBy(a) c.dilateBy(a).erodeBy(a) self.assertEqual(c, d) self.assertEqual(c, Circle(UnitVector3d.X()))
def test_relationships(self): c = Circle(UnitVector3d.X(), Angle.fromDegrees(0.1)) d = Circle(UnitVector3d(1, 1, 1), Angle(math.pi / 2)) e = Circle(-UnitVector3d.X()) self.assertTrue(c.contains(UnitVector3d.X())) self.assertTrue(UnitVector3d.X() in c) self.assertTrue(d.contains(c)) self.assertTrue(c.isWithin(d)) self.assertTrue(c.intersects(d)) self.assertTrue(c.intersects(UnitVector3d.X())) self.assertTrue(e.isDisjointFrom(d)) self.assertEqual(d.relate(c), CONTAINS) self.assertEqual(e.relate(d), DISJOINT)
def test_expanding_and_clipping(self): a = Circle.empty() b = (a.expandedTo(UnitVector3d.X()).expandedTo( Circle(UnitVector3d.Y(), 1)).clippedTo(Circle(UnitVector3d(1, 1, 0), 1)).clippedTo(UnitVector3d.Y())) a.expandTo(UnitVector3d.X()) a.expandTo(Circle(UnitVector3d.Y(), 1)) a.clipTo(Circle(UnitVector3d(1, 1, 0), 1)) a.clipTo(UnitVector3d.Y()) self.assertEqual(a, b) self.assertEqual(a, Circle(UnitVector3d.Y())) a.clipTo(UnitVector3d.Z()) self.assertTrue(a.isEmpty())
def test_complement(self): c = Circle(UnitVector3d.X(), 2.0) d = c.complemented() c.complement() self.assertEqual(c, d) self.assertEqual(c.getCenter(), -UnitVector3d.X()) self.assertEqual(c.getSquaredChordLength(), 2.0)
def test_construction(self): self.assertTrue(Circle.empty().isEmpty()) self.assertTrue(Circle().isEmpty()) self.assertTrue(Circle.full().isFull()) c = Circle(UnitVector3d.X()) self.assertEqual(c.getOpeningAngle(), Angle(0)) self.assertEqual(c.getSquaredChordLength(), 0) c = Circle(UnitVector3d.Z(), 2.0) self.assertTrue(c.contains(UnitVector3d.Z())) c = Circle(UnitVector3d.Z(), Angle(math.pi)) self.assertTrue(c.isFull()) d = c.clone() self.assertEqual(c, d) self.assertNotEqual(id(c), id(d)) e = Circle(d) self.assertEqual(d, e)
def _makePixelRanges(): """Generate pixel ID ranges for some envelope region""" pointing_v = UnitVector3d(1., 1., -1.) fov = 0.05 # radians region = Circle(pointing_v, Angle(fov / 2)) pixelator = HtmPixelization(HTM_LEVEL) indices = pixelator.envelope(region, 128) return indices.ranges()
def test_relationships(self): e = Ellipse(UnitVector3d.X(), Angle(math.pi / 3), Angle(math.pi / 6), Angle(0)) self.assertTrue(e.contains(UnitVector3d.X())) self.assertTrue(UnitVector3d.X() in e) c = Circle(UnitVector3d.X(), Angle(math.pi / 2)) self.assertEqual(c.relate(e), CONTAINS) self.assertEqual(e.relate(c), WITHIN)
def test_string(self): c = Circle(UnitVector3d.Z(), Angle(1.0)) self.assertEqual(str(c), 'Circle([0.0, 0.0, 1.0], 1.0)') self.assertEqual(repr(c), 'Circle(UnitVector3d(0.0, 0.0, 1.0), Angle(1.0))') self.assertEqual( c, eval(repr(c), dict(Angle=Angle, Circle=Circle, UnitVector3d=UnitVector3d)))
def test_envelope_and_interior(self): pixelization = Mq3cPixelization(1) c = Circle(UnitVector3d(1.0, -0.5, -0.5), Angle.fromDegrees(0.1)) rs = pixelization.envelope(c) self.assertTrue(rs == RangeSet(44)) rs = pixelization.envelope(c, 1) self.assertTrue(rs == RangeSet(44)) self.assertTrue(rs.isWithin(pixelization.universe())) rs = pixelization.interior(c) self.assertTrue(rs.empty())
def test_envelope_and_interior(self): pixelization = HtmPixelization(3) c = Circle(UnitVector3d(1, 1, 1), Angle.fromDegrees(0.1)) rs = pixelization.envelope(c) self.assertTrue(rs == RangeSet(0x3ff)) rs = pixelization.envelope(c, 1) self.assertTrue(rs == RangeSet(0x3ff)) self.assertTrue(rs.isWithin(pixelization.universe())) rs = pixelization.interior(c) self.assertTrue(rs.empty())
def test_construction(self): self.assertTrue(Ellipse.empty().isEmpty()) self.assertTrue(Ellipse().isEmpty()) self.assertTrue(Ellipse.full().isFull()) e = Ellipse(Circle(UnitVector3d.X(), Angle(math.pi / 2))) f = Ellipse(UnitVector3d.X(), Angle(math.pi / 2)) self.assertEqual(e, f) self.assertEqual(e.getAlpha(), e.getBeta()) self.assertTrue(e.isCircle()) self.assertTrue(e.isGreatCircle()) g = Ellipse(e) h = e.clone() self.assertEqual(e, g) self.assertEqual(g, h) self.assertNotEqual(id(e), id(g)) self.assertNotEqual(id(g), id(h))
def testRelationships(self): p = ConvexPolygon( [UnitVector3d.Z(), UnitVector3d.X(), UnitVector3d.Y()]) self.assertTrue(p.contains(p.getCentroid())) boundingCircle = p.getBoundingCircle() self.assertEqual(boundingCircle.relate(p), CONTAINS) self.assertTrue(p.isWithin(boundingCircle)) self.assertTrue(p.intersects(boundingCircle)) self.assertFalse(p.isDisjointFrom(boundingCircle)) self.assertFalse(p.contains(boundingCircle)) tinyCircle = Circle(boundingCircle.getCenter()) self.assertFalse(p.isWithin(tinyCircle)) self.assertTrue(p.intersects(tinyCircle)) self.assertFalse(p.isDisjointFrom(tinyCircle)) self.assertTrue(p.contains(tinyCircle))
def test_area(self): c = Circle(UnitVector3d(1, 1, 1), 2.0) self.assertAlmostEqual(c.getArea(), 2 * math.pi)
def test_center_and_dimensions(self): c = Circle(UnitVector3d.X(), 1) self.assertEqual(c.getCenter(), UnitVector3d.X()) self.assertEqual(c.getSquaredChordLength(), 1) self.assertAlmostEqual(c.getOpeningAngle().asRadians(), math.pi / 3)
def test_comparison_operators(self): c = Circle(UnitVector3d.X(), 4.0) d = Circle(UnitVector3d.Y(), 4.0) self.assertEqual(c, d) self.assertTrue(c.isFull()) self.assertNotEqual(c, Circle(UnitVector3d.Z()))
def test_codec(self): c = Circle(UnitVector3d.Y(), 1.0) s = c.encode() self.assertEqual(Circle.decode(s), c) self.assertEqual(Region.decode(s), c)
def test_yaml(self): a = Circle(UnitVector3d(1, -1, 1), 1.0) b = yaml.safe_load(yaml.dump(a)) self.assertEqual(a, b)
def test_pickle(self): a = Circle(UnitVector3d(1, -1, 1), 1.0) b = pickle.loads(pickle.dumps(a, pickle.HIGHEST_PROTOCOL)) self.assertEqual(a, b)
def run(self) -> Optional[int]: """Run whole shebang. """ # load configurations if self.args.app_config: self.config.load(self.args.app_config) if self.args.backend == "sql": self.dbconfig = ApdbSqlConfig() if self.args.config: self.dbconfig.load(self.args.config) elif self.args.backend == "cassandra": self.dbconfig = ApdbCassandraConfig() if self.args.config: self.dbconfig.load(self.args.config) if self.args.dump_config: self.config.saveToStream(sys.stdout) self.dbconfig.saveToStream(sys.stdout) return 0 # instantiate db interface db: Apdb if self.args.backend == "sql": db = ApdbSql(config=self.dbconfig) elif self.args.backend == "cassandra": db = ApdbCassandra(config=self.dbconfig) visitInfoStore = VisitInfoStore(self.args.visits_file) num_tiles = 1 if self.config.divide != 1: tiles = geom.make_tiles(self.config.FOV_rad, self.config.divide) num_tiles = len(tiles) # check that we have reasonable MPI setup if self.config.mp_mode == "mpi": comm = MPI.COMM_WORLD num_proc = comm.Get_size() rank = comm.Get_rank() node = MPI.Get_processor_name() _LOG.info( COLOR_YELLOW + "MPI job rank=%d size=%d, node %s" + COLOR_RESET, rank, num_proc, node) if num_proc != num_tiles: raise ValueError( f"Number of MPI processes ({num_proc}) " f"does not match number of tiles ({num_tiles})") if rank != 0: # run simple loop for all non-master processes self.run_mpi_tile_loop(db, comm) return None # Initialize starting values from database visits table last_visit = visitInfoStore.lastVisit() if last_visit is not None: start_visit_id = last_visit.visitId + 1 nsec = last_visit.visitTime.nsecs( DateTime.TAI) + self.config.interval * 1_000_000_000 start_time = DateTime(nsec, DateTime.TAI) else: start_visit_id = self.config.start_visit_id start_time = self.config.start_time_dt if self.config.divide > 1: _LOG.info("Will divide FOV into %d regions", num_tiles) src_read_period = self.config.src_read_period src_read_visits = round(self.config.src_read_period * self.config.src_read_duty_cycle) _LOG.info("Will read sources for %d visits out of %d", src_read_visits, src_read_period) # read sources file _LOG.info("Start loading variable sources from %r", self.config.sources_file) var_sources = numpy.load(self.config.sources_file) _LOG.info("Finished loading variable sources, count = %s", len(var_sources)) # diaObjectId for last new DIA object, for variable sources we use their # index as objectId, for transients we want to use ID outside that range if last_visit is not None and last_visit.lastObjectId is not None: self.lastObjectId = max(self.lastObjectId, last_visit.lastObjectId) if self.lastObjectId < len(var_sources): _LOG.error('next object id is too low: %s', self.lastObjectId) return 1 _LOG.debug("lastObjectId: %s", self.lastObjectId) # diaSourceId for last DIA source stored in database if last_visit is not None and last_visit.lastSourceId is not None: self.lastSourceId = max(self.lastSourceId, last_visit.lastSourceId) _LOG.info("lastSourceId: %s", self.lastSourceId) # loop over visits visitTimes = _visitTimes(start_time, self.config.interval, self.args.num_visits) for visit_id, dt in enumerate(visitTimes, start_visit_id): if visit_id % 1000 == 0: _LOG.info(COLOR_YELLOW + "+++ Start daily activities" + COLOR_RESET) db.dailyJob() _LOG.info(COLOR_YELLOW + "+++ Done with daily activities" + COLOR_RESET) _LOG.info( COLOR_GREEN + "+++ Start processing visit %s at %s" + COLOR_RESET, visit_id, dt) loop_timer = timer.Timer().start() with timer.Timer("DIA"): # point telescope in random southern direction pointing_xyz = generators.rand_sphere_xyz(1, -1)[0] pointing_v = UnitVector3d(pointing_xyz[0], pointing_xyz[1], pointing_xyz[2]) ra = LonLat.longitudeOf(pointing_v).asDegrees() decl = LonLat.latitudeOf(pointing_v).asDegrees() # sphgeom.Circle opening angle is actually a half of opening angle region = Circle(pointing_v, Angle(self.config.FOV_rad / 2)) _LOG.info("Pointing ra, decl = %s, %s; xyz = %s", ra, decl, pointing_xyz) # Simulating difference image analysis dia = DIA.DIA( pointing_xyz, self.config.FOV_rad, var_sources, self.config.false_per_visit + self.config.transient_per_visit) sources, indices = dia.makeSources() _LOG.info("DIA generated %s sources", len(sources)) # assign IDs to transients for i in range(len(sources)): if indices[i] < 0: self.lastObjectId += 1 indices[i] = self.lastObjectId # print current database row counts, this takes long time # so only do it once in a while modu = 200 if visit_id <= 10000 else 1000 if visit_id % modu == 0: if hasattr(db, "tableRowCount"): counts = db.tableRowCount() # type: ignore for tbl, count in sorted(counts.items()): _LOG.info('%s row count: %s', tbl, count) # numpy seems to do some multi-threaded stuff which "leaks" CPU cycles to the code below # and it gets counted as resource usage in timers, add a short delay here so that threads # finish and don't influence our timers below. time.sleep(0.1) if self.config.divide == 1: # do it in-process with timer.Timer("VisitProcessing"): self.visit(db, visit_id, dt, region, sources, indices) else: if self.config.mp_mode == "fork": tiles = geom.make_tiles(self.config.FOV_rad, self.config.divide, pointing_v) with timer.Timer("VisitProcessing"): # spawn subprocesses to handle individual tiles children = [] for ix, iy, region in tiles: # make sure lastSourceId is unique in in each process self.lastSourceId += len(sources) tile = (ix, iy) pid = os.fork() if pid == 0: # child self.visit(db, visit_id, dt, region, sources, indices, tile) # stop here sys.exit(0) else: _LOG.debug("Forked process %d for tile %s", pid, tile) children.append(pid) # wait until all children finish for pid in children: try: pid, status = os.waitpid(pid, 0) if status != 0: _LOG.warning( COLOR_RED + "Child process PID=%s failed: %s" + COLOR_RESET, pid, status) except OSError as exc: _LOG.warning( COLOR_RED + "wait failed for PID=%s: %s" + COLOR_RESET, pid, exc) elif self.config.mp_mode == "mpi": tiles = geom.make_tiles(self.config.FOV_rad, self.config.divide, pointing_v) _LOG.info("Split FOV into %d tiles for MPI", len(tiles)) # spawn subprocesses to handle individual tiles, special # care needed for self.lastSourceId because it's # propagated back from (0, 0) lastSourceId = self.lastSourceId tile_data = [] for ix, iy, region in tiles: lastSourceId += len(sources) tile = (ix, iy) tile_data += [(visit_id, dt, region, sources, indices, tile, lastSourceId)] # make sure lastSourceId is unique in in each process with timer.Timer("VisitProcessing"): _LOG.info("Scatter sources to %d tile processes", len(tile_data)) self.run_mpi_tile(db, MPI.COMM_WORLD, tile_data) self.lastSourceId = lastSourceId if not self.args.no_update: # store last visit info visitInfoStore.saveVisit(visit_id, dt, self.lastObjectId, self.lastSourceId) _LOG.info( COLOR_BLUE + "--- Finished processing visit %s, time: %s" + COLOR_RESET, visit_id, loop_timer) # stop MPI slaves if num_tiles > 1 and self.config.mp_mode == "mpi": _LOG.info("Stopping MPI tile processes") tile_data_stop = [None] * self.config.divide**2 self.run_mpi_tile(db, MPI.COMM_WORLD, tile_data_stop) return 0