def test_relationships(self):
e = Ellipse(UnitVector3d.X(), Angle(math.pi / 3), Angle(math.pi / 6),
Angle(0))
self.assertTrue(e.contains(UnitVector3d.X()))
self.assertTrue(UnitVector3d.X() in e)
c = Circle(UnitVector3d.X(), Angle(math.pi / 2))
self.assertEqual(c.relate(e), CONTAINS)
self.assertEqual(e.relate(c), WITHIN)
def _makePixelRanges():
"""Generate pixel ID ranges for some envelope region"""
pointing_v = UnitVector3d(1., 1., -1.)
fov = 0.05 # radians
region = Circle(pointing_v, Angle(fov / 2))
pixelator = HtmPixelization(HTM_LEVEL)
indices = pixelator.envelope(region, 128)
return indices.ranges()
def test_complement(self):
c = Circle(UnitVector3d.X(), 2.0)
d = c.complemented()
c.complement()
self.assertEqual(c, d)
self.assertEqual(c.getCenter(), -UnitVector3d.X())
self.assertEqual(c.getSquaredChordLength(), 2.0)
def test_string(self):
c = Circle(UnitVector3d.Z(), Angle(1.0))
self.assertEqual(str(c), 'Circle([0.0, 0.0, 1.0], 1.0)')
self.assertEqual(repr(c),
'Circle(UnitVector3d(0.0, 0.0, 1.0), Angle(1.0))')
self.assertEqual(
c,
eval(repr(c),
dict(Angle=Angle, Circle=Circle, UnitVector3d=UnitVector3d)))
def test_envelope_and_interior(self):
pixelization = Mq3cPixelization(1)
c = Circle(UnitVector3d(1.0, -0.5, -0.5), Angle.fromDegrees(0.1))
rs = pixelization.envelope(c)
self.assertTrue(rs == RangeSet(44))
rs = pixelization.envelope(c, 1)
self.assertTrue(rs == RangeSet(44))
self.assertTrue(rs.isWithin(pixelization.universe()))
rs = pixelization.interior(c)
self.assertTrue(rs.empty())
def test_envelope_and_interior(self):
pixelization = HtmPixelization(3)
c = Circle(UnitVector3d(1, 1, 1), Angle.fromDegrees(0.1))
rs = pixelization.envelope(c)
self.assertTrue(rs == RangeSet(0x3ff))
rs = pixelization.envelope(c, 1)
self.assertTrue(rs == RangeSet(0x3ff))
self.assertTrue(rs.isWithin(pixelization.universe()))
rs = pixelization.interior(c)
self.assertTrue(rs.empty())
def test_dilation_and_erosion(self):
a = Angle(math.pi / 2)
c = Circle(UnitVector3d.X())
d = c.dilatedBy(a).erodedBy(a)
c.dilateBy(a).erodeBy(a)
self.assertEqual(c, d)
self.assertEqual(c, Circle(UnitVector3d.X()))
def test_construction(self):
self.assertTrue(Ellipse.empty().isEmpty())
self.assertTrue(Ellipse().isEmpty())
self.assertTrue(Ellipse.full().isFull())
e = Ellipse(Circle(UnitVector3d.X(), Angle(math.pi / 2)))
f = Ellipse(UnitVector3d.X(), Angle(math.pi / 2))
self.assertEqual(e, f)
self.assertEqual(e.getAlpha(), e.getBeta())
self.assertTrue(e.isCircle())
self.assertTrue(e.isGreatCircle())
g = Ellipse(e)
h = e.clone()
self.assertEqual(e, g)
self.assertEqual(g, h)
self.assertNotEqual(id(e), id(g))
self.assertNotEqual(id(g), id(h))
def testRelationships(self):
p = ConvexPolygon(
[UnitVector3d.Z(),
UnitVector3d.X(),
UnitVector3d.Y()])
self.assertTrue(p.contains(p.getCentroid()))
boundingCircle = p.getBoundingCircle()
self.assertEqual(boundingCircle.relate(p), CONTAINS)
self.assertTrue(p.isWithin(boundingCircle))
self.assertTrue(p.intersects(boundingCircle))
self.assertFalse(p.isDisjointFrom(boundingCircle))
self.assertFalse(p.contains(boundingCircle))
tinyCircle = Circle(boundingCircle.getCenter())
self.assertFalse(p.isWithin(tinyCircle))
self.assertTrue(p.intersects(tinyCircle))
self.assertFalse(p.isDisjointFrom(tinyCircle))
self.assertTrue(p.contains(tinyCircle))
def test_expanding_and_clipping(self):
a = Circle.empty()
b = (a.expandedTo(UnitVector3d.X()).expandedTo(
Circle(UnitVector3d.Y(),
1)).clippedTo(Circle(UnitVector3d(1, 1, 0),
1)).clippedTo(UnitVector3d.Y()))
a.expandTo(UnitVector3d.X())
a.expandTo(Circle(UnitVector3d.Y(), 1))
a.clipTo(Circle(UnitVector3d(1, 1, 0), 1))
a.clipTo(UnitVector3d.Y())
self.assertEqual(a, b)
self.assertEqual(a, Circle(UnitVector3d.Y()))
a.clipTo(UnitVector3d.Z())
self.assertTrue(a.isEmpty())
def test_relationships(self):
c = Circle(UnitVector3d.X(), Angle.fromDegrees(0.1))
d = Circle(UnitVector3d(1, 1, 1), Angle(math.pi / 2))
e = Circle(-UnitVector3d.X())
self.assertTrue(c.contains(UnitVector3d.X()))
self.assertTrue(UnitVector3d.X() in c)
self.assertTrue(d.contains(c))
self.assertTrue(c.isWithin(d))
self.assertTrue(c.intersects(d))
self.assertTrue(c.intersects(UnitVector3d.X()))
self.assertTrue(e.isDisjointFrom(d))
self.assertEqual(d.relate(c), CONTAINS)
self.assertEqual(e.relate(d), DISJOINT)
def test_center_and_dimensions(self):
c = Circle(UnitVector3d.X(), 1)
self.assertEqual(c.getCenter(), UnitVector3d.X())
self.assertEqual(c.getSquaredChordLength(), 1)
self.assertAlmostEqual(c.getOpeningAngle().asRadians(), math.pi / 3)
def test_comparison_operators(self):
c = Circle(UnitVector3d.X(), 4.0)
d = Circle(UnitVector3d.Y(), 4.0)
self.assertEqual(c, d)
self.assertTrue(c.isFull())
self.assertNotEqual(c, Circle(UnitVector3d.Z()))
def test_construction(self):
self.assertTrue(Circle.empty().isEmpty())
self.assertTrue(Circle().isEmpty())
self.assertTrue(Circle.full().isFull())
c = Circle(UnitVector3d.X())
self.assertEqual(c.getOpeningAngle(), Angle(0))
self.assertEqual(c.getSquaredChordLength(), 0)
c = Circle(UnitVector3d.Z(), 2.0)
self.assertTrue(c.contains(UnitVector3d.Z()))
c = Circle(UnitVector3d.Z(), Angle(math.pi))
self.assertTrue(c.isFull())
d = c.clone()
self.assertEqual(c, d)
self.assertNotEqual(id(c), id(d))
e = Circle(d)
self.assertEqual(d, e)
def test_yaml(self):
a = Circle(UnitVector3d(1, -1, 1), 1.0)
b = yaml.safe_load(yaml.dump(a))
self.assertEqual(a, b)
def test_codec(self):
c = Circle(UnitVector3d.Y(), 1.0)
s = c.encode()
self.assertEqual(Circle.decode(s), c)
self.assertEqual(Region.decode(s), c)
def test_area(self):
c = Circle(UnitVector3d(1, 1, 1), 2.0)
self.assertAlmostEqual(c.getArea(), 2 * math.pi)
def test_pickle(self):
a = Circle(UnitVector3d(1, -1, 1), 1.0)
b = pickle.loads(pickle.dumps(a, pickle.HIGHEST_PROTOCOL))
self.assertEqual(a, b)
def run(self) -> Optional[int]:
"""Run whole shebang.
"""
# load configurations
if self.args.app_config:
self.config.load(self.args.app_config)
if self.args.backend == "sql":
self.dbconfig = ApdbSqlConfig()
if self.args.config:
self.dbconfig.load(self.args.config)
elif self.args.backend == "cassandra":
self.dbconfig = ApdbCassandraConfig()
if self.args.config:
self.dbconfig.load(self.args.config)
if self.args.dump_config:
self.config.saveToStream(sys.stdout)
self.dbconfig.saveToStream(sys.stdout)
return 0
# instantiate db interface
db: Apdb
if self.args.backend == "sql":
db = ApdbSql(config=self.dbconfig)
elif self.args.backend == "cassandra":
db = ApdbCassandra(config=self.dbconfig)
visitInfoStore = VisitInfoStore(self.args.visits_file)
num_tiles = 1
if self.config.divide != 1:
tiles = geom.make_tiles(self.config.FOV_rad, self.config.divide)
num_tiles = len(tiles)
# check that we have reasonable MPI setup
if self.config.mp_mode == "mpi":
comm = MPI.COMM_WORLD
num_proc = comm.Get_size()
rank = comm.Get_rank()
node = MPI.Get_processor_name()
_LOG.info(
COLOR_YELLOW + "MPI job rank=%d size=%d, node %s" +
COLOR_RESET, rank, num_proc, node)
if num_proc != num_tiles:
raise ValueError(
f"Number of MPI processes ({num_proc}) "
f"does not match number of tiles ({num_tiles})")
if rank != 0:
# run simple loop for all non-master processes
self.run_mpi_tile_loop(db, comm)
return None
# Initialize starting values from database visits table
last_visit = visitInfoStore.lastVisit()
if last_visit is not None:
start_visit_id = last_visit.visitId + 1
nsec = last_visit.visitTime.nsecs(
DateTime.TAI) + self.config.interval * 1_000_000_000
start_time = DateTime(nsec, DateTime.TAI)
else:
start_visit_id = self.config.start_visit_id
start_time = self.config.start_time_dt
if self.config.divide > 1:
_LOG.info("Will divide FOV into %d regions", num_tiles)
src_read_period = self.config.src_read_period
src_read_visits = round(self.config.src_read_period *
self.config.src_read_duty_cycle)
_LOG.info("Will read sources for %d visits out of %d", src_read_visits,
src_read_period)
# read sources file
_LOG.info("Start loading variable sources from %r",
self.config.sources_file)
var_sources = numpy.load(self.config.sources_file)
_LOG.info("Finished loading variable sources, count = %s",
len(var_sources))
# diaObjectId for last new DIA object, for variable sources we use their
# index as objectId, for transients we want to use ID outside that range
if last_visit is not None and last_visit.lastObjectId is not None:
self.lastObjectId = max(self.lastObjectId, last_visit.lastObjectId)
if self.lastObjectId < len(var_sources):
_LOG.error('next object id is too low: %s', self.lastObjectId)
return 1
_LOG.debug("lastObjectId: %s", self.lastObjectId)
# diaSourceId for last DIA source stored in database
if last_visit is not None and last_visit.lastSourceId is not None:
self.lastSourceId = max(self.lastSourceId, last_visit.lastSourceId)
_LOG.info("lastSourceId: %s", self.lastSourceId)
# loop over visits
visitTimes = _visitTimes(start_time, self.config.interval,
self.args.num_visits)
for visit_id, dt in enumerate(visitTimes, start_visit_id):
if visit_id % 1000 == 0:
_LOG.info(COLOR_YELLOW + "+++ Start daily activities" +
COLOR_RESET)
db.dailyJob()
_LOG.info(COLOR_YELLOW + "+++ Done with daily activities" +
COLOR_RESET)
_LOG.info(
COLOR_GREEN + "+++ Start processing visit %s at %s" +
COLOR_RESET, visit_id, dt)
loop_timer = timer.Timer().start()
with timer.Timer("DIA"):
# point telescope in random southern direction
pointing_xyz = generators.rand_sphere_xyz(1, -1)[0]
pointing_v = UnitVector3d(pointing_xyz[0], pointing_xyz[1],
pointing_xyz[2])
ra = LonLat.longitudeOf(pointing_v).asDegrees()
decl = LonLat.latitudeOf(pointing_v).asDegrees()
# sphgeom.Circle opening angle is actually a half of opening angle
region = Circle(pointing_v, Angle(self.config.FOV_rad / 2))
_LOG.info("Pointing ra, decl = %s, %s; xyz = %s", ra, decl,
pointing_xyz)
# Simulating difference image analysis
dia = DIA.DIA(
pointing_xyz, self.config.FOV_rad, var_sources,
self.config.false_per_visit +
self.config.transient_per_visit)
sources, indices = dia.makeSources()
_LOG.info("DIA generated %s sources", len(sources))
# assign IDs to transients
for i in range(len(sources)):
if indices[i] < 0:
self.lastObjectId += 1
indices[i] = self.lastObjectId
# print current database row counts, this takes long time
# so only do it once in a while
modu = 200 if visit_id <= 10000 else 1000
if visit_id % modu == 0:
if hasattr(db, "tableRowCount"):
counts = db.tableRowCount() # type: ignore
for tbl, count in sorted(counts.items()):
_LOG.info('%s row count: %s', tbl, count)
# numpy seems to do some multi-threaded stuff which "leaks" CPU cycles to the code below
# and it gets counted as resource usage in timers, add a short delay here so that threads
# finish and don't influence our timers below.
time.sleep(0.1)
if self.config.divide == 1:
# do it in-process
with timer.Timer("VisitProcessing"):
self.visit(db, visit_id, dt, region, sources, indices)
else:
if self.config.mp_mode == "fork":
tiles = geom.make_tiles(self.config.FOV_rad,
self.config.divide, pointing_v)
with timer.Timer("VisitProcessing"):
# spawn subprocesses to handle individual tiles
children = []
for ix, iy, region in tiles:
# make sure lastSourceId is unique in in each process
self.lastSourceId += len(sources)
tile = (ix, iy)
pid = os.fork()
if pid == 0:
# child
self.visit(db, visit_id, dt, region, sources,
indices, tile)
# stop here
sys.exit(0)
else:
_LOG.debug("Forked process %d for tile %s",
pid, tile)
children.append(pid)
# wait until all children finish
for pid in children:
try:
pid, status = os.waitpid(pid, 0)
if status != 0:
_LOG.warning(
COLOR_RED +
"Child process PID=%s failed: %s" +
COLOR_RESET, pid, status)
except OSError as exc:
_LOG.warning(
COLOR_RED + "wait failed for PID=%s: %s" +
COLOR_RESET, pid, exc)
elif self.config.mp_mode == "mpi":
tiles = geom.make_tiles(self.config.FOV_rad,
self.config.divide, pointing_v)
_LOG.info("Split FOV into %d tiles for MPI", len(tiles))
# spawn subprocesses to handle individual tiles, special
# care needed for self.lastSourceId because it's
# propagated back from (0, 0)
lastSourceId = self.lastSourceId
tile_data = []
for ix, iy, region in tiles:
lastSourceId += len(sources)
tile = (ix, iy)
tile_data += [(visit_id, dt, region, sources, indices,
tile, lastSourceId)]
# make sure lastSourceId is unique in in each process
with timer.Timer("VisitProcessing"):
_LOG.info("Scatter sources to %d tile processes",
len(tile_data))
self.run_mpi_tile(db, MPI.COMM_WORLD, tile_data)
self.lastSourceId = lastSourceId
if not self.args.no_update:
# store last visit info
visitInfoStore.saveVisit(visit_id, dt, self.lastObjectId,
self.lastSourceId)
_LOG.info(
COLOR_BLUE + "--- Finished processing visit %s, time: %s" +
COLOR_RESET, visit_id, loop_timer)
# stop MPI slaves
if num_tiles > 1 and self.config.mp_mode == "mpi":
_LOG.info("Stopping MPI tile processes")
tile_data_stop = [None] * self.config.divide**2
self.run_mpi_tile(db, MPI.COMM_WORLD, tile_data_stop)
return 0