def test_comparison_operators(self):
self.assertEqual(Box(LonLat.fromDegrees(45, 45)),
LonLat.fromDegrees(45, 45))
self.assertEqual(Box.fromDegrees(90, -45, 180, 45),
Box(NormalizedAngleInterval.fromDegrees(90, 180),
AngleInterval.fromDegrees(-45, 45)))
self.assertNotEqual(Box(LonLat.fromDegrees(45, 45)),
LonLat.fromDegrees(45, 90))
self.assertNotEqual(Box.fromDegrees(90, -45, 180, 45),
Box.fromDegrees(90, -45, 180, 90))
def testConstruction(self):
a1 = NormalizedAngle(1.0)
a2 = NormalizedAngle.fromRadians(1.0)
a3 = NormalizedAngle.fromDegrees(57.29577951308232)
self.assertEqual(a1, a2)
self.assertEqual(a1.asRadians(), 1.0)
self.assertEqual(a1, a3)
self.assertEqual(a1.asDegrees(), 57.29577951308232)
self.assertEqual(NormalizedAngle.between(NormalizedAngle(0), NormalizedAngle(1)),
NormalizedAngle(1))
a = NormalizedAngle.center(NormalizedAngle(0), NormalizedAngle(1))
self.assertAlmostEqual(a.asRadians(), 0.5, places=15)
a = NormalizedAngle(LonLat.fromDegrees(45, 0), LonLat.fromDegrees(90, 0))
self.assertAlmostEqual(a.asDegrees(), 45.0, places=13)
a = NormalizedAngle(UnitVector3d.Y(), UnitVector3d.Z())
self.assertAlmostEqual(a.asDegrees(), 90.0, places=13)
def test_center_and_dimensions(self):
b = Box.fromDegrees(-90, -45, 90, 45)
self.assertEqual(b.getCenter(), LonLat.fromDegrees(0, 0))
self.assertEqual(b.getWidth(), Angle.fromDegrees(180))
self.assertEqual(b.getHeight(), Angle.fromDegrees(90))
self.assertEqual(b.getLon().getA(), NormalizedAngle.fromDegrees(-90))
self.assertEqual(b.getLat().getB(), Angle.fromDegrees(45))
def testConstruction(self):
a1 = NormalizedAngle(1.0)
a2 = NormalizedAngle.fromRadians(1.0)
a3 = NormalizedAngle.fromDegrees(57.29577951308232)
self.assertEqual(a1, a2)
self.assertEqual(a1.asRadians(), 1.0)
self.assertEqual(a1, a3)
self.assertEqual(a1.asDegrees(), 57.29577951308232)
self.assertEqual(
NormalizedAngle.between(NormalizedAngle(0), NormalizedAngle(1)),
NormalizedAngle(1))
a = NormalizedAngle.center(NormalizedAngle(0), NormalizedAngle(1))
self.assertAlmostEqual(a.asRadians(), 0.5, places=15)
a = NormalizedAngle(LonLat.fromDegrees(45, 0),
LonLat.fromDegrees(90, 0))
self.assertAlmostEqual(a.asDegrees(), 45.0, places=13)
a = NormalizedAngle(UnitVector3d.Y(), UnitVector3d.Z())
self.assertAlmostEqual(a.asDegrees(), 90.0, places=13)
def testConstruction(self):
v = Vector3d(1, 1, 1)
u = UnitVector3d.orthogonalTo(v)
self.assertAlmostEqual(u.dot(v), 0.0, places=15)
a = UnitVector3d(1, 1, 1)
self.assertEqual(a, UnitVector3d(Vector3d(1, 1, 1)))
self.assertAlmostEqual(a.x(), math.sqrt(3.0) / 3.0, places=15)
self.assertAlmostEqual(a.y(), math.sqrt(3.0) / 3.0, places=15)
self.assertAlmostEqual(a.z(), math.sqrt(3.0) / 3.0, places=15)
b = UnitVector3d(Angle.fromDegrees(45), Angle.fromDegrees(45))
self.assertEqual(b, UnitVector3d(LonLat.fromDegrees(45, 45)))
self.assertAlmostEqual(b.x(), 0.5, places=15)
self.assertAlmostEqual(b.y(), 0.5, places=15)
self.assertAlmostEqual(b.z(), 0.5 * math.sqrt(2.0), places=15)
c = UnitVector3d.northFrom(b)
d = UnitVector3d(LonLat.fromDegrees(225, 45))
self.assertAlmostEqual(c.x(), d.x(), places=15)
self.assertAlmostEqual(c.y(), d.y(), places=15)
self.assertAlmostEqual(c.z(), d.z(), places=15)
def testConstruction(self):
v = Vector3d(1, 1, 1)
u = UnitVector3d.orthogonalTo(v)
self.assertAlmostEqual(u.dot(v), 0.0, places=15)
u = UnitVector3d(1, 1, 1)
self.assertEqual(u, UnitVector3d(Vector3d(1, 1, 1)))
self.assertAlmostEqual(u.x(), math.sqrt(3.0) / 3.0, places=15)
self.assertAlmostEqual(u.y(), math.sqrt(3.0) / 3.0, places=15)
self.assertAlmostEqual(u.z(), math.sqrt(3.0) / 3.0, places=15)
u = UnitVector3d(Angle.fromDegrees(45), Angle.fromDegrees(45))
self.assertEqual(u, UnitVector3d(LonLat.fromDegrees(45, 45)))
self.assertAlmostEqual(u.x(), 0.5, places=15)
self.assertAlmostEqual(u.y(), 0.5, places=15)
self.assertAlmostEqual(u.z(), 0.5 * math.sqrt(2.0), places=15)
u = UnitVector3d.northFrom(u.asVector3d())
w = UnitVector3d(LonLat.fromDegrees(225, 45))
self.assertAlmostEqual(u.x(), w.x(), places=15)
self.assertAlmostEqual(u.y(), w.y(), places=15)
self.assertAlmostEqual(u.z(), w.z(), places=15)
def test_relationships(self):
b1 = Box.fromDegrees(90, 0, 180, 45)
p = LonLat.fromDegrees(135, 10)
self.assertTrue(p in b1)
self.assertTrue(b1.contains(p))
b2 = Box.fromDegrees(135, 15, 135, 30)
self.assertTrue(b1.contains(b2))
self.assertTrue(b2.isWithin(b1))
b3 = Box.fromDegrees(0, -45, 90, 0)
u = UnitVector3d(1, 1, -1)
self.assertTrue(b1.intersects(b3))
self.assertTrue(u in b3)
self.assertTrue(b3.contains(u))
b4 = Box.fromDegrees(200, 10, 300, 20)
self.assertTrue(b1.isDisjointFrom(b4))
r = b1.relate(LonLat.fromDegrees(135, 10))
self.assertEqual(r, CONTAINS)
r = b4.relate(b1)
self.assertEqual(r, DISJOINT)
def test_construction(self):
b = Box(Box.allLongitudes(), Box.allLatitudes())
self.assertTrue(b.isFull())
b = Box.fromDegrees(-90, -45, 90, 45)
self.assertEqual(b, Box(b.getLon(), b.getLat()))
a = Box.fromRadians(-0.5 * math.pi, -0.25 * math.pi, 0.5 * math.pi,
0.25 * math.pi)
b = Box(LonLat.fromRadians(-0.5 * math.pi, -0.25 * math.pi),
LonLat.fromRadians(0.5 * math.pi, 0.25 * math.pi))
c = Box(LonLat.fromRadians(0, 0), Angle(0.5 * math.pi),
Angle(0.25 * math.pi))
d = c.clone()
self.assertEqual(a, b)
self.assertEqual(b, c)
self.assertEqual(c, d)
self.assertNotEqual(id(c), id(d))
b = Box()
self.assertTrue(b.isEmpty())
self.assertTrue(Box.empty().isEmpty())
self.assertTrue(Box.full().isFull())
def _region_to_columns(self, region: Optional[Region],
record: Dict[str, Any]) -> None:
if region is None:
return
# Get spatial parameters from the bounding circle.
circle = region.getBoundingCircle()
center = LonLat(circle.getCenter())
record["s_ra"] = center.getLon().asDegrees()
record["s_dec"] = center.getLat().asDegrees()
record["s_fov"] = circle.getOpeningAngle().asDegrees() * 2
if isinstance(region, ConvexPolygon):
poly = ["POLYGON ICRS"]
for vertex in region.getVertices():
lon_lat = LonLat(vertex)
poly += [
f"{lon_lat.getLon().asDegrees():.6f}",
f"{lon_lat.getLat().asDegrees():.6f}",
]
record["s_region"] = " ".join(poly)
else:
_LOG.warning(f"Unexpected region type: {type(region)}")
def testConstruction(self):
p = LonLat.fromDegrees(45, 45)
self.assertEqual(p, LonLat(NormalizedAngle.fromDegrees(45),
Angle.fromDegrees(45)))
u = UnitVector3d(p)
q = LonLat(u)
self.assertAlmostEqual(
p.getLon().asRadians(), q.getLon().asRadians(), places=13)
self.assertAlmostEqual(
p.getLat().asRadians(), q.getLat().asRadians(), places=13)
self.assertAlmostEqual(p.getLon().asRadians(),
LonLat.latitudeOf(u).asRadians(),
places=13)
self.assertAlmostEqual(p.getLon().asRadians(),
LonLat.longitudeOf(u).asRadians(),
places=13)
def testConstruction(self):
p = LonLat.fromDegrees(45, 45)
self.assertEqual(p, LonLat(NormalizedAngle.fromDegrees(45),
Angle.fromDegrees(45)))
u = UnitVector3d(p)
q = LonLat(u)
self.assertAlmostEqual(p.getLon().asRadians(), q.getLon().asRadians(), places=13)
self.assertAlmostEqual(p.getLat().asRadians(), q.getLat().asRadians(), places=13)
self.assertAlmostEqual(p.getLon().asRadians(), LonLat.latitudeOf(u).asRadians(), places=13)
self.assertAlmostEqual(p.getLon().asRadians(), LonLat.longitudeOf(u).asRadians(), places=13)
def test_expanding_and_clipping(self):
a = Box.fromDegrees(0, 0, 10, 10)
b = (a.expandedTo(LonLat.fromDegrees(20, 20)).expandedTo(
Box.fromDegrees(0, 0, 30, 10)).clippedTo(
Box.fromDegrees(10, 10, 15,
15)).clippedTo(LonLat.fromDegrees(11, 11)))
a.expandTo(LonLat.fromDegrees(20, 20))
a.expandTo(Box.fromDegrees(0, 0, 30, 10))
a.clipTo(Box.fromDegrees(10, 10, 15, 15))
a.clipTo(LonLat.fromDegrees(11, 11))
self.assertEqual(a, b)
self.assertEqual(a, LonLat.fromDegrees(11, 11))
a.clipTo(LonLat.fromDegrees(0, 0))
self.assertTrue(a.isEmpty())
def _filterDiaObjects(self, latest_objects, region):
"""Filter out objects from a catalog which are outside region.
Parameters
----------
latest_objects : `afw.table.BaseCatalog`
Catalog containing DiaObject records
region : `sphgem.Region`
Returns
-------
Filtered `afw.table.BaseCatalog` containing only records contained
in the region.
"""
mask = numpy.ndarray(len(latest_objects), dtype=bool)
for i, obj in enumerate(latest_objects):
# TODO: For now we use PPDB units (degrees), will have to be changed
# in case we adopt afw units.
lonLat = LonLat.fromRadians(obj['coord_ra'].asRadians(), obj['coord_dec'].asRadians())
dir_obj = UnitVector3d(lonLat)
mask[i] = region.contains(dir_obj)
return latest_objects.subset(mask)
def testString(self):
self.assertEqual(str(LonLat.fromRadians(1, 1)), "[1, 1]")
self.assertEqual(repr(LonLat.fromRadians(1, 1)), "LonLat.fromRadians(1.0, 1.0)")
def testComparisonOperators(self):
self.assertEqual(LonLat.fromDegrees(45, 45), LonLat.fromDegrees(45, 45))
self.assertNotEqual(LonLat.fromDegrees(0, 0), LonLat.fromDegrees(45, 45))
def run(self):
"""Run whole shebang.
"""
if self.args.config:
self.config.load(self.args.config)
if self.args.dump_config:
self.config.saveToStream(sys.stdout)
return 0
# instantiate db interface
db = Ppdb(self.config)
if self.config.divide > 1:
# check that we have reasonable MPI setup
if self.config.mp_mode == "mpi":
comm = MPI.COMM_WORLD
num_proc = comm.Get_size()
rank = comm.Get_rank()
node = MPI.Get_processor_name()
_LOG.info(COLOR_YELLOW + "MPI job rank=%d size=%d, node %s" + COLOR_RESET,
rank, num_proc, node)
num_tiles = self.config.divide**2
if num_proc != num_tiles:
raise ValueError(f"Number of MPI processes ({num_proc}) "
f"does not match number of tiles ({num_tiles})")
if rank != 0:
# run simple loop for all non-master processes
return self.run_mpi_tile_loop(db, comm)
# Initialize starting values from database visits table
last_visit = db.lastVisit()
if last_visit is not None:
start_visit_id = last_visit.visitId + 1
start_time = last_visit.visitTime + timedelta(seconds=self.config.interval)
else:
start_visit_id = self.config.start_visit_id
start_time = self.config.start_time_dt
if self.config.divide > 1:
_LOG.info("Will divide FOV into %dx%d regions", self.config.divide, self.config.divide)
_LOG.info("Max. number of ranges for pixelator: %d", self.config.htm_max_ranges)
# read sources file
_LOG.info("Start loading variable sources from %r", self.config.sources_file)
var_sources = numpy.load(self.config.sources_file)
_LOG.info("Finished loading variable sources, count = %s", len(var_sources))
# diaObjectId for last new DIA object, for variable sources we use their
# index as objectId, for transients we want to use ID outside that range
if last_visit is not None and last_visit.lastObjectId is not None:
self.lastObjectId = max(self.lastObjectId, last_visit.lastObjectId)
if self.lastObjectId < len(var_sources):
_LOG.error('next object id is too low: %s', self.lastObjectId)
return 1
_LOG.debug("lastObjectId: %s", self.lastObjectId)
# diaSourceId for last DIA source stored in database
if last_visit is not None and last_visit.lastSourceId is not None:
self.lastSourceId = max(self.lastSourceId, last_visit.lastSourceId)
_LOG.info("lastSourceId: %s", self.lastSourceId)
# loop over visits
visitTimes = _visitTimes(start_time, self.config.interval, self.args.num_visits)
for visit_id, dt in enumerate(visitTimes, start_visit_id):
if visit_id % 1000 == 0:
_LOG.info(COLOR_YELLOW + "+++ Start daily activities" + COLOR_RESET)
db.dailyJob()
_LOG.info(COLOR_YELLOW + "+++ Done with daily activities" + COLOR_RESET)
_LOG.info(COLOR_GREEN + "+++ Start processing visit %s at %s" + COLOR_RESET, visit_id, dt)
loop_timer = timer.Timer().start()
with timer.Timer("DIA"):
# point telescope in random southern direction
pointing_xyz = generators.rand_sphere_xyz(1, -1)[0]
pointing_v = UnitVector3d(pointing_xyz[0], pointing_xyz[1], pointing_xyz[2])
ra = LonLat.longitudeOf(pointing_v).asDegrees()
decl = LonLat.latitudeOf(pointing_v).asDegrees()
# sphgeom.Circle opening angle is actually a half of opening angle
region = Circle(pointing_v, Angle(self.config.FOV_rad/2))
_LOG.info("Pointing ra, decl = %s, %s; xyz = %s", ra, decl, pointing_xyz)
# Simulating difference image analysis
dia = DIA.DIA(pointing_xyz, self.config.FOV_rad, var_sources,
self.config.false_per_visit + self.config.transient_per_visit)
sources, indices = dia.makeSources()
_LOG.info("DIA generated %s sources", len(sources))
# assign IDs to transients
for i in range(len(sources)):
if indices[i] < 0:
self.lastObjectId += 1
indices[i] = self.lastObjectId
# print current database row counts, this takes long time
# so only do it once in a while
modu = 200 if visit_id <= 10000 else 1000
if visit_id % modu == 0:
counts = db.tableRowCount()
for tbl, count in sorted(counts.items()):
_LOG.info('%s row count: %s', tbl, count)
# numpy seems to do some multi-threaded stuff which "leaks" CPU cycles to the code below
# and it gets counted as resource usage in timers, add a short delay here so that threads
# finish and don't influence our timers below.
time.sleep(0.1)
if self.config.divide == 1:
# do it in-process
with timer.Timer("VisitProcessing"):
self.visit(db, visit_id, dt, region, sources, indices)
else:
if self.config.mp_mode == "fork":
tiles = geom.make_square_tiles(
self.config.FOV_rad, self.config.divide, self.config.divide, pointing_v)
with timer.Timer("VisitProcessing"):
# spawn subprocesses to handle individual tiles
children = []
for ix, iy, region in tiles:
# make sure lastSourceId is unique in in each process
self.lastSourceId += len(sources)
tile = (ix, iy)
pid = os.fork()
if pid == 0:
# child
self.visit(db, visit_id, dt, region, sources, indices, tile)
# stop here
sys.exit(0)
else:
_LOG.debug("Forked process %d for tile %s", pid, tile)
children.append(pid)
# wait until all children finish
for pid in children:
try:
pid, status = os.waitpid(pid, 0)
if status != 0:
_LOG.warning(COLOR_RED + "Child process PID=%s failed: %s" +
COLOR_RESET, pid, status)
except OSError as exc:
_LOG.warning(COLOR_RED + "wait failed for PID=%s: %s" +
COLOR_RESET, pid, exc)
elif self.config.mp_mode == "mpi":
tiles = geom.make_square_tiles(
self.config.FOV_rad, self.config.divide, self.config.divide, pointing_v, False)
_LOG.info("Split FOV into %d tiles for MPI", len(tiles))
# spawn subprocesses to handle individual tiles, special
# care needed for self.lastSourceId because it's
# propagated back from (0, 0)
lastSourceId = self.lastSourceId
tile_data = []
for ix, iy, region in tiles:
lastSourceId += len(sources)
tile = (ix, iy)
tile_data += [(visit_id, dt, region, sources, indices, tile, lastSourceId)]
# make sure lastSourceId is unique in in each process
with timer.Timer("VisitProcessing"):
_LOG.info("Scatter sources to %d tile processes", len(tile_data))
self.run_mpi_tile(db, MPI.COMM_WORLD, tile_data)
self.lastSourceId = lastSourceId
if not self.args.no_update:
# store last visit info
db.saveVisit(visit_id, dt)
_LOG.info(COLOR_BLUE + "--- Finished processing visit %s, time: %s" +
COLOR_RESET, visit_id, loop_timer)
# stop MPI slaves
if self.config.divide > 1 and self.config.mp_mode == "mpi":
_LOG.info("Stopping MPI tile processes")
tile_data = [None] * self.config.divide**2
self.run_mpi_tile(db, MPI.COMM_WORLD, tile_data)
return 0
def run(self) -> Optional[int]:
"""Run whole shebang.
"""
# load configurations
if self.args.app_config:
self.config.load(self.args.app_config)
if self.args.backend == "sql":
self.dbconfig = ApdbSqlConfig()
if self.args.config:
self.dbconfig.load(self.args.config)
elif self.args.backend == "cassandra":
self.dbconfig = ApdbCassandraConfig()
if self.args.config:
self.dbconfig.load(self.args.config)
if self.args.dump_config:
self.config.saveToStream(sys.stdout)
self.dbconfig.saveToStream(sys.stdout)
return 0
# instantiate db interface
db: Apdb
if self.args.backend == "sql":
db = ApdbSql(config=self.dbconfig)
elif self.args.backend == "cassandra":
db = ApdbCassandra(config=self.dbconfig)
visitInfoStore = VisitInfoStore(self.args.visits_file)
num_tiles = 1
if self.config.divide != 1:
tiles = geom.make_tiles(self.config.FOV_rad, self.config.divide)
num_tiles = len(tiles)
# check that we have reasonable MPI setup
if self.config.mp_mode == "mpi":
comm = MPI.COMM_WORLD
num_proc = comm.Get_size()
rank = comm.Get_rank()
node = MPI.Get_processor_name()
_LOG.info(
COLOR_YELLOW + "MPI job rank=%d size=%d, node %s" +
COLOR_RESET, rank, num_proc, node)
if num_proc != num_tiles:
raise ValueError(
f"Number of MPI processes ({num_proc}) "
f"does not match number of tiles ({num_tiles})")
if rank != 0:
# run simple loop for all non-master processes
self.run_mpi_tile_loop(db, comm)
return None
# Initialize starting values from database visits table
last_visit = visitInfoStore.lastVisit()
if last_visit is not None:
start_visit_id = last_visit.visitId + 1
nsec = last_visit.visitTime.nsecs(
DateTime.TAI) + self.config.interval * 1_000_000_000
start_time = DateTime(nsec, DateTime.TAI)
else:
start_visit_id = self.config.start_visit_id
start_time = self.config.start_time_dt
if self.config.divide > 1:
_LOG.info("Will divide FOV into %d regions", num_tiles)
src_read_period = self.config.src_read_period
src_read_visits = round(self.config.src_read_period *
self.config.src_read_duty_cycle)
_LOG.info("Will read sources for %d visits out of %d", src_read_visits,
src_read_period)
# read sources file
_LOG.info("Start loading variable sources from %r",
self.config.sources_file)
var_sources = numpy.load(self.config.sources_file)
_LOG.info("Finished loading variable sources, count = %s",
len(var_sources))
# diaObjectId for last new DIA object, for variable sources we use their
# index as objectId, for transients we want to use ID outside that range
if last_visit is not None and last_visit.lastObjectId is not None:
self.lastObjectId = max(self.lastObjectId, last_visit.lastObjectId)
if self.lastObjectId < len(var_sources):
_LOG.error('next object id is too low: %s', self.lastObjectId)
return 1
_LOG.debug("lastObjectId: %s", self.lastObjectId)
# diaSourceId for last DIA source stored in database
if last_visit is not None and last_visit.lastSourceId is not None:
self.lastSourceId = max(self.lastSourceId, last_visit.lastSourceId)
_LOG.info("lastSourceId: %s", self.lastSourceId)
# loop over visits
visitTimes = _visitTimes(start_time, self.config.interval,
self.args.num_visits)
for visit_id, dt in enumerate(visitTimes, start_visit_id):
if visit_id % 1000 == 0:
_LOG.info(COLOR_YELLOW + "+++ Start daily activities" +
COLOR_RESET)
db.dailyJob()
_LOG.info(COLOR_YELLOW + "+++ Done with daily activities" +
COLOR_RESET)
_LOG.info(
COLOR_GREEN + "+++ Start processing visit %s at %s" +
COLOR_RESET, visit_id, dt)
loop_timer = timer.Timer().start()
with timer.Timer("DIA"):
# point telescope in random southern direction
pointing_xyz = generators.rand_sphere_xyz(1, -1)[0]
pointing_v = UnitVector3d(pointing_xyz[0], pointing_xyz[1],
pointing_xyz[2])
ra = LonLat.longitudeOf(pointing_v).asDegrees()
decl = LonLat.latitudeOf(pointing_v).asDegrees()
# sphgeom.Circle opening angle is actually a half of opening angle
region = Circle(pointing_v, Angle(self.config.FOV_rad / 2))
_LOG.info("Pointing ra, decl = %s, %s; xyz = %s", ra, decl,
pointing_xyz)
# Simulating difference image analysis
dia = DIA.DIA(
pointing_xyz, self.config.FOV_rad, var_sources,
self.config.false_per_visit +
self.config.transient_per_visit)
sources, indices = dia.makeSources()
_LOG.info("DIA generated %s sources", len(sources))
# assign IDs to transients
for i in range(len(sources)):
if indices[i] < 0:
self.lastObjectId += 1
indices[i] = self.lastObjectId
# print current database row counts, this takes long time
# so only do it once in a while
modu = 200 if visit_id <= 10000 else 1000
if visit_id % modu == 0:
if hasattr(db, "tableRowCount"):
counts = db.tableRowCount() # type: ignore
for tbl, count in sorted(counts.items()):
_LOG.info('%s row count: %s', tbl, count)
# numpy seems to do some multi-threaded stuff which "leaks" CPU cycles to the code below
# and it gets counted as resource usage in timers, add a short delay here so that threads
# finish and don't influence our timers below.
time.sleep(0.1)
if self.config.divide == 1:
# do it in-process
with timer.Timer("VisitProcessing"):
self.visit(db, visit_id, dt, region, sources, indices)
else:
if self.config.mp_mode == "fork":
tiles = geom.make_tiles(self.config.FOV_rad,
self.config.divide, pointing_v)
with timer.Timer("VisitProcessing"):
# spawn subprocesses to handle individual tiles
children = []
for ix, iy, region in tiles:
# make sure lastSourceId is unique in in each process
self.lastSourceId += len(sources)
tile = (ix, iy)
pid = os.fork()
if pid == 0:
# child
self.visit(db, visit_id, dt, region, sources,
indices, tile)
# stop here
sys.exit(0)
else:
_LOG.debug("Forked process %d for tile %s",
pid, tile)
children.append(pid)
# wait until all children finish
for pid in children:
try:
pid, status = os.waitpid(pid, 0)
if status != 0:
_LOG.warning(
COLOR_RED +
"Child process PID=%s failed: %s" +
COLOR_RESET, pid, status)
except OSError as exc:
_LOG.warning(
COLOR_RED + "wait failed for PID=%s: %s" +
COLOR_RESET, pid, exc)
elif self.config.mp_mode == "mpi":
tiles = geom.make_tiles(self.config.FOV_rad,
self.config.divide, pointing_v)
_LOG.info("Split FOV into %d tiles for MPI", len(tiles))
# spawn subprocesses to handle individual tiles, special
# care needed for self.lastSourceId because it's
# propagated back from (0, 0)
lastSourceId = self.lastSourceId
tile_data = []
for ix, iy, region in tiles:
lastSourceId += len(sources)
tile = (ix, iy)
tile_data += [(visit_id, dt, region, sources, indices,
tile, lastSourceId)]
# make sure lastSourceId is unique in in each process
with timer.Timer("VisitProcessing"):
_LOG.info("Scatter sources to %d tile processes",
len(tile_data))
self.run_mpi_tile(db, MPI.COMM_WORLD, tile_data)
self.lastSourceId = lastSourceId
if not self.args.no_update:
# store last visit info
visitInfoStore.saveVisit(visit_id, dt, self.lastObjectId,
self.lastSourceId)
_LOG.info(
COLOR_BLUE + "--- Finished processing visit %s, time: %s" +
COLOR_RESET, visit_id, loop_timer)
# stop MPI slaves
if num_tiles > 1 and self.config.mp_mode == "mpi":
_LOG.info("Stopping MPI tile processes")
tile_data_stop = [None] * self.config.divide**2
self.run_mpi_tile(db, MPI.COMM_WORLD, tile_data_stop)
return 0
def test_dilation_and_erosion(self):
a = Box.fromRadians(0.5, -0.5, 1.5, 0.5)
b = a.dilatedBy(Angle(0.5), Angle(0.5)).erodedBy(Angle(1), Angle(1))
a.dilateBy(Angle(0.5), Angle(0.5)).erodeBy(Angle(1), Angle(1))
self.assertEqual(a, b)
self.assertEqual(a, LonLat.fromRadians(1, 0))
def testPickle(self):
p = LonLat.fromRadians(2, 1)
q = pickle.loads(pickle.dumps(p))
self.assertEqual(p, q)
def testComparisonOperators(self):
self.assertEqual(LonLat.fromDegrees(45, 45),
LonLat.fromDegrees(45, 45))
self.assertNotEqual(LonLat.fromDegrees(0, 0),
LonLat.fromDegrees(45, 45))
def testSkyMapDimensions(self):
"""Test involving only skymap dimensions, no joins to instrument"""
registry = self.registry
# need a bunch of dimensions and datasets for test, we want
# "abstract_filter" in the test so also have to add physical_filter
# dimensions
registry.addDimensionEntry("instrument", dict(instrument="DummyCam"))
registry.addDimensionEntry("physical_filter", dict(instrument="DummyCam",
physical_filter="dummy_r",
abstract_filter="r"))
registry.addDimensionEntry("physical_filter", dict(instrument="DummyCam",
physical_filter="dummy_i",
abstract_filter="i"))
registry.addDimensionEntry("skymap", dict(skymap="DummyMap", hash="sha!".encode("utf8")))
for tract in range(10):
registry.addDimensionEntry("tract", dict(skymap="DummyMap", tract=tract))
for patch in range(10):
registry.addDimensionEntry("patch", dict(skymap="DummyMap", tract=tract, patch=patch,
cell_x=0, cell_y=0,
region=Box(LonLat(NormalizedAngle(0.), Angle(0.)))))
# dataset types
collection = "test"
run = registry.makeRun(collection=collection)
storageClass = StorageClass("testDataset")
registry.storageClasses.registerStorageClass(storageClass)
calexpType = DatasetType(name="deepCoadd_calexp",
dimensions=registry.dimensions.extract(("skymap", "tract", "patch",
"abstract_filter")),
storageClass=storageClass)
registry.registerDatasetType(calexpType)
mergeType = DatasetType(name="deepCoadd_mergeDet",
dimensions=registry.dimensions.extract(("skymap", "tract", "patch")),
storageClass=storageClass)
registry.registerDatasetType(mergeType)
measType = DatasetType(name="deepCoadd_meas",
dimensions=registry.dimensions.extract(("skymap", "tract", "patch",
"abstract_filter")),
storageClass=storageClass)
registry.registerDatasetType(measType)
dimensions = registry.dimensions.empty.union(calexpType.dimensions, mergeType.dimensions,
measType.dimensions, implied=True)
# add pre-existing datasets
for tract in (1, 3, 5):
for patch in (2, 4, 6, 7):
dataId = dict(skymap="DummyMap", tract=tract, patch=patch)
registry.addDataset(mergeType, dataId=dataId, run=run)
for aFilter in ("i", "r"):
dataId = dict(skymap="DummyMap", tract=tract, patch=patch, abstract_filter=aFilter)
registry.addDataset(calexpType, dataId=dataId, run=run)
# with empty expression
builder = DataIdQueryBuilder.fromDimensions(registry, dimensions)
builder.requireDataset(calexpType, collections=[collection])
builder.requireDataset(mergeType, collections=[collection])
rows = list(builder.execute())
self.assertEqual(len(rows), 3*4*2) # 4 tracts x 4 patches x 2 filters
for dataId in rows:
self.assertCountEqual(dataId.keys(), ("skymap", "tract", "patch", "abstract_filter"))
self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5))
self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7))
self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r"))
# limit to 2 tracts and 2 patches
builder = DataIdQueryBuilder.fromDimensions(registry, dimensions)
builder.requireDataset(calexpType, collections=[collection])
builder.requireDataset(mergeType, collections=[collection])
builder.whereParsedExpression("tract IN (1, 5) AND patch.patch IN (2, 7)")
rows = list(builder.execute())
self.assertEqual(len(rows), 2*2*2) # 4 tracts x 4 patches x 2 filters
self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 5))
self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 7))
self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i", "r"))
# limit to single filter
builder = DataIdQueryBuilder.fromDimensions(registry, dimensions)
builder.requireDataset(calexpType, collections=[collection])
builder.requireDataset(mergeType, collections=[collection])
builder.whereParsedExpression("abstract_filter = 'i'")
rows = list(builder.execute())
self.assertEqual(len(rows), 3*4*1) # 4 tracts x 4 patches x 2 filters
self.assertCountEqual(set(dataId["tract"] for dataId in rows), (1, 3, 5))
self.assertCountEqual(set(dataId["patch"] for dataId in rows), (2, 4, 6, 7))
self.assertCountEqual(set(dataId["abstract_filter"] for dataId in rows), ("i",))
# expression excludes everything, specifying non-existing skymap is
# not a fatal error, it's operator error
builder = DataIdQueryBuilder.fromDimensions(registry, dimensions)
builder.requireDataset(calexpType, collections=[collection])
builder.requireDataset(mergeType, collections=[collection])
builder.whereParsedExpression("skymap = 'Mars'")
rows = list(builder.execute())
self.assertEqual(len(rows), 0)
def testString(self):
p = LonLat.fromRadians(1, 1)
self.assertEqual(str(p), '[1.0, 1.0]')
self.assertEqual(repr(p), 'LonLat.fromRadians(1.0, 1.0)')
self.assertEqual(p, eval(repr(p), dict(LonLat=LonLat)))
def in_region(obj: Any) -> bool:
lonLat = LonLat.fromDegrees(obj['ra'], obj['decl'])
dir_obj = UnitVector3d(lonLat)
return region.contains(dir_obj)