def test_storeObjectsLast(self):
"""Store and retrieve DiaObjects using DiaObjectLast table."""
# don't care about sources.
config = ApdbConfig(db_url="sqlite:///",
isolation_level="READ_UNCOMMITTED",
dia_object_index="last_object_table",
object_last_replace=True)
apdb = Apdb(config)
apdb.makeSchema()
pixel_ranges = _makePixelRanges()
visit_time = datetime.datetime.now()
# make afw catalog with Objects
if self.use_pandas:
catalog = _makeObjectCatalogPandas(pixel_ranges)
else:
catalog = _makeObjectCatalog(pixel_ranges)
# store catalog
apdb.storeDiaObjects(catalog, visit_time)
# read it back and check sizes
res = apdb.getDiaObjects(pixel_ranges, return_pandas=self.use_pandas)
self._assertCatalog(res, len(catalog), type=self.data_type)
def test_emptyGetsBaseline(self):
"""Test for getting data from empty database.
All get() methods should return empty results, only useful for
checking that code is not broken.
"""
# use non-zero months for Forced/Source fetching
config = ApdbConfig(db_url="sqlite:///",
isolation_level="READ_UNCOMMITTED",
read_sources_months=12,
read_forced_sources_months=12)
apdb = Apdb(config)
apdb.makeSchema()
pixel_ranges = _makePixelRanges()
visit_time = datetime.datetime.now()
# get objects by region
res = apdb.getDiaObjects(pixel_ranges, return_pandas=self.use_pandas)
self._assertCatalog(res, 0, type=self.data_type)
# get sources by region
res = apdb.getDiaSourcesInRegion(pixel_ranges,
visit_time,
return_pandas=self.use_pandas)
self._assertCatalog(res, 0, type=self.data_type)
# get sources by object ID, empty object list, should return None
res = apdb.getDiaSources([], visit_time, return_pandas=self.use_pandas)
self.assertIs(res, None)
# get sources by object ID, non-empty object list
res = apdb.getDiaSources([1, 2, 3],
visit_time,
return_pandas=self.use_pandas)
self._assertCatalog(res, 0, type=self.data_type)
# get forced sources by object ID, empty object list
res = apdb.getDiaForcedSources([],
visit_time,
return_pandas=self.use_pandas)
self.assertIs(res, None)
# get sources by object ID, non-empty object list
res = apdb.getDiaForcedSources([1, 2, 3],
visit_time,
return_pandas=self.use_pandas)
self._assertCatalog(res, 0, type=self.data_type)
def test_emptyGetsObjectLast(self):
"""Test for getting DiaObjects from empty database using DiaObjectLast
table.
All get() methods should return empty results, only useful for
checking that code is not broken.
"""
# don't care about sources.
config = ApdbConfig(db_url="sqlite:///",
isolation_level="READ_UNCOMMITTED",
dia_object_index="last_object_table")
apdb = Apdb(config)
apdb.makeSchema()
pixel_ranges = _makePixelRanges()
# get objects by region
res = apdb.getDiaObjects(pixel_ranges, return_pandas=self.use_pandas)
self._assertCatalog(res, 0, type=self.data_type)
def _roundTripThroughApdb(objects, sources, forcedSources, dateTime):
"""Run object and source catalogs through the Apdb to get the correct
table schemas.
Parameters
----------
objects : `pandas.DataFrame`
Set of test DiaObjects to round trip.
sources : `pandas.DataFrame`
Set of test DiaSources to round trip.
forcedSources : `pandas.DataFrame`
Set of test DiaForcedSources to round trip.
dateTime : `datetime.datetime`
Time for the Apdb.
Returns
-------
objects : `pandas.DataFrame`
Round tripped objects.
sources : `pandas.DataFrame`
Round tripped sources.
"""
tmpFile = tempfile.NamedTemporaryFile()
apdbConfig = ApdbConfig()
apdbConfig.db_url = "sqlite:///" + tmpFile.name
apdbConfig.isolation_level = "READ_UNCOMMITTED"
apdbConfig.dia_object_index = "baseline"
apdbConfig.dia_object_columns = []
apdbConfig.schema_file = _data_file_name("apdb-schema.yaml", "dax_apdb")
apdbConfig.column_map = _data_file_name("apdb-ap-pipe-afw-map.yaml",
"ap_association")
apdbConfig.extra_schema_file = _data_file_name(
"apdb-ap-pipe-schema-extra.yaml", "ap_association")
apdb = Apdb(config=apdbConfig,
afw_schemas=dict(DiaObject=make_dia_object_schema(),
DiaSource=make_dia_source_schema()))
apdb.makeSchema()
minId = objects["pixelId"].min()
maxId = objects["pixelId"].max()
diaObjects = apdb.getDiaObjects([[minId, maxId + 1]],
return_pandas=True).append(objects)
diaSources = apdb.getDiaSources(np.unique(objects["diaObjectId"]),
dateTime,
return_pandas=True).append(sources)
diaForcedSources = apdb.getDiaForcedSources(
np.unique(objects["diaObjectId"]), dateTime,
return_pandas=True).append(forcedSources)
apdb.storeDiaSources(diaSources)
apdb.storeDiaForcedSources(diaForcedSources)
apdb.storeDiaObjects(diaObjects, dateTime)
diaObjects = apdb.getDiaObjects([[minId, maxId + 1]], return_pandas=True)
diaSources = apdb.getDiaSources(np.unique(diaObjects["diaObjectId"]),
dateTime,
return_pandas=True)
diaForcedSources = apdb.getDiaForcedSources(np.unique(
diaObjects["diaObjectId"]),
dateTime,
return_pandas=True)
diaObjects.set_index("diaObjectId", drop=False, inplace=True)
diaSources.set_index(["diaObjectId", "filterName", "diaSourceId"],
drop=False,
inplace=True)
diaForcedSources.set_index(["diaObjectId"], drop=False, inplace=True)
return (diaObjects, diaSources, diaForcedSources)
def visit(self,
db: Apdb,
visit_id: int,
dt: DateTime,
region: Region,
sources: numpy.ndarray,
indices: numpy.ndarray,
tile: Optional[Tuple[int, int]] = None) -> None:
"""AP processing of a single visit (with known sources)
Parameters
----------
db : `Apdb`
APDB interface
visit_id : `int`
Visit ID.
dt : `DateTime`
Time of visit
region : `sphgeom.Region`
Region, could be the whole FOV (Circle) or small piece of it
sources : `numpy.array`
Array of xyz coordinates of sources, this has all visit sources,
not only current tile
indices : `numpy.array`
array of indices of sources, 1-dim ndarray, transient sources
have negative indices
tile : `tuple`
tile position (x, y)
"""
name = ""
if tile is not None:
name = "tile={}x{} ".format(*tile)
src_read_period = self.config.src_read_period
src_read_visits = round(self.config.src_read_period *
self.config.src_read_duty_cycle)
do_read_src = visit_id % src_read_period < src_read_visits
# make a mask
for i in range(len(sources)):
xyz = sources[i]
if not region.contains(UnitVector3d(xyz[0], xyz[1], xyz[2])):
indices[i] = _OUTSIDER
with timer.Timer(name + "Objects-read"):
# Retrieve DiaObjects (latest versions) from database for matching,
# this will produce wider coverage so further filtering is needed
latest_objects = db.getDiaObjects(region)
_LOG.info(name + 'database found %s objects',
_nrows(latest_objects))
# filter database objects to a mask
latest_objects = self._filterDiaObjects(latest_objects, region)
_LOG.info(name + 'after filtering %s objects',
_nrows(latest_objects))
with timer.Timer(name + "S2O-matching"):
# create all new DiaObjects
objects = self._makeDiaObjects(sources, indices, dt)
# make all sources
srcs = self._makeDiaSources(sources, indices, dt, visit_id)
# do forced photometry (can extends objects)
fsrcs, objects = self._forcedPhotometry(objects, latest_objects,
dt, visit_id)
if self.config.fill_empty_fields:
self._fillRandomData(objects, ApdbTables.DiaObject, db)
self._fillRandomData(srcs, ApdbTables.DiaSource, db)
self._fillRandomData(fsrcs, ApdbTables.DiaForcedSource, db)
if do_read_src:
with timer.Timer(name + "Source-read"):
latest_objects_ids = list(latest_objects['diaObjectId'])
read_srcs = db.getDiaSources(region, latest_objects_ids, dt)
_LOG.info(name + 'database found %s sources',
_nrows(read_srcs))
read_srcs = db.getDiaForcedSources(region, latest_objects_ids,
dt)
_LOG.info(name + 'database found %s forced sources',
_nrows(read_srcs))
else:
_LOG.info("skipping reading of sources for this visit")
if not self.args.no_update:
with timer.Timer(name + "L1-store"):
# store new versions of objects
_LOG.info(name + 'will store %d Objects', len(objects))
_LOG.info(name + 'will store %d Sources', len(srcs))
_LOG.info(name + 'will store %d ForcedSources', len(fsrcs))
db.store(dt, objects, srcs, fsrcs)