def __init__(self, db_file):
"""Create similar configuration for tasks ad in ap_pipe.
"""
self.log = Log.getLogger("RunAssociation")
self.apdbConfig = ApdbConfig()
self.apdbConfig.db_url = "sqlite:///" + db_file
self.apdbConfig.isolation_level = "READ_UNCOMMITTED"
self.apdbConfig.dia_object_index = "baseline"
self.apdbConfig.dia_object_columns = []
self.apdbConfig.connection_timeout = 240
self.apdbConfig.schema_file = _data_file_name("apdb-schema.yaml",
"dax_apdb")
self.apdbConfig.column_map = _data_file_name(
"apdb-ap-pipe-afw-map.yaml", "ap_association")
self.apdbConfig.extra_schema_file = _data_file_name(
"apdb-ap-pipe-schema-extra.yaml", "ap_association")
self.apdb = Apdb(config=self.apdbConfig,
afw_schemas=dict(DiaObject=make_dia_object_schema(),
DiaSource=make_dia_source_schema()))
# apdb.makeSchema()
self.differencerConfig = ImageDifferenceConfig()
# Schema is different if we do decorrelation
self.differencerConfig.doDecorrelation = True
self.differencerSchema = ImageDifferenceTask(
config=self.differencerConfig).schema
self.diaSourceDpddifier = MapDiaSourceTask(
inputSchema=self.differencerSchema)
self.associator = AssociationTask()
self.diffType = "deep"
def test_storeForcedSources(self):
"""Store and retrieve DiaForcedSources."""
config = ApdbConfig(db_url="sqlite:///",
isolation_level="READ_UNCOMMITTED",
read_sources_months=12,
read_forced_sources_months=12)
apdb = Apdb(config)
apdb.makeSchema()
pixel_ranges = _makePixelRanges()
visit_time = datetime.datetime.now()
# have to store Objects first
if self.use_pandas:
objects = _makeObjectCatalogPandas(pixel_ranges)
catalog, oids = _makeForcedSourceCatalogPandas(objects)
else:
objects = _makeObjectCatalog(pixel_ranges)
catalog, oids = _makeForcedSourceCatalog(objects)
apdb.storeDiaObjects(objects, visit_time)
# save them
apdb.storeDiaForcedSources(catalog)
# read it back and check sizes
res = apdb.getDiaForcedSources(oids,
visit_time,
return_pandas=self.use_pandas)
self._assertCatalog(res, len(catalog), type=self.data_type)
def test_storeObjectsLast(self):
"""Store and retrieve DiaObjects using DiaObjectLast table."""
# don't care about sources.
config = ApdbConfig(db_url="sqlite:///",
isolation_level="READ_UNCOMMITTED",
dia_object_index="last_object_table",
object_last_replace=True)
apdb = Apdb(config)
apdb.makeSchema()
pixel_ranges = _makePixelRanges()
visit_time = datetime.datetime.now()
# make afw catalog with Objects
if self.use_pandas:
catalog = _makeObjectCatalogPandas(pixel_ranges)
else:
catalog = _makeObjectCatalog(pixel_ranges)
# store catalog
apdb.storeDiaObjects(catalog, visit_time)
# read it back and check sizes
res = apdb.getDiaObjects(pixel_ranges, return_pandas=self.use_pandas)
self._assertCatalog(res, len(catalog), type=self.data_type)
def test_makeSchema(self):
"""Test for making an instance of Apdb using in-memory sqlite engine.
"""
# sqlite does not support default READ_COMMITTED, for in-memory
# database have to use connection pool
config = ApdbConfig(db_url="sqlite://",
isolation_level="READ_UNCOMMITTED")
apdb = Apdb(config)
# the essence of a test here is that there are no exceptions.
apdb.makeSchema()
def makeDbInfo(cls):
"""Return an object that can be passed as input to an `ApdbMetricTask`.
This method is intended for generic tests that simply need to call
``run`` on some valid input. If a test depends on specific input, it
should create that input directly.
The default implementation creates a `~lsst.pex.config.Config` that
will be accepted by `~lsst.verify.tasks.DirectApdbLoader`. Test suites
that use a different loader should override this method.
"""
return ApdbConfig()
def test_emptyGetsBaseline(self):
"""Test for getting data from empty database.
All get() methods should return empty results, only useful for
checking that code is not broken.
"""
# use non-zero months for Forced/Source fetching
config = ApdbConfig(db_url="sqlite:///",
isolation_level="READ_UNCOMMITTED",
read_sources_months=12,
read_forced_sources_months=12)
apdb = Apdb(config)
apdb.makeSchema()
pixel_ranges = _makePixelRanges()
visit_time = datetime.datetime.now()
# get objects by region
res = apdb.getDiaObjects(pixel_ranges, return_pandas=self.use_pandas)
self._assertCatalog(res, 0, type=self.data_type)
# get sources by region
res = apdb.getDiaSourcesInRegion(pixel_ranges,
visit_time,
return_pandas=self.use_pandas)
self._assertCatalog(res, 0, type=self.data_type)
# get sources by object ID, empty object list, should return None
res = apdb.getDiaSources([], visit_time, return_pandas=self.use_pandas)
self.assertIs(res, None)
# get sources by object ID, non-empty object list
res = apdb.getDiaSources([1, 2, 3],
visit_time,
return_pandas=self.use_pandas)
self._assertCatalog(res, 0, type=self.data_type)
# get forced sources by object ID, empty object list
res = apdb.getDiaForcedSources([],
visit_time,
return_pandas=self.use_pandas)
self.assertIs(res, None)
# get sources by object ID, non-empty object list
res = apdb.getDiaForcedSources([1, 2, 3],
visit_time,
return_pandas=self.use_pandas)
self._assertCatalog(res, 0, type=self.data_type)
def test_emptyGetsObjectLast(self):
"""Test for getting DiaObjects from empty database using DiaObjectLast
table.
All get() methods should return empty results, only useful for
checking that code is not broken.
"""
# don't care about sources.
config = ApdbConfig(db_url="sqlite:///",
isolation_level="READ_UNCOMMITTED",
dia_object_index="last_object_table")
apdb = Apdb(config)
apdb.makeSchema()
pixel_ranges = _makePixelRanges()
# get objects by region
res = apdb.getDiaObjects(pixel_ranges, return_pandas=self.use_pandas)
self._assertCatalog(res, 0, type=self.data_type)
def _dummyApdbConfig():
config = ApdbConfig()
config.db_url = "sqlite://" # in-memory DB
config.isolation_level = "READ_UNCOMMITTED"
return config
def _roundTripThroughApdb(objects, sources, forcedSources, dateTime):
"""Run object and source catalogs through the Apdb to get the correct
table schemas.
Parameters
----------
objects : `pandas.DataFrame`
Set of test DiaObjects to round trip.
sources : `pandas.DataFrame`
Set of test DiaSources to round trip.
forcedSources : `pandas.DataFrame`
Set of test DiaForcedSources to round trip.
dateTime : `datetime.datetime`
Time for the Apdb.
Returns
-------
objects : `pandas.DataFrame`
Round tripped objects.
sources : `pandas.DataFrame`
Round tripped sources.
"""
tmpFile = tempfile.NamedTemporaryFile()
apdbConfig = ApdbConfig()
apdbConfig.db_url = "sqlite:///" + tmpFile.name
apdbConfig.isolation_level = "READ_UNCOMMITTED"
apdbConfig.dia_object_index = "baseline"
apdbConfig.dia_object_columns = []
apdbConfig.schema_file = _data_file_name("apdb-schema.yaml", "dax_apdb")
apdbConfig.column_map = _data_file_name("apdb-ap-pipe-afw-map.yaml",
"ap_association")
apdbConfig.extra_schema_file = _data_file_name(
"apdb-ap-pipe-schema-extra.yaml", "ap_association")
apdb = Apdb(config=apdbConfig,
afw_schemas=dict(DiaObject=make_dia_object_schema(),
DiaSource=make_dia_source_schema()))
apdb.makeSchema()
minId = objects["pixelId"].min()
maxId = objects["pixelId"].max()
diaObjects = apdb.getDiaObjects([[minId, maxId + 1]],
return_pandas=True).append(objects)
diaSources = apdb.getDiaSources(np.unique(objects["diaObjectId"]),
dateTime,
return_pandas=True).append(sources)
diaForcedSources = apdb.getDiaForcedSources(
np.unique(objects["diaObjectId"]), dateTime,
return_pandas=True).append(forcedSources)
apdb.storeDiaSources(diaSources)
apdb.storeDiaForcedSources(diaForcedSources)
apdb.storeDiaObjects(diaObjects, dateTime)
diaObjects = apdb.getDiaObjects([[minId, maxId + 1]], return_pandas=True)
diaSources = apdb.getDiaSources(np.unique(diaObjects["diaObjectId"]),
dateTime,
return_pandas=True)
diaForcedSources = apdb.getDiaForcedSources(np.unique(
diaObjects["diaObjectId"]),
dateTime,
return_pandas=True)
diaObjects.set_index("diaObjectId", drop=False, inplace=True)
diaSources.set_index(["diaObjectId", "filterName", "diaSourceId"],
drop=False,
inplace=True)
diaForcedSources.set_index(["diaObjectId"], drop=False, inplace=True)
return (diaObjects, diaSources, diaForcedSources)