def test_score_and_match(self):
"""Test association between a set of sources and an existing
DIAObjectCollection.
"""
assoc_task = AssociationTask()
score_struct = assoc_task.score(self.diaObjects,
self.diaSourceZeroScatter,
1.0 * geom.arcseconds)
self.assertFalse(np.isfinite(score_struct.scores[0]))
for src_idx in range(1, len(self.diaSources)):
# Our scores should be extremely close to 0 but not exactly so due
# to machine noise.
self.assertAlmostEqual(score_struct.scores[src_idx],
0.0,
places=16)
# After matching each DIAObject should now contain 2 DIASources
# except the last DIAObject in this collection which should be
# newly created during the matching step and contain only one
# DIASource.
match_result = assoc_task.match(self.diaObjects, self.diaSources,
score_struct)
self.assertEqual(match_result.nUpdatedDiaObjects, 4)
self.assertEqual(match_result.nUnassociatedDiaObjects, 1)
def __init__(self, db_file):
"""Create similar configuration for tasks ad in ap_pipe.
"""
self.log = Log.getLogger("RunAssociation")
self.apdbConfig = ApdbConfig()
self.apdbConfig.db_url = "sqlite:///" + db_file
self.apdbConfig.isolation_level = "READ_UNCOMMITTED"
self.apdbConfig.dia_object_index = "baseline"
self.apdbConfig.dia_object_columns = []
self.apdbConfig.connection_timeout = 240
self.apdbConfig.schema_file = _data_file_name("apdb-schema.yaml",
"dax_apdb")
self.apdbConfig.column_map = _data_file_name(
"apdb-ap-pipe-afw-map.yaml", "ap_association")
self.apdbConfig.extra_schema_file = _data_file_name(
"apdb-ap-pipe-schema-extra.yaml", "ap_association")
self.apdb = Apdb(config=self.apdbConfig,
afw_schemas=dict(DiaObject=make_dia_object_schema(),
DiaSource=make_dia_source_schema()))
# apdb.makeSchema()
self.differencerConfig = ImageDifferenceConfig()
# Schema is different if we do decorrelation
self.differencerConfig.doDecorrelation = True
self.differencerSchema = ImageDifferenceTask(
config=self.differencerConfig).schema
self.diaSourceDpddifier = MapDiaSourceTask(
inputSchema=self.differencerSchema)
self.associator = AssociationTask()
self.diffType = "deep"
def test_remove_nan_dia_sources(self):
"""Test removing DiaSources with NaN locations.
"""
self.diaSources.loc[2, "ra"] = np.nan
self.diaSources.loc[3, "decl"] = np.nan
self.diaSources.loc[4, "ra"] = np.nan
self.diaSources.loc[4, "decl"] = np.nan
assoc_task = AssociationTask()
out_dia_sources = assoc_task.check_dia_source_radec(self.diaSources)
self.assertEqual(len(out_dia_sources), len(self.diaSources) - 3)
def test_run_no_existing_objects(self):
"""Test the run method with a completely empty database.
"""
assocTask = AssociationTask()
results = assocTask.run(
self.diaSources,
pd.DataFrame(columns=["ra", "decl", "diaObjectId"]))
self.assertEqual(results.nUpdatedDiaObjects, 0)
self.assertEqual(results.nUnassociatedDiaObjects, 0)
self.assertEqual(len(results.matchedDiaSources), 0)
self.assertTrue(np.all(results.unAssocDiaSources["diaObjectId"] == 0))
def test_associate_sources(self):
"""Test the performance of the associate_sources method in
AssociationTask.
"""
assoc_task = AssociationTask()
assoc_result = assoc_task.associate_sources(self.diaObjects,
self.diaSources)
for test_obj_id, expected_obj_id in zip(
assoc_result.diaSources["diaObjectId"].to_numpy(),
[0, 1, 2, 3, 4]):
self.assertEqual(test_obj_id, expected_obj_id)
def test_run(self):
"""Test the full task by associating a set of diaSources to
existing diaObjects.
"""
assocTask = AssociationTask()
results = assocTask.run(self.diaSources, self.diaObjects)
self.assertEqual(results.nUpdatedDiaObjects, len(self.diaObjects) - 1)
self.assertEqual(results.nUnassociatedDiaObjects, 1)
self.assertEqual(len(results.matchedDiaSources),
len(self.diaObjects) - 1)
self.assertEqual(len(results.unAssocDiaSources), 1)
for test_obj_id, expected_obj_id in zip(
results.matchedDiaSources["diaObjectId"].to_numpy(),
[1, 2, 3, 4]):
self.assertEqual(test_obj_id, expected_obj_id)
for test_obj_id, expected_obj_id in zip(
results.unAssocDiaSources["diaObjectId"].to_numpy(), [0]):
self.assertEqual(test_obj_id, expected_obj_id)
def test_remove_nan_dia_sources(self):
n_sources = 6
dia_sources = create_test_points_pandas(
point_locs_deg=[
[0.04 * (src_idx + 1),
0.04 * (src_idx + 1)]
for src_idx in range(n_sources)],
start_id=0,
scatter_arcsec=-1)
dia_sources.rename(columns={"coord_ra": "ra",
"coord_dec": "decl",
"id": "diaSourceId"},
inplace=True)
dia_sources.loc[2, "ra"] = np.nan
dia_sources.loc[3, "decl"] = np.nan
dia_sources.loc[4, "ra"] = np.nan
dia_sources.loc[4, "decl"] = np.nan
assoc_task = AssociationTask()
out_dia_sources = assoc_task.check_dia_source_radec(dia_sources)
self.assertEqual(len(out_dia_sources), n_sources - 3)
def test_associate_sources(self):
"""Test the performance of the associate_sources method in
AssociationTask.
"""
n_objects = 5
dia_objects = create_test_points_pandas(
point_locs_deg=[[0.04 * obj_idx, 0.04 * obj_idx]
for obj_idx in range(n_objects)],
start_id=0,
schema=self.dia_object_schema,
scatter_arcsec=-1,)
dia_objects.rename(columns={"coord_ra": "ra",
"coord_dec": "decl",
"id": "diaObjectId"},
inplace=True)
n_sources = 5
dia_sources = create_test_points_pandas(
point_locs_deg=[
[0.04 * (src_idx + 1),
0.04 * (src_idx + 1)]
for src_idx in range(n_sources)],
start_id=n_objects,
scatter_arcsec=0.1)
dia_sources.rename(columns={"coord_ra": "ra",
"coord_dec": "decl",
"id": "diaSourceId"},
inplace=True)
assoc_task = AssociationTask()
assoc_result = assoc_task.associate_sources(
dia_objects, dia_sources)
for test_obj_id, expected_obj_id in zip(
assoc_result.associated_dia_object_ids,
[1, 2, 3, 4, 9]):
self.assertEqual(test_obj_id, expected_obj_id)
def test_score_and_match(self):
"""Test association between a set of sources and an existing
DIAObjectCollection.
This also tests that a DIASource that can't be associated within
tolerance is appended to the DIAObjectCollection as a new
DIAObject.
"""
assoc_task = AssociationTask()
# Create a set of DIAObjects that contain only one DIASource
n_objects = 5
dia_objects = create_test_points_pandas(
point_locs_deg=[[0.04 * obj_idx, 0.04 * obj_idx]
for obj_idx in range(n_objects)],
start_id=0,
schema=self.dia_object_schema,
scatter_arcsec=-1,)
dia_objects.rename(columns={"coord_ra": "ra",
"coord_dec": "decl",
"id": "diaObjectId"},
inplace=True)
n_sources = 5
dia_sources = create_test_points_pandas(
point_locs_deg=[
[0.04 * (src_idx + 1),
0.04 * (src_idx + 1)]
for src_idx in range(n_sources)],
start_id=n_objects,
scatter_arcsec=-1)
dia_sources.rename(columns={"coord_ra": "ra",
"coord_dec": "decl",
"id": "diaSourceId"},
inplace=True)
score_struct = assoc_task.score(dia_objects,
dia_sources,
1.0 * geom.arcseconds)
self.assertFalse(np.isfinite(score_struct.scores[-1]))
for src_idx in range(4):
# Our scores should be extremely close to 0 but not exactly so due
# to machine noise.
self.assertAlmostEqual(score_struct.scores[src_idx], 0.0,
places=16)
# After matching each DIAObject should now contain 2 DIASources
# except the last DIAObject in this collection which should be
# newly created during the matching step and contain only one
# DIASource.
match_result = assoc_task.match(dia_objects, dia_sources, score_struct)
updated_ids = match_result.associated_dia_object_ids
self.assertEqual(len(updated_ids), 5)
self.assertEqual(match_result.n_updated_dia_objects, 4)
self.assertEqual(match_result.n_new_dia_objects, 1)
self.assertEqual(match_result.n_unassociated_dia_objects, 1)
# Test updating all DiaObjects
n_objects = 4
dia_objects = create_test_points_pandas(
point_locs_deg=[[0.04 * obj_idx, 0.04 * obj_idx]
for obj_idx in range(n_objects)],
start_id=0,
schema=self.dia_object_schema,
scatter_arcsec=-1,)
dia_objects.rename(columns={"coord_ra": "ra",
"coord_dec": "decl",
"id": "diaObjectId"},
inplace=True)
n_sources = 4
dia_sources = create_test_points_pandas(
point_locs_deg=[
[0.04 * src_idx,
0.04 * src_idx]
for src_idx in range(n_sources)],
start_id=n_objects,
scatter_arcsec=-1)
dia_sources.rename(columns={"coord_ra": "ra",
"coord_dec": "decl",
"id": "diaSourceId"},
inplace=True)
score_struct = assoc_task.score(dia_objects[1:],
dia_sources[:-1],
1.0 * geom.arcseconds)
match_result = assoc_task.match(dia_objects, dia_sources, score_struct)
updated_ids = match_result.associated_dia_object_ids
self.assertEqual(len(updated_ids), 4)
def _run_association_and_retrieve_objects(self, create_objects=False):
"""Convenience method for testing the Association run method.
Parameters
----------
create_objects : `bool`
Boolean specifying if seed DIAObjects and DIASources should be
inserted into the database before association.
Return
------
dia_objects : `lsst.afw.table.SourceCatalog`
Final set of DIAObjects to be tested.
"""
if create_objects:
diaObjects, diaSourceHistory = \
self._create_dia_objects_and_sources()
else:
diaObjects = pd.DataFrame(columns=["diaObjectId"])
diaSourceHistory = pd.DataFrame(columns=["diaObjectId",
"filterName",
"diaSourceId"])
diaObjects.set_index("diaObjectId",
inplace=True,
drop=False)
diaSourceHistory.set_index(["diaObjectId",
"filterName",
"diaSourceId"],
inplace=True,
drop=False)
source_centers = [
[self.wcs.pixelToSky(idx, idx).getRa().asDegrees(),
self.wcs.pixelToSky(idx, idx).getDec().asDegrees()]
for idx in np.linspace(1, 1000, 10)[1:]]
dia_sources = create_test_points(
point_locs_deg=source_centers,
start_id=10,
scatter_arcsec=-1)
for dia_source in dia_sources:
self._set_source_values(
dia_source=dia_source,
flux=10000,
fluxErr=100,
# TODO DM-27170: fix this [0] workaround which gets a
# single character representation of the band.
filterName=self.exposure.getFilter().getCanonicalName()[0],
filterId=self.exposure.getFilter().getId(),
ccdVisitId=self.exposure.getInfo().getVisitInfo().getExposureId(),
midPointTai=self.exposure.getInfo().getVisitInfo().getDate().get(system=dafBase.DateTime.MJD))
assoc_task = AssociationTask()
diaSources = dia_sources.asAstropy().to_pandas()
diaSources.rename(columns={"coord_ra": "ra",
"coord_dec": "decl",
"id": "diaSourceId",
"parent": "parentDiaSourceId"},
inplace=True)
diaSources["ra"] = np.degrees(diaSources["ra"])
diaSources["decl"] = np.degrees(diaSources["decl"])
if len(diaObjects) == 0:
diaSourceHistory = pd.DataFrame(columns=["diaObjectId",
"filterName",
"diaSourceId"])
diaSourceHistory.set_index(
["diaObjectId", "filterName", "diaSourceId"],
drop=False,
inplace=True)
results = assoc_task.run(diaSources,
diaObjects,
diaSourceHistory)
return results.diaObjects
class RunAssociation:
def __init__(self, db_file):
"""Create similar configuration for tasks ad in ap_pipe.
"""
self.log = Log.getLogger("RunAssociation")
self.apdbConfig = ApdbConfig()
self.apdbConfig.db_url = "sqlite:///" + db_file
self.apdbConfig.isolation_level = "READ_UNCOMMITTED"
self.apdbConfig.dia_object_index = "baseline"
self.apdbConfig.dia_object_columns = []
self.apdbConfig.connection_timeout = 240
self.apdbConfig.schema_file = _data_file_name("apdb-schema.yaml",
"dax_apdb")
self.apdbConfig.column_map = _data_file_name(
"apdb-ap-pipe-afw-map.yaml", "ap_association")
self.apdbConfig.extra_schema_file = _data_file_name(
"apdb-ap-pipe-schema-extra.yaml", "ap_association")
self.apdb = Apdb(config=self.apdbConfig,
afw_schemas=dict(DiaObject=make_dia_object_schema(),
DiaSource=make_dia_source_schema()))
# apdb.makeSchema()
self.differencerConfig = ImageDifferenceConfig()
# Schema is different if we do decorrelation
self.differencerConfig.doDecorrelation = True
self.differencerSchema = ImageDifferenceTask(
config=self.differencerConfig).schema
self.diaSourceDpddifier = MapDiaSourceTask(
inputSchema=self.differencerSchema)
self.associator = AssociationTask()
self.diffType = "deep"
def procOneExposure(self, dataId):
"""Run AssociationTask on the exposure selected by ``dataId``.
"""
try:
catalog = self.butler.get(self.diffType + "Diff_diaSrc",
dataId=dataId)
diffim = self.butler.get(self.diffType + "Diff_differenceExp",
dataId=dataId)
except butlerExceptions.NoResults:
self.log.info("Data does not exist %s", dataId)
return
dia_sources = self.diaSourceDpddifier.run(catalog,
diffim,
return_pandas=True)
self.associator.run(dia_sources, diffim, self.apdb)
def run(self, repo, visitIds):
"""Loop through visits and process all available exposures within.
"""
self.butler = dafPersist.Butler(repo)
for v in visitIds:
ids = self.butler.queryMetadata(self.diffType + "Diff_diaSrc",
['visit', 'ccdnum', 'filter'],
{'visit': v})
for x in ids:
dataId = {'visit': x[0], 'ccdnum': x[1], 'filter': x[2]}
self.log.info("Processing %s", dataId)
self.procOneExposure(dataId)