def test_score_and_match(self): """Test association between a set of sources and an existing DIAObjectCollection. """ assoc_task = AssociationTask() score_struct = assoc_task.score(self.diaObjects, self.diaSourceZeroScatter, 1.0 * geom.arcseconds) self.assertFalse(np.isfinite(score_struct.scores[0])) for src_idx in range(1, len(self.diaSources)): # Our scores should be extremely close to 0 but not exactly so due # to machine noise. self.assertAlmostEqual(score_struct.scores[src_idx], 0.0, places=16) # After matching each DIAObject should now contain 2 DIASources # except the last DIAObject in this collection which should be # newly created during the matching step and contain only one # DIASource. match_result = assoc_task.match(self.diaObjects, self.diaSources, score_struct) self.assertEqual(match_result.nUpdatedDiaObjects, 4) self.assertEqual(match_result.nUnassociatedDiaObjects, 1)
def __init__(self, db_file): """Create similar configuration for tasks ad in ap_pipe. """ self.log = Log.getLogger("RunAssociation") self.apdbConfig = ApdbConfig() self.apdbConfig.db_url = "sqlite:///" + db_file self.apdbConfig.isolation_level = "READ_UNCOMMITTED" self.apdbConfig.dia_object_index = "baseline" self.apdbConfig.dia_object_columns = [] self.apdbConfig.connection_timeout = 240 self.apdbConfig.schema_file = _data_file_name("apdb-schema.yaml", "dax_apdb") self.apdbConfig.column_map = _data_file_name( "apdb-ap-pipe-afw-map.yaml", "ap_association") self.apdbConfig.extra_schema_file = _data_file_name( "apdb-ap-pipe-schema-extra.yaml", "ap_association") self.apdb = Apdb(config=self.apdbConfig, afw_schemas=dict(DiaObject=make_dia_object_schema(), DiaSource=make_dia_source_schema())) # apdb.makeSchema() self.differencerConfig = ImageDifferenceConfig() # Schema is different if we do decorrelation self.differencerConfig.doDecorrelation = True self.differencerSchema = ImageDifferenceTask( config=self.differencerConfig).schema self.diaSourceDpddifier = MapDiaSourceTask( inputSchema=self.differencerSchema) self.associator = AssociationTask() self.diffType = "deep"
def test_remove_nan_dia_sources(self): """Test removing DiaSources with NaN locations. """ self.diaSources.loc[2, "ra"] = np.nan self.diaSources.loc[3, "decl"] = np.nan self.diaSources.loc[4, "ra"] = np.nan self.diaSources.loc[4, "decl"] = np.nan assoc_task = AssociationTask() out_dia_sources = assoc_task.check_dia_source_radec(self.diaSources) self.assertEqual(len(out_dia_sources), len(self.diaSources) - 3)
def test_run_no_existing_objects(self): """Test the run method with a completely empty database. """ assocTask = AssociationTask() results = assocTask.run( self.diaSources, pd.DataFrame(columns=["ra", "decl", "diaObjectId"])) self.assertEqual(results.nUpdatedDiaObjects, 0) self.assertEqual(results.nUnassociatedDiaObjects, 0) self.assertEqual(len(results.matchedDiaSources), 0) self.assertTrue(np.all(results.unAssocDiaSources["diaObjectId"] == 0))
def test_associate_sources(self): """Test the performance of the associate_sources method in AssociationTask. """ assoc_task = AssociationTask() assoc_result = assoc_task.associate_sources(self.diaObjects, self.diaSources) for test_obj_id, expected_obj_id in zip( assoc_result.diaSources["diaObjectId"].to_numpy(), [0, 1, 2, 3, 4]): self.assertEqual(test_obj_id, expected_obj_id)
def test_run(self): """Test the full task by associating a set of diaSources to existing diaObjects. """ assocTask = AssociationTask() results = assocTask.run(self.diaSources, self.diaObjects) self.assertEqual(results.nUpdatedDiaObjects, len(self.diaObjects) - 1) self.assertEqual(results.nUnassociatedDiaObjects, 1) self.assertEqual(len(results.matchedDiaSources), len(self.diaObjects) - 1) self.assertEqual(len(results.unAssocDiaSources), 1) for test_obj_id, expected_obj_id in zip( results.matchedDiaSources["diaObjectId"].to_numpy(), [1, 2, 3, 4]): self.assertEqual(test_obj_id, expected_obj_id) for test_obj_id, expected_obj_id in zip( results.unAssocDiaSources["diaObjectId"].to_numpy(), [0]): self.assertEqual(test_obj_id, expected_obj_id)
def test_remove_nan_dia_sources(self): n_sources = 6 dia_sources = create_test_points_pandas( point_locs_deg=[ [0.04 * (src_idx + 1), 0.04 * (src_idx + 1)] for src_idx in range(n_sources)], start_id=0, scatter_arcsec=-1) dia_sources.rename(columns={"coord_ra": "ra", "coord_dec": "decl", "id": "diaSourceId"}, inplace=True) dia_sources.loc[2, "ra"] = np.nan dia_sources.loc[3, "decl"] = np.nan dia_sources.loc[4, "ra"] = np.nan dia_sources.loc[4, "decl"] = np.nan assoc_task = AssociationTask() out_dia_sources = assoc_task.check_dia_source_radec(dia_sources) self.assertEqual(len(out_dia_sources), n_sources - 3)
def test_associate_sources(self): """Test the performance of the associate_sources method in AssociationTask. """ n_objects = 5 dia_objects = create_test_points_pandas( point_locs_deg=[[0.04 * obj_idx, 0.04 * obj_idx] for obj_idx in range(n_objects)], start_id=0, schema=self.dia_object_schema, scatter_arcsec=-1,) dia_objects.rename(columns={"coord_ra": "ra", "coord_dec": "decl", "id": "diaObjectId"}, inplace=True) n_sources = 5 dia_sources = create_test_points_pandas( point_locs_deg=[ [0.04 * (src_idx + 1), 0.04 * (src_idx + 1)] for src_idx in range(n_sources)], start_id=n_objects, scatter_arcsec=0.1) dia_sources.rename(columns={"coord_ra": "ra", "coord_dec": "decl", "id": "diaSourceId"}, inplace=True) assoc_task = AssociationTask() assoc_result = assoc_task.associate_sources( dia_objects, dia_sources) for test_obj_id, expected_obj_id in zip( assoc_result.associated_dia_object_ids, [1, 2, 3, 4, 9]): self.assertEqual(test_obj_id, expected_obj_id)
def test_score_and_match(self): """Test association between a set of sources and an existing DIAObjectCollection. This also tests that a DIASource that can't be associated within tolerance is appended to the DIAObjectCollection as a new DIAObject. """ assoc_task = AssociationTask() # Create a set of DIAObjects that contain only one DIASource n_objects = 5 dia_objects = create_test_points_pandas( point_locs_deg=[[0.04 * obj_idx, 0.04 * obj_idx] for obj_idx in range(n_objects)], start_id=0, schema=self.dia_object_schema, scatter_arcsec=-1,) dia_objects.rename(columns={"coord_ra": "ra", "coord_dec": "decl", "id": "diaObjectId"}, inplace=True) n_sources = 5 dia_sources = create_test_points_pandas( point_locs_deg=[ [0.04 * (src_idx + 1), 0.04 * (src_idx + 1)] for src_idx in range(n_sources)], start_id=n_objects, scatter_arcsec=-1) dia_sources.rename(columns={"coord_ra": "ra", "coord_dec": "decl", "id": "diaSourceId"}, inplace=True) score_struct = assoc_task.score(dia_objects, dia_sources, 1.0 * geom.arcseconds) self.assertFalse(np.isfinite(score_struct.scores[-1])) for src_idx in range(4): # Our scores should be extremely close to 0 but not exactly so due # to machine noise. self.assertAlmostEqual(score_struct.scores[src_idx], 0.0, places=16) # After matching each DIAObject should now contain 2 DIASources # except the last DIAObject in this collection which should be # newly created during the matching step and contain only one # DIASource. match_result = assoc_task.match(dia_objects, dia_sources, score_struct) updated_ids = match_result.associated_dia_object_ids self.assertEqual(len(updated_ids), 5) self.assertEqual(match_result.n_updated_dia_objects, 4) self.assertEqual(match_result.n_new_dia_objects, 1) self.assertEqual(match_result.n_unassociated_dia_objects, 1) # Test updating all DiaObjects n_objects = 4 dia_objects = create_test_points_pandas( point_locs_deg=[[0.04 * obj_idx, 0.04 * obj_idx] for obj_idx in range(n_objects)], start_id=0, schema=self.dia_object_schema, scatter_arcsec=-1,) dia_objects.rename(columns={"coord_ra": "ra", "coord_dec": "decl", "id": "diaObjectId"}, inplace=True) n_sources = 4 dia_sources = create_test_points_pandas( point_locs_deg=[ [0.04 * src_idx, 0.04 * src_idx] for src_idx in range(n_sources)], start_id=n_objects, scatter_arcsec=-1) dia_sources.rename(columns={"coord_ra": "ra", "coord_dec": "decl", "id": "diaSourceId"}, inplace=True) score_struct = assoc_task.score(dia_objects[1:], dia_sources[:-1], 1.0 * geom.arcseconds) match_result = assoc_task.match(dia_objects, dia_sources, score_struct) updated_ids = match_result.associated_dia_object_ids self.assertEqual(len(updated_ids), 4)
def _run_association_and_retrieve_objects(self, create_objects=False): """Convenience method for testing the Association run method. Parameters ---------- create_objects : `bool` Boolean specifying if seed DIAObjects and DIASources should be inserted into the database before association. Return ------ dia_objects : `lsst.afw.table.SourceCatalog` Final set of DIAObjects to be tested. """ if create_objects: diaObjects, diaSourceHistory = \ self._create_dia_objects_and_sources() else: diaObjects = pd.DataFrame(columns=["diaObjectId"]) diaSourceHistory = pd.DataFrame(columns=["diaObjectId", "filterName", "diaSourceId"]) diaObjects.set_index("diaObjectId", inplace=True, drop=False) diaSourceHistory.set_index(["diaObjectId", "filterName", "diaSourceId"], inplace=True, drop=False) source_centers = [ [self.wcs.pixelToSky(idx, idx).getRa().asDegrees(), self.wcs.pixelToSky(idx, idx).getDec().asDegrees()] for idx in np.linspace(1, 1000, 10)[1:]] dia_sources = create_test_points( point_locs_deg=source_centers, start_id=10, scatter_arcsec=-1) for dia_source in dia_sources: self._set_source_values( dia_source=dia_source, flux=10000, fluxErr=100, # TODO DM-27170: fix this [0] workaround which gets a # single character representation of the band. filterName=self.exposure.getFilter().getCanonicalName()[0], filterId=self.exposure.getFilter().getId(), ccdVisitId=self.exposure.getInfo().getVisitInfo().getExposureId(), midPointTai=self.exposure.getInfo().getVisitInfo().getDate().get(system=dafBase.DateTime.MJD)) assoc_task = AssociationTask() diaSources = dia_sources.asAstropy().to_pandas() diaSources.rename(columns={"coord_ra": "ra", "coord_dec": "decl", "id": "diaSourceId", "parent": "parentDiaSourceId"}, inplace=True) diaSources["ra"] = np.degrees(diaSources["ra"]) diaSources["decl"] = np.degrees(diaSources["decl"]) if len(diaObjects) == 0: diaSourceHistory = pd.DataFrame(columns=["diaObjectId", "filterName", "diaSourceId"]) diaSourceHistory.set_index( ["diaObjectId", "filterName", "diaSourceId"], drop=False, inplace=True) results = assoc_task.run(diaSources, diaObjects, diaSourceHistory) return results.diaObjects
class RunAssociation: def __init__(self, db_file): """Create similar configuration for tasks ad in ap_pipe. """ self.log = Log.getLogger("RunAssociation") self.apdbConfig = ApdbConfig() self.apdbConfig.db_url = "sqlite:///" + db_file self.apdbConfig.isolation_level = "READ_UNCOMMITTED" self.apdbConfig.dia_object_index = "baseline" self.apdbConfig.dia_object_columns = [] self.apdbConfig.connection_timeout = 240 self.apdbConfig.schema_file = _data_file_name("apdb-schema.yaml", "dax_apdb") self.apdbConfig.column_map = _data_file_name( "apdb-ap-pipe-afw-map.yaml", "ap_association") self.apdbConfig.extra_schema_file = _data_file_name( "apdb-ap-pipe-schema-extra.yaml", "ap_association") self.apdb = Apdb(config=self.apdbConfig, afw_schemas=dict(DiaObject=make_dia_object_schema(), DiaSource=make_dia_source_schema())) # apdb.makeSchema() self.differencerConfig = ImageDifferenceConfig() # Schema is different if we do decorrelation self.differencerConfig.doDecorrelation = True self.differencerSchema = ImageDifferenceTask( config=self.differencerConfig).schema self.diaSourceDpddifier = MapDiaSourceTask( inputSchema=self.differencerSchema) self.associator = AssociationTask() self.diffType = "deep" def procOneExposure(self, dataId): """Run AssociationTask on the exposure selected by ``dataId``. """ try: catalog = self.butler.get(self.diffType + "Diff_diaSrc", dataId=dataId) diffim = self.butler.get(self.diffType + "Diff_differenceExp", dataId=dataId) except butlerExceptions.NoResults: self.log.info("Data does not exist %s", dataId) return dia_sources = self.diaSourceDpddifier.run(catalog, diffim, return_pandas=True) self.associator.run(dia_sources, diffim, self.apdb) def run(self, repo, visitIds): """Loop through visits and process all available exposures within. """ self.butler = dafPersist.Butler(repo) for v in visitIds: ids = self.butler.queryMetadata(self.diffType + "Diff_diaSrc", ['visit', 'ccdnum', 'filter'], {'visit': v}) for x in ids: dataId = {'visit': x[0], 'ccdnum': x[1], 'filter': x[2]} self.log.info("Processing %s", dataId) self.procOneExposure(dataId)