def test_run_dia_source(self): """Test the DiaSource specific data product mapper/calibrator. """ mapApDConfig = self._create_map_dia_source_config() mapApD = MapDiaSourceTask(inputSchema=self.inputCatalog.schema, config=mapApDConfig) outputCatalog = mapApD.run(self.inputCatalog, self.exposure) expectedMeanDip = 2. expectedDiffFlux = 0. expectedLength = 0.19999994425496748 expectedBBox = 18 for inObj, outObj in zip(self.inputCatalog, outputCatalog): self.assertEqual( outObj["ccdVisitId"], self.exposure.getInfo().getVisitInfo().getExposureId()) self.assertEqual( outObj["midPointTai"], self.exposure.getInfo().getVisitInfo().getDate().get( system=dafBase.DateTime.MJD)) self.assertEqual(outObj["flags"], 1 * 2**0 + 1 * 2**1) for inputName, outputName in mapApDConfig.copyColumns.items(): if inputName.startswith("slot_PsfFlux"): self._test_calibrated_flux(inObj, outObj) else: self.assertEqual(inObj[inputName], outObj[outputName]) self.assertEqual(outObj["dipMeanFlux"], expectedMeanDip) self.assertEqual(outObj["dipFluxDiff"], expectedDiffFlux) self.assertAlmostEqual(outObj["dipLength"], expectedLength) self.assertEqual(outObj["bboxSize"], expectedBBox) # Mapper should always emit standardized filters self.assertEqual(outObj["filterName"], 'g')
def test_bit_unpacker(self): """Test that the integer bit packer is functioning correctly. """ mapApDConfig = self._create_map_dia_source_config() mapApD = MapDiaSourceTask(inputSchema=self.inputCatalog.schema, config=mapApDConfig) for idx, obj in enumerate(self.inputCatalog): if idx in [1, 3, 5]: obj.set("base_PixelFlags_flag", 0) if idx in [1, 4, 6]: obj.set("base_PixelFlags_flag_offimage", 0) outputCatalog = mapApD.run(self.inputCatalog, self.exposure) unpacker = UnpackApdbFlags(mapApDConfig.flagMap, "DiaSource") flag_values = unpacker.unpack(outputCatalog.get("flags"), "flags") for idx, flag in enumerate(flag_values): if idx in [1, 3, 5]: self.assertFalse(flag['base_PixelFlags_flag']) else: self.assertTrue(flag['base_PixelFlags_flag']) if idx in [1, 4, 6]: self.assertFalse(flag['base_PixelFlags_flag_offimage']) else: self.assertTrue(flag['base_PixelFlags_flag_offimage'])
def __init__(self, db_file): """Create similar configuration for tasks ad in ap_pipe. """ self.log = Log.getLogger("RunAssociation") self.apdbConfig = ApdbConfig() self.apdbConfig.db_url = "sqlite:///" + db_file self.apdbConfig.isolation_level = "READ_UNCOMMITTED" self.apdbConfig.dia_object_index = "baseline" self.apdbConfig.dia_object_columns = [] self.apdbConfig.connection_timeout = 240 self.apdbConfig.schema_file = _data_file_name("apdb-schema.yaml", "dax_apdb") self.apdbConfig.column_map = _data_file_name( "apdb-ap-pipe-afw-map.yaml", "ap_association") self.apdbConfig.extra_schema_file = _data_file_name( "apdb-ap-pipe-schema-extra.yaml", "ap_association") self.apdb = Apdb(config=self.apdbConfig, afw_schemas=dict(DiaObject=make_dia_object_schema(), DiaSource=make_dia_source_schema())) # apdb.makeSchema() self.differencerConfig = ImageDifferenceConfig() # Schema is different if we do decorrelation self.differencerConfig.doDecorrelation = True self.differencerSchema = ImageDifferenceTask( config=self.differencerConfig).schema self.diaSourceDpddifier = MapDiaSourceTask( inputSchema=self.differencerSchema) self.associator = AssociationTask() self.diffType = "deep"
def test_computeBBoxSize(self): """Test the values created for diaSourceBBox. """ outSchema = afwTable.SourceTable.makeMinimalSchema() outSchema.addField("bboxSize", type="I") outputCatalog = afwTable.SourceCatalog(outSchema) outRecord = outputCatalog.addNew() mapApDConfig = self._create_map_dia_source_config() mapApD = MapDiaSourceTask(inputSchema=self.inputCatalog.schema, config=mapApDConfig) mapApD.computeBBoxSize(self.inputCatalog[0], outRecord) self.assertEqual(outRecord["bboxSize"], 18)
def test_calibrateFluxes(self): """Test that flux calibration works as expected. """ outSchema = afwTable.SourceTable.makeMinimalSchema() outSchema.addField("psFlux", type="D") outSchema.addField("psFluxErr", type="D") outputCatalog = afwTable.SourceCatalog(outSchema) outRecord = outputCatalog.addNew() mapApDConfig = self._create_map_dia_source_config() mapApD = MapDiaSourceTask(inputSchema=self.inputCatalog.schema, config=mapApDConfig) mapApD.calibrateFluxes(self.inputCatalog[0], outRecord, self.photoCalib) self._test_calibrated_flux(self.inputCatalog[0], outRecord)
def test_run_dia_source_wrong_flags(self): """Test that the proper errors are thrown when requesting flag columns that are not in the input schema. """ mapApDConfig = self._create_map_dia_source_config() with self.assertRaises(KeyError): MapDiaSourceTask(inputSchema=self.inputCatalogNoFlags.schema, config=mapApDConfig)
class RunAssociation: def __init__(self, db_file): """Create similar configuration for tasks ad in ap_pipe. """ self.log = Log.getLogger("RunAssociation") self.apdbConfig = ApdbConfig() self.apdbConfig.db_url = "sqlite:///" + db_file self.apdbConfig.isolation_level = "READ_UNCOMMITTED" self.apdbConfig.dia_object_index = "baseline" self.apdbConfig.dia_object_columns = [] self.apdbConfig.connection_timeout = 240 self.apdbConfig.schema_file = _data_file_name("apdb-schema.yaml", "dax_apdb") self.apdbConfig.column_map = _data_file_name( "apdb-ap-pipe-afw-map.yaml", "ap_association") self.apdbConfig.extra_schema_file = _data_file_name( "apdb-ap-pipe-schema-extra.yaml", "ap_association") self.apdb = Apdb(config=self.apdbConfig, afw_schemas=dict(DiaObject=make_dia_object_schema(), DiaSource=make_dia_source_schema())) # apdb.makeSchema() self.differencerConfig = ImageDifferenceConfig() # Schema is different if we do decorrelation self.differencerConfig.doDecorrelation = True self.differencerSchema = ImageDifferenceTask( config=self.differencerConfig).schema self.diaSourceDpddifier = MapDiaSourceTask( inputSchema=self.differencerSchema) self.associator = AssociationTask() self.diffType = "deep" def procOneExposure(self, dataId): """Run AssociationTask on the exposure selected by ``dataId``. """ try: catalog = self.butler.get(self.diffType + "Diff_diaSrc", dataId=dataId) diffim = self.butler.get(self.diffType + "Diff_differenceExp", dataId=dataId) except butlerExceptions.NoResults: self.log.info("Data does not exist %s", dataId) return dia_sources = self.diaSourceDpddifier.run(catalog, diffim, return_pandas=True) self.associator.run(dia_sources, diffim, self.apdb) def run(self, repo, visitIds): """Loop through visits and process all available exposures within. """ self.butler = dafPersist.Butler(repo) for v in visitIds: ids = self.butler.queryMetadata(self.diffType + "Diff_diaSrc", ['visit', 'ccdnum', 'filter'], {'visit': v}) for x in ids: dataId = {'visit': x[0], 'ccdnum': x[1], 'filter': x[2]} self.log.info("Processing %s", dataId) self.procOneExposure(dataId)