def test_run_dia_source(self):
"""Test the DiaSource specific data product mapper/calibrator.
"""
mapApDConfig = self._create_map_dia_source_config()
mapApD = MapDiaSourceTask(inputSchema=self.inputCatalog.schema,
config=mapApDConfig)
outputCatalog = mapApD.run(self.inputCatalog, self.exposure)
expectedMeanDip = 2.
expectedDiffFlux = 0.
expectedLength = 0.19999994425496748
expectedBBox = 18
for inObj, outObj in zip(self.inputCatalog, outputCatalog):
self.assertEqual(
outObj["ccdVisitId"],
self.exposure.getInfo().getVisitInfo().getExposureId())
self.assertEqual(
outObj["midPointTai"],
self.exposure.getInfo().getVisitInfo().getDate().get(
system=dafBase.DateTime.MJD))
self.assertEqual(outObj["flags"], 1 * 2**0 + 1 * 2**1)
for inputName, outputName in mapApDConfig.copyColumns.items():
if inputName.startswith("slot_PsfFlux"):
self._test_calibrated_flux(inObj, outObj)
else:
self.assertEqual(inObj[inputName], outObj[outputName])
self.assertEqual(outObj["dipMeanFlux"], expectedMeanDip)
self.assertEqual(outObj["dipFluxDiff"], expectedDiffFlux)
self.assertAlmostEqual(outObj["dipLength"], expectedLength)
self.assertEqual(outObj["bboxSize"], expectedBBox)
# Mapper should always emit standardized filters
self.assertEqual(outObj["filterName"], 'g')
def test_bit_unpacker(self):
"""Test that the integer bit packer is functioning correctly.
"""
mapApDConfig = self._create_map_dia_source_config()
mapApD = MapDiaSourceTask(inputSchema=self.inputCatalog.schema,
config=mapApDConfig)
for idx, obj in enumerate(self.inputCatalog):
if idx in [1, 3, 5]:
obj.set("base_PixelFlags_flag", 0)
if idx in [1, 4, 6]:
obj.set("base_PixelFlags_flag_offimage", 0)
outputCatalog = mapApD.run(self.inputCatalog, self.exposure)
unpacker = UnpackApdbFlags(mapApDConfig.flagMap, "DiaSource")
flag_values = unpacker.unpack(outputCatalog.get("flags"), "flags")
for idx, flag in enumerate(flag_values):
if idx in [1, 3, 5]:
self.assertFalse(flag['base_PixelFlags_flag'])
else:
self.assertTrue(flag['base_PixelFlags_flag'])
if idx in [1, 4, 6]:
self.assertFalse(flag['base_PixelFlags_flag_offimage'])
else:
self.assertTrue(flag['base_PixelFlags_flag_offimage'])
def __init__(self, db_file):
"""Create similar configuration for tasks ad in ap_pipe.
"""
self.log = Log.getLogger("RunAssociation")
self.apdbConfig = ApdbConfig()
self.apdbConfig.db_url = "sqlite:///" + db_file
self.apdbConfig.isolation_level = "READ_UNCOMMITTED"
self.apdbConfig.dia_object_index = "baseline"
self.apdbConfig.dia_object_columns = []
self.apdbConfig.connection_timeout = 240
self.apdbConfig.schema_file = _data_file_name("apdb-schema.yaml",
"dax_apdb")
self.apdbConfig.column_map = _data_file_name(
"apdb-ap-pipe-afw-map.yaml", "ap_association")
self.apdbConfig.extra_schema_file = _data_file_name(
"apdb-ap-pipe-schema-extra.yaml", "ap_association")
self.apdb = Apdb(config=self.apdbConfig,
afw_schemas=dict(DiaObject=make_dia_object_schema(),
DiaSource=make_dia_source_schema()))
# apdb.makeSchema()
self.differencerConfig = ImageDifferenceConfig()
# Schema is different if we do decorrelation
self.differencerConfig.doDecorrelation = True
self.differencerSchema = ImageDifferenceTask(
config=self.differencerConfig).schema
self.diaSourceDpddifier = MapDiaSourceTask(
inputSchema=self.differencerSchema)
self.associator = AssociationTask()
self.diffType = "deep"
def test_computeBBoxSize(self):
"""Test the values created for diaSourceBBox.
"""
outSchema = afwTable.SourceTable.makeMinimalSchema()
outSchema.addField("bboxSize", type="I")
outputCatalog = afwTable.SourceCatalog(outSchema)
outRecord = outputCatalog.addNew()
mapApDConfig = self._create_map_dia_source_config()
mapApD = MapDiaSourceTask(inputSchema=self.inputCatalog.schema,
config=mapApDConfig)
mapApD.computeBBoxSize(self.inputCatalog[0], outRecord)
self.assertEqual(outRecord["bboxSize"], 18)
def test_calibrateFluxes(self):
"""Test that flux calibration works as expected.
"""
outSchema = afwTable.SourceTable.makeMinimalSchema()
outSchema.addField("psFlux", type="D")
outSchema.addField("psFluxErr", type="D")
outputCatalog = afwTable.SourceCatalog(outSchema)
outRecord = outputCatalog.addNew()
mapApDConfig = self._create_map_dia_source_config()
mapApD = MapDiaSourceTask(inputSchema=self.inputCatalog.schema,
config=mapApDConfig)
mapApD.calibrateFluxes(self.inputCatalog[0], outRecord,
self.photoCalib)
self._test_calibrated_flux(self.inputCatalog[0], outRecord)
def test_run_dia_source_wrong_flags(self):
"""Test that the proper errors are thrown when requesting flag columns
that are not in the input schema.
"""
mapApDConfig = self._create_map_dia_source_config()
with self.assertRaises(KeyError):
MapDiaSourceTask(inputSchema=self.inputCatalogNoFlags.schema,
config=mapApDConfig)
class RunAssociation:
def __init__(self, db_file):
"""Create similar configuration for tasks ad in ap_pipe.
"""
self.log = Log.getLogger("RunAssociation")
self.apdbConfig = ApdbConfig()
self.apdbConfig.db_url = "sqlite:///" + db_file
self.apdbConfig.isolation_level = "READ_UNCOMMITTED"
self.apdbConfig.dia_object_index = "baseline"
self.apdbConfig.dia_object_columns = []
self.apdbConfig.connection_timeout = 240
self.apdbConfig.schema_file = _data_file_name("apdb-schema.yaml",
"dax_apdb")
self.apdbConfig.column_map = _data_file_name(
"apdb-ap-pipe-afw-map.yaml", "ap_association")
self.apdbConfig.extra_schema_file = _data_file_name(
"apdb-ap-pipe-schema-extra.yaml", "ap_association")
self.apdb = Apdb(config=self.apdbConfig,
afw_schemas=dict(DiaObject=make_dia_object_schema(),
DiaSource=make_dia_source_schema()))
# apdb.makeSchema()
self.differencerConfig = ImageDifferenceConfig()
# Schema is different if we do decorrelation
self.differencerConfig.doDecorrelation = True
self.differencerSchema = ImageDifferenceTask(
config=self.differencerConfig).schema
self.diaSourceDpddifier = MapDiaSourceTask(
inputSchema=self.differencerSchema)
self.associator = AssociationTask()
self.diffType = "deep"
def procOneExposure(self, dataId):
"""Run AssociationTask on the exposure selected by ``dataId``.
"""
try:
catalog = self.butler.get(self.diffType + "Diff_diaSrc",
dataId=dataId)
diffim = self.butler.get(self.diffType + "Diff_differenceExp",
dataId=dataId)
except butlerExceptions.NoResults:
self.log.info("Data does not exist %s", dataId)
return
dia_sources = self.diaSourceDpddifier.run(catalog,
diffim,
return_pandas=True)
self.associator.run(dia_sources, diffim, self.apdb)
def run(self, repo, visitIds):
"""Loop through visits and process all available exposures within.
"""
self.butler = dafPersist.Butler(repo)
for v in visitIds:
ids = self.butler.queryMetadata(self.diffType + "Diff_diaSrc",
['visit', 'ccdnum', 'filter'],
{'visit': v})
for x in ids:
dataId = {'visit': x[0], 'ccdnum': x[1], 'filter': x[2]}
self.log.info("Processing %s", dataId)
self.procOneExposure(dataId)