def check(self, expectFactor):
schema = SourceTable.makeMinimalSchema()
task = DynamicDetectionTask(config=self.config, schema=schema)
table = SourceTable.make(schema)
results = task.run(table, self.exposure, expId=12345)
self.assertFloatsAlmostEqual(results.factor, expectFactor, rtol=self.rtol)
def check(self, expectFactor):
schema = SourceTable.makeMinimalSchema()
task = DynamicDetectionTask(config=self.config, schema=schema)
table = SourceTable.make(schema)
results = task.run(table, self.exposure, expId=12345)
self.assertFloatsAlmostEqual(results.factor,
expectFactor,
rtol=self.rtol)
def __init__(self, *args, **kwargs):
"""Constructor
Besides the usual initialisation of configurables, we also set up
the forced measurement which is deliberately not represented in
this Task's configuration parameters because we're using it as part
of the algorithm and we don't want to allow it to be modified.
"""
SourceDetectionTask.__init__(self, *args, **kwargs)
self.makeSubtask("skyObjects")
# Set up forced measurement.
config = ForcedMeasurementTask.ConfigClass()
config.plugins.names = [
'base_TransformedCentroid', 'base_PsfFlux', 'base_LocalBackground'
]
# We'll need the "centroid" and "psfFlux" slots
for slot in ("shape", "psfShape", "apFlux", "modelFlux",
"gaussianFlux", "calibFlux"):
setattr(config.slots, slot, None)
config.copyColumns = {}
self.skySchema = SourceTable.makeMinimalSchema()
self.skyMeasurement = ForcedMeasurementTask(config=config,
name="skyMeasurement",
parentTask=self,
refSchema=self.skySchema)
def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
"""!Construct a CharacterizeImageTask
@param[in] butler A butler object is passed to the refObjLoader constructor in case
it is needed to load catalogs. May be None if a catalog-based star selector is
not used, if the reference object loader constructor does not require a butler,
or if a reference object loader is passed directly via the refObjLoader argument.
@param[in] refObjLoader An instance of LoadReferenceObjectsTasks that supplies an
external reference catalog to a catalog-based star selector. May be None if a
catalog star selector is not used or the loader can be constructed from the
butler argument.
@param[in,out] schema initial schema (an lsst.afw.table.SourceTable), or None
@param[in,out] kwargs other keyword arguments for lsst.pipe.base.CmdLineTask
"""
pipeBase.CmdLineTask.__init__(self, **kwargs)
if schema is None:
schema = SourceTable.makeMinimalSchema()
self.schema = schema
self.makeSubtask("background")
self.makeSubtask("installSimplePsf")
self.makeSubtask("repair")
self.makeSubtask("measurePsf", schema=self.schema)
if self.config.doMeasurePsf and self.measurePsf.usesMatches:
if not refObjLoader:
self.makeSubtask('refObjLoader', butler=butler)
refObjLoader = self.refObjLoader
self.makeSubtask("astrometry", refObjLoader=refObjLoader)
self.makeSubtask("detectAndMeasure", schema=self.schema)
self._initialFrame = getDebugFrame(self._display, "frame") or 1
self._frame = self._initialFrame
self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
def run(self, exposure):
table = SourceTable.make(self.schema)
background = None
detRes = self.detection.run(table=table, exposure=exposure)
sourceCat = detRes.sources
if background is None:
background = BackgroundList()
if detRes.fpSets.background:
background.append(detRes.fpSets.background)
if self.config.doDeblend:
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(
measCat=sourceCat,
exposure=exposure
)
#results2 = self.deblend.run(...)
#results3 = self.measurement.run(...)
return pipeBase.Struct(
exposure = exposure,
background = background,
sourceCat = sourceCat
)
def __init__(self, schema=None, butler=None, **kwargs):
"""Initalize things! This should go above in the class docstring
"""
super().__init__(**kwargs)
if schema is None:
schema = SourceTable.makeMinimalSchema()
self.schema = schema
self.makeSubtask("insertFakes")
self.makeSubtask("calibrate")
def testNoPsfUsed(self):
"""Test that the "calib_psfUsed" is required to measure aperture correction
I hope someday DetectAndMeasureTask can determine for itself
which sources are suitable for measuring aperture correction,
at which point I expect this test to be deleted.
"""
schema = SourceTable.makeMinimalSchema()
config = DetectAndMeasureTask.ConfigClass()
config.doMeasureApCorr = True
with self.assertRaises(Exception):
DetectAndMeasureTask(config=config, schema=schema)
def run(self, exposure, exposureIdInfo, background=None, allowApCorr=True):
"""!Detect, deblend and perform single-frame measurement on sources and refine the background model
@param[in,out] exposure exposure to process. Background must already be subtracted
to reasonable accuracy, else detection will fail.
The background model is refined and resubtracted.
apCorrMap is set if measuring aperture correction.
@param[in] exposureIdInfo ID info for exposure (an lsst.daf.butlerUtils.ExposureIdInfo)
@param[in,out] background background model to be modified (an lsst.afw.math.BackgroundList),
or None to create a new background model
@param[in] allowApCorr allow measuring and applying aperture correction?
@return pipe_base Struct containing these fields:
- exposure: input exposure (as modified in the course of runing)
- sourceCat: source catalog (an lsst.afw.table.SourceTable)
- background: background model (input as modified, or a new model if input is None);
an lsst.afw.math.BackgroundList
"""
if background is None:
background = BackgroundList()
sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
exposureIdInfo.unusedBits)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
detRes = self.detection.run(table=table,
exposure=exposure,
doSmooth=True)
sourceCat = detRes.sources
if detRes.fpSets.background:
background.append(detRes.fpSets.background)
if self.config.doDeblend:
self.deblend.run(
exposure=exposure,
sources=sourceCat,
psf=exposure.getPsf(),
)
self.measure(
exposure=exposure,
exposureIdInfo=exposureIdInfo,
sourceCat=sourceCat,
allowApCorr=allowApCorr,
)
return pipeBase.Struct(
exposure=exposure,
sourceCat=sourceCat,
background=background,
)
def testNoPsfUsed(self):
"""Test that the "calib_psfUsed" is required to measure aperture correction
I hope someday DetectAndMeasureTask can determine for itself
which sources are suitable for measuring aperture correction,
at which point I expect this test to be deleted.
"""
schema = SourceTable.makeMinimalSchema()
config = DetectAndMeasureTask.ConfigClass()
config.doMeasureApCorr = True
with self.assertRaises(Exception):
DetectAndMeasureTask(config=config, schema=schema)
def run(self, exposure, exposureIdInfo, background=None, allowApCorr=True):
"""!Detect, deblend and perform single-frame measurement on sources and refine the background model
@param[in,out] exposure exposure to process. Background must already be subtracted
to reasonable accuracy, else detection will fail.
The background model is refined and resubtracted.
apCorrMap is set if measuring aperture correction.
@param[in] exposureIdInfo ID info for exposure (an lsst.daf.butlerUtils.ExposureIdInfo)
@param[in,out] background background model to be modified (an lsst.afw.math.BackgroundList),
or None to create a new background model
@param[in] allowApCorr allow measuring and applying aperture correction?
@return pipe_base Struct containing these fields:
- exposure: input exposure (as modified in the course of runing)
- sourceCat: source catalog (an lsst.afw.table.SourceTable)
- background: background model (input as modified, or a new model if input is None);
an lsst.afw.math.BackgroundList
"""
if background is None:
background = BackgroundList()
sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId, exposureIdInfo.unusedBits)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True)
sourceCat = detRes.sources
if detRes.fpSets.background:
background.append(detRes.fpSets.background)
if self.config.doDeblend:
self.deblend.run(
exposure = exposure,
sources = sourceCat,
psf = exposure.getPsf(),
)
self.measure(
exposure = exposure,
exposureIdInfo = exposureIdInfo,
sourceCat = sourceCat,
allowApCorr = allowApCorr,
)
return pipeBase.Struct(
exposure = exposure,
sourceCat = sourceCat,
background = background,
)
def testBasics(self):
"""Test detection and measurement on simple synthesized data
"""
bbox = Box2I(Point2I(256, 100), Extent2I(128, 127))
minCounts = 5000
maxCounts = 50000
starSigma = 1.5
numX = 5
numY = 5
coordList = self.makeCoordList(
bbox=bbox,
numX=numX,
numY=numY,
minCounts=minCounts,
maxCounts=maxCounts,
sigma=starSigma,
)
kwid = 11 # kernel width
sky = 2000
# create an exposure without a Wcs; add the Wcs later
exposure = plantSources(bbox=bbox,
kwid=kwid,
sky=sky,
coordList=coordList,
addPoissonNoise=True)
schema = SourceTable.makeMinimalSchema()
config = DetectAndMeasureTask.ConfigClass()
task = DetectAndMeasureTask(config=config, schema=schema)
butler = Butler(root=InputDir)
dataRef = butler.dataRef("calexp", dataId=dict(visit=1))
wcs = dataRef.get("raw").getWcs()
exposure.setWcs(wcs)
exposureIdInfo = dataRef.get("expIdInfo")
taskRes = task.run(exposure=exposure, exposureIdInfo=exposureIdInfo)
self.assertEqual(len(taskRes.sourceCat), numX * numY)
schema = taskRes.sourceCat.schema
centroidFlagKey = schema.find("slot_Centroid_flag").getKey()
parentKey = schema.find("parent").getKey()
psfFluxFlagKey = schema.find("slot_PsfFlux_flag").getKey()
psfFluxKey = schema.find("slot_PsfFlux_flux").getKey()
for src in taskRes.sourceCat:
self.assertFalse(src.get(centroidFlagKey)) # centroid found
self.assertEqual(src.get(parentKey), 0) # not debelended
self.assertFalse(src.get(psfFluxFlagKey)) # flux measured
self.assertGreater(src.get(psfFluxKey), 4000) # flux sane
def __init__(self, butler=None, refObjLoader=None, schema=None, **kwargs):
"""!Construct a CharacterizeImageTask
@param[in] butler A butler object is passed to the refObjLoader constructor in case
it is needed to load catalogs. May be None if a catalog-based star selector is
not used, if the reference object loader constructor does not require a butler,
or if a reference object loader is passed directly via the refObjLoader argument.
# TODO DM-34769: remove rebObjLoader kwarg here.
@param[in] refObjLoader An instance of LoadReferenceObjectsTasks that supplies an
external reference catalog to a catalog-based star selector. May be None if a
catalog star selector is not used or the loader can be constructed from the
butler argument.
@param[in,out] schema initial schema (an lsst.afw.table.SourceTable), or None
@param[in,out] kwargs other keyword arguments for lsst.pipe.base.CmdLineTask
"""
super().__init__(**kwargs)
if schema is None:
schema = SourceTable.makeMinimalSchema()
self.schema = schema
self.makeSubtask("background")
self.makeSubtask("installSimplePsf")
self.makeSubtask("repair")
self.makeSubtask("measurePsf", schema=self.schema)
# TODO DM-34769: remove this `if` block
if self.config.doMeasurePsf and self.measurePsf.usesMatches:
if not refObjLoader:
self.makeSubtask('refObjLoader', butler=butler)
refObjLoader = self.refObjLoader
self.makeSubtask("ref_match", refObjLoader=refObjLoader)
self.algMetadata = dafBase.PropertyList()
self.makeSubtask('detection', schema=self.schema)
if self.config.doDeblend:
self.makeSubtask("deblend", schema=self.schema)
self.makeSubtask('measurement',
schema=self.schema,
algMetadata=self.algMetadata)
if self.config.doApCorr:
self.makeSubtask('measureApCorr', schema=self.schema)
self.makeSubtask('applyApCorr', schema=self.schema)
self.makeSubtask('catalogCalculation', schema=self.schema)
self._initialFrame = getDebugFrame(self._display, "frame") or 1
self._frame = self._initialFrame
self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
self.outputSchema = afwTable.SourceCatalog(self.schema)
def __init__(self, schema=None, **kwargs):
"""!Construct a CharacterizeImageTask
@param[in,out] schema initial schema (an lsst.afw.table.SourceTable), or None
@param[in,out] kwargs other keyword arguments for lsst.pipe.base.CmdLineTask
"""
pipeBase.CmdLineTask.__init__(self, **kwargs)
if schema is None:
schema = SourceTable.makeMinimalSchema()
self.schema = schema
self.makeSubtask("installSimplePsf")
self.makeSubtask("repair")
self.makeSubtask("measurePsf", schema=self.schema)
if self.config.doMeasurePsf and self.measurePsf.usesMatches:
self.makeSubtask("astrometry")
self.makeSubtask("detectAndMeasure", schema=self.schema)
self._initialFrame = getDebugFrame(self._display, "frame") or 1
self._frame = self._initialFrame
def __init__(self, schema=None, butler=None, **kwargs):
"""Initalize things! This should go above in the class docstring
"""
super().__init__(**kwargs)
if schema is None:
schema = SourceTable.makeMinimalSchema()
self.schema = schema
self.makeSubtask("insertFakes")
self.algMetadata = dafBase.PropertyList()
self.makeSubtask("detection", schema=self.schema)
self.makeSubtask("deblend", schema=self.schema)
self.makeSubtask("measurement",
schema=self.schema,
algMetadata=self.algMetadata)
self.makeSubtask("applyApCorr", schema=self.schema)
self.makeSubtask("catalogCalculation", schema=self.schema)
def __init__(self, schema=None, **kwargs):
"""!Construct a CharacterizeImageTask
@param[in,out] schema initial schema (an lsst.afw.table.SourceTable), or None
@param[in,out] kwargs other keyword arguments for lsst.pipe.base.CmdLineTask
"""
pipeBase.CmdLineTask.__init__(self, **kwargs)
if schema is None:
schema = SourceTable.makeMinimalSchema()
self.schema = schema
self.makeSubtask("installSimplePsf")
self.makeSubtask("repair")
self.makeSubtask("measurePsf", schema=self.schema)
if self.config.doMeasurePsf and self.measurePsf.usesMatches:
self.makeSubtask("astrometry")
self.makeSubtask("detectAndMeasure", schema=self.schema)
self._initialFrame = getDebugFrame(self._display, "frame") or 1
self._frame = self._initialFrame
def testBasics(self):
"""Test detection and measurement on simple synthesized data
"""
bbox = Box2I(Point2I(256, 100), Extent2I(128, 127))
minCounts = 5000
maxCounts = 50000
starSigma = 1.5
numX = 5
numY = 5
coordList = self.makeCoordList(
bbox=bbox,
numX=numX,
numY=numY,
minCounts=minCounts,
maxCounts=maxCounts,
sigma=starSigma,
)
kwid = 11 # kernel width
sky = 2000
# create an exposure without a Wcs; add the Wcs later
exposure = plantSources(bbox=bbox, kwid=kwid, sky=sky, coordList=coordList, addPoissonNoise=True)
schema = SourceTable.makeMinimalSchema()
config = DetectAndMeasureTask.ConfigClass()
task = DetectAndMeasureTask(config=config, schema=schema)
butler = Butler(root=InputDir)
dataRef = butler.dataRef("calexp", dataId=dict(visit=1))
wcs = dataRef.get("raw").getWcs()
exposure.setWcs(wcs)
exposureIdInfo = dataRef.get("expIdInfo")
taskRes = task.run(exposure=exposure, exposureIdInfo=exposureIdInfo)
self.assertEqual(len(taskRes.sourceCat), numX * numY)
schema = taskRes.sourceCat.schema
centroidFlagKey = schema.find("slot_Centroid_flag").getKey()
parentKey = schema.find("parent").getKey()
psfFluxFlagKey = schema.find("slot_PsfFlux_flag").getKey()
psfFluxKey = schema.find("slot_PsfFlux_flux").getKey()
for src in taskRes.sourceCat:
self.assertFalse(src.get(centroidFlagKey)) # centroid found
self.assertEqual(src.get(parentKey), 0) # not debelended
self.assertFalse(src.get(psfFluxFlagKey)) # flux measured
self.assertGreater(src.get(psfFluxKey), 4000) # flux sane
def measure(self, exposure, exposureIdInfo, sourceCat, allowApCorr=True):
"""Measure sources
@param[in,out] exposure exposure to process. Background must already be subtracted
to reasonable accuracy, else detection will fail.
Set apCorrMap if measuring aperture correction.
@param[in] exposureIdInfo ID info for exposure (an lsst.daf.butlerUtils.ExposureIdInfo)
@param[in,out] background background model to be modified (an lsst.afw.math.BackgroundList),
or None to create a new background model
@param[in] allowApCorr allow measuring and applying aperture correction?
"""
if self.config.doMeasureApCorr and allowApCorr:
# perform measurements before aperture correction
self.measurement.run(
measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId,
endOrder=BasePlugin.APCORR_ORDER,
)
sourceCat.sort(SourceTable.getParentKey())
# measure aperture correction
apCorrMap = self.measureApCorr.run(bbox=exposure.getBBox(),
catalog=sourceCat).apCorrMap
exposure.getInfo().setApCorrMap(apCorrMap)
# perform remaining measurements
self.measurement.run(
measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId,
beginOrder=BasePlugin.APCORR_ORDER,
allowApCorr=allowApCorr,
)
else:
self.measurement.run(
measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId,
allowApCorr=allowApCorr,
)
def __init__(self, *args, **kwargs):
SourceDetectionTask.__init__(self, *args, **kwargs)
self.makeSubtask("skyObjects")
# Set up forced measurement.
config = ForcedMeasurementTask.ConfigClass()
config.plugins.names = [
'base_TransformedCentroid', 'base_PsfFlux', 'base_LocalBackground'
]
# We'll need the "centroid" and "psfFlux" slots
for slot in ("shape", "psfShape", "apFlux", "modelFlux",
"gaussianFlux", "calibFlux"):
setattr(config.slots, slot, None)
config.copyColumns = {}
self.skySchema = SourceTable.makeMinimalSchema()
self.skyMeasurement = ForcedMeasurementTask(config=config,
name="skyMeasurement",
parentTask=self,
refSchema=self.skySchema)
def measure(self, exposure, exposureIdInfo, sourceCat, allowApCorr=True):
"""Measure sources
@param[in,out] exposure exposure to process. Background must already be subtracted
to reasonable accuracy, else detection will fail.
Set apCorrMap if measuring aperture correction.
@param[in] exposureIdInfo ID info for exposure (an lsst.daf.butlerUtils.ExposureIdInfo)
@param[in,out] background background model to be modified (an lsst.afw.math.BackgroundList),
or None to create a new background model
@param[in] allowApCorr allow measuring and applying aperture correction?
"""
if self.config.doMeasureApCorr and allowApCorr:
# perform measurements before aperture correction
self.measurement.run(
measCat = sourceCat,
exposure = exposure,
exposureId = exposureIdInfo.expId,
endOrder = BasePlugin.APCORR_ORDER,
)
sourceCat.sort(SourceTable.getParentKey())
# measure aperture correction
apCorrMap = self.measureApCorr.run(bbox=exposure.getBBox(), catalog=sourceCat).apCorrMap
exposure.getInfo().setApCorrMap(apCorrMap)
# perform remaining measurements
self.measurement.run(
measCat = sourceCat,
exposure = exposure,
exposureId = exposureIdInfo.expId,
beginOrder = BasePlugin.APCORR_ORDER,
allowApCorr = allowApCorr,
)
else:
self.measurement.run(
measCat = sourceCat,
exposure = exposure,
exposureId = exposureIdInfo.expId,
allowApCorr = allowApCorr,
)
def __init__(self, *args, **kwargs):
"""Constructor
Besides the usual initialisation of configurables, we also set up
the forced measurement which is deliberately not represented in
this Task's configuration parameters because we're using it as part
of the algorithm and we don't want to allow it to be modified.
"""
SourceDetectionTask.__init__(self, *args, **kwargs)
self.makeSubtask("skyObjects")
# Set up forced measurement.
config = ForcedMeasurementTask.ConfigClass()
config.plugins.names = ['base_TransformedCentroid', 'base_PsfFlux', 'base_LocalBackground']
# We'll need the "centroid" and "psfFlux" slots
for slot in ("shape", "psfShape", "apFlux", "modelFlux", "gaussianFlux", "calibFlux"):
setattr(config.slots, slot, None)
config.copyColumns = {}
self.skySchema = SourceTable.makeMinimalSchema()
self.skyMeasurement = ForcedMeasurementTask(config=config, name="skyMeasurement", parentTask=self,
refSchema=self.skySchema)
def run(self, exposure, exposureIdInfo=None):
if not exposure.hasPsf():
self.installSimplePsf.run(exposure=exposure)
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
try:
self.repair.run(exposure=exposure, keepCRs=True)
except LengthError:
self.log.info("Skipping cosmic ray detection: Too many CR pixels (max %0.f)" % self.repair.cosmicray.nCrPixelMax)
sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
filtered = maximum_filter(exposure.getImage().array, size=self.config.maximumFilterBoxWidth)
detected = (filtered == exposure.getImage().getArray()) & (filtered > self.config.thresholdValue)
detectedImage = afwImage.ImageF(detected.astype(np.float32))
fps = afwDetect.FootprintSet(detectedImage, afwDetect.Threshold(0.5))
fp_ctrl = afwDetect.FootprintControl(True, True)
fps = afwDetect.FootprintSet(fps, self.config.footprintGrowValue, fp_ctrl)
sources = afwTable.SourceCatalog(table)
fps.makeSources(sources)
self.measurement.run(measCat=sources, exposure=exposure, exposureId=exposureIdInfo.expId)
self.catalogCalculation.run(sources)
## Add metadata to source catalog
md = exposure.getMetadata()
sources.getMetadata().add("BOTXCAM", md["BOTXCAM"])
sources.getMetadata().add("BOTYCAM", md["BOTYCAM"])
self.display("measure", exposure=exposure, sourceCat=sources)
return pipeBase.Struct(sourceCat=sources)
def __init__(self, dataPrefix="", schema=None, **kwargs):
"""Construct a DetectAndMeasureTask
Arguments in addition to the standard Task arguments:
@param[in] dataPrefix prefix for name of source tables;
- for calexp use the default of ""
- for coadds use coaddName + "Coadd"
@param[in,out] schema schema for sources; if None then one is constructed
"""
pipeBase.Task.__init__(self, **kwargs)
self.dataPrefix = dataPrefix
self.algMetadata = dafBase.PropertyList()
if schema is None:
schema = SourceTable.makeMinimalSchema()
self.schema = schema
self.makeSubtask("detection", schema=self.schema)
if self.config.doDeblend:
self.makeSubtask("deblend", schema=self.schema)
self.makeSubtask("measurement", schema=self.schema, algMetadata=self.algMetadata)
if self.config.doMeasureApCorr:
# add field to flag stars useful for measuring aperture correction
self.makeSubtask("measureApCorr", schema=schema)
def __init__(self, butler=None, schema=None, **kwargs):
"""!Construct a CharacterizeSpotsTask
@param[in] butler A butler object is passed to the refObjLoader constructor in case
it is needed to load catalogs. May be None if a catalog-based star selector is
not used, if the reference object loader constructor does not require a butler,
or if a reference object loader is passed directly via the refObjLoader argument.
@param[in,out] schema initial schema (an lsst.afw.table.SourceTable), or None
@param[in,out] kwargs other keyword arguments for lsst.pipe.base.CmdLineTask
"""
super().__init__(**kwargs)
if schema is None:
schema = SourceTable.makeMinimalSchema()
self.schema = schema
self.makeSubtask("installSimplePsf")
self.makeSubtask("repair")
self.algMetadata = dafBase.PropertyList()
self.makeSubtask('measurement', schema=self.schema, algMetadata=self.algMetadata)
self.makeSubtask('catalogCalculation', schema=self.schema)
self._initialFrame = getDebugFrame(self._display, "frame") or 1
self._frame = self._initialFrame
self.schema.checkUnits(parse_strict=self.config.checkUnitsParseStrict)
self.outputSchema = afwTable.SourceCatalog(self.schema)
def __init__(self, dataPrefix="", schema=None, **kwargs):
"""Construct a DetectAndMeasureTask
Arguments in addition to the standard Task arguments:
@param[in] dataPrefix prefix for name of source tables;
- for calexp use the default of ""
- for coadds use coaddName + "Coadd"
@param[in,out] schema schema for sources; if None then one is constructed
"""
pipeBase.Task.__init__(self, **kwargs)
self.dataPrefix = dataPrefix
self.algMetadata = dafBase.PropertyList()
if schema is None:
schema = SourceTable.makeMinimalSchema()
self.schema = schema
self.makeSubtask("detection", schema=self.schema)
if self.config.doDeblend:
self.makeSubtask("deblend", schema=self.schema)
self.makeSubtask("measurement",
schema=self.schema,
algMetadata=self.algMetadata)
if self.config.doMeasureApCorr:
# add field to flag stars useful for measuring aperture correction
self.makeSubtask("measureApCorr", schema=schema)
def getClumps(self, sigma=1.0, display=False):
if self._num <= 0:
raise RuntimeError("No candidate PSF sources")
psfImage = self.getImage()
#
# Embed psfImage into a larger image so we can smooth when measuring it
#
width, height = psfImage.getWidth(), psfImage.getHeight()
largeImg = psfImage.Factory(afwGeom.ExtentI(2*width, 2*height))
largeImg.set(0)
bbox = afwGeom.BoxI(afwGeom.PointI(width, height), afwGeom.ExtentI(width, height))
largeImg.assign(psfImage, bbox, afwImage.LOCAL)
#
# Now measure that image, looking for the highest peak. Start by building an Exposure
#
msk = afwImage.MaskU(largeImg.getDimensions())
msk.set(0)
var = afwImage.ImageF(largeImg.getDimensions())
var.set(1)
mpsfImage = afwImage.MaskedImageF(largeImg, msk, var)
mpsfImage.setXY0(afwGeom.PointI(-width, -height))
del msk
del var
exposure = afwImage.makeExposure(mpsfImage)
#
# Next run an object detector
#
maxVal = afwMath.makeStatistics(psfImage, afwMath.MAX).getValue()
threshold = maxVal - sigma*math.sqrt(maxVal)
if threshold <= 0.0:
threshold = maxVal
threshold = afwDetection.Threshold(threshold)
ds = afwDetection.FootprintSet(mpsfImage, threshold, "DETECTED")
#
# And measure it. This policy isn't the one we use to measure
# Sources, it's only used to characterize this PSF histogram
#
schema = SourceTable.makeMinimalSchema()
psfImageConfig = SingleFrameMeasurementConfig()
psfImageConfig.doApplyApCorr = "no"
psfImageConfig.slots.centroid = "base_SdssCentroid"
psfImageConfig.slots.psfFlux = None #"base_PsfFlux"
psfImageConfig.slots.apFlux = "base_CircularApertureFlux_3_0"
psfImageConfig.slots.modelFlux = None
psfImageConfig.slots.instFlux = None
psfImageConfig.slots.calibFlux = None
psfImageConfig.slots.shape = "base_SdssShape"
# Formerly, this code had centroid.sdss, flux.psf, flux.naive,
# flags.pixel, and shape.sdss
psfImageConfig.algorithms.names = ["base_SdssCentroid", "base_CircularApertureFlux", "base_SdssShape"]
psfImageConfig.algorithms["base_CircularApertureFlux"].radii = [3.0]
psfImageConfig.validate()
task = SingleFrameMeasurementTask(schema, config=psfImageConfig)
sourceCat = SourceCatalog(schema)
gaussianWidth = 1.5 # Gaussian sigma for detection convolution
exposure.setPsf(algorithmsLib.DoubleGaussianPsf(11, 11, gaussianWidth))
ds.makeSources(sourceCat)
#
# Show us the Histogram
#
if display:
frame = 1
dispImage = mpsfImage.Factory(mpsfImage, afwGeom.BoxI(afwGeom.PointI(width, height),
afwGeom.ExtentI(width, height)),
afwImage.LOCAL)
ds9.mtv(dispImage,title="PSF Selection Image", frame=frame)
clumps = list() # List of clumps, to return
e = None # thrown exception
IzzMin = 1.0 # Minimum value for second moments
IzzMax = (self._xSize/8.0)**2 # Max value ... clump r < clumpImgSize/8
# diameter should be < 1/4 clumpImgSize
apFluxes = []
task.run(exposure, sourceCat) # notes that this is backwards for the new framework
for i, source in enumerate(sourceCat):
if source.getCentroidFlag():
continue
x, y = source.getX(), source.getY()
apFluxes.append(source.getApFlux())
val = mpsfImage.getImage().get(int(x) + width, int(y) + height)
psfClumpIxx = source.getIxx()
psfClumpIxy = source.getIxy()
psfClumpIyy = source.getIyy()
if display:
if i == 0:
ds9.pan(x, y, frame=frame)
ds9.dot("+", x, y, ctype=ds9.YELLOW, frame=frame)
ds9.dot("@:%g,%g,%g" % (psfClumpIxx, psfClumpIxy, psfClumpIyy), x, y,
ctype=ds9.YELLOW, frame=frame)
if psfClumpIxx < IzzMin or psfClumpIyy < IzzMin:
psfClumpIxx = max(psfClumpIxx, IzzMin)
psfClumpIyy = max(psfClumpIyy, IzzMin)
if display:
ds9.dot("@:%g,%g,%g" % (psfClumpIxx, psfClumpIxy, psfClumpIyy), x, y,
ctype=ds9.RED, frame=frame)
det = psfClumpIxx*psfClumpIyy - psfClumpIxy*psfClumpIxy
try:
a, b, c = psfClumpIyy/det, -psfClumpIxy/det, psfClumpIxx/det
except ZeroDivisionError:
a, b, c = 1e4, 0, 1e4
clumps.append(Clump(peak=val, x=x, y=y, a=a, b=b, c=c,
ixx=psfClumpIxx, ixy=psfClumpIxy, iyy=psfClumpIyy))
if len(clumps) == 0:
msg = "Failed to determine center of PSF clump"
if e:
msg += ": %s" % e
raise RuntimeError(msg)
# if it's all we got return it
if len(clumps) == 1:
return clumps
# which clump is the best?
# if we've undistorted the moments, stars should only have 1 clump
# use the apFlux from the clump measurement, and take the highest
# ... this clump has more psf star candidate neighbours than the others.
# get rid of any that are huge, and thus poorly defined
goodClumps = []
for clump in clumps:
if clump.ixx < IzzMax and clump.iyy < IzzMax:
goodClumps.append(clump)
# if culling > IzzMax cost us all clumps, we'll have to take what we have
if len(goodClumps) == 0:
goodClumps = clumps
# use the 'brightest' clump
iBestClump = numpy.argsort(apFluxes)[0]
clumps = [clumps[iBestClump]]
return clumps
def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo, background):
"""!Perform one iteration of detect, measure and estimate PSF
Performs the following operations:
- if config.doMeasurePsf or not exposure.hasPsf():
- install a simple PSF model (replacing the existing one, if need be)
- interpolate over cosmic rays with keepCRs=True
- estimate background and subtract it from the exposure
- detect, deblend and measure sources, and subtract a refined background model;
- if config.doMeasurePsf:
- measure PSF
@param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar)
The following changes are made:
- update or set psf
- update detection and cosmic ray mask planes
- subtract background
@param[in] exposureIdInfo ID info for exposure (an lsst.obs_base.ExposureIdInfo)
@param[in,out] background initial model of background already subtracted from exposure
(an lsst.afw.math.BackgroundList).
@return pipe_base Struct containing these fields, all from the final iteration
of detect sources, measure sources and estimate PSF:
- exposure characterized exposure; image is repaired by interpolating over cosmic rays,
mask is updated accordingly, and the PSF model is set
- sourceCat detected sources (an lsst.afw.table.SourceCatalog)
- background model of background subtracted from exposure (an lsst.afw.math.BackgroundList)
- psfCellSet spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet)
"""
# install a simple PSF model, if needed or wanted
if not exposure.hasPsf() or (self.config.doMeasurePsf and self.config.useSimplePsf):
self.log.warn("Source catalog detected and measured with placeholder or default PSF")
self.installSimplePsf.run(exposure=exposure)
# run repair, but do not interpolate over cosmic rays (do that elsewhere, with the final PSF model)
self.repair.run(exposure=exposure, keepCRs=True)
self.display("repair_iter", exposure=exposure)
if background is None:
background = BackgroundList()
sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId, exposureIdInfo.unusedBits)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
detRes = self.detection.run(table=table, exposure=exposure, doSmooth=True)
sourceCat = detRes.sources
if detRes.fpSets.background:
for bg in detRes.fpSets.background:
background.append(bg)
if self.config.doDeblend:
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(measCat=sourceCat, exposure=exposure, exposureId=exposureIdInfo.expId)
measPsfRes = pipeBase.Struct(cellSet=None)
if self.config.doMeasurePsf:
if self.measurePsf.usesMatches:
matches = self.ref_match.loadAndMatch(exposure=exposure, sourceCat=sourceCat).matches
else:
matches = None
measPsfRes = self.measurePsf.run(exposure=exposure, sources=sourceCat, matches=matches,
expId=exposureIdInfo.expId)
self.display("measure_iter", exposure=exposure, sourceCat=sourceCat)
return pipeBase.Struct(
exposure=exposure,
sourceCat=sourceCat,
background=background,
psfCellSet=measPsfRes.cellSet,
)
def run(self, exposure, exposureIdInfo=None, background=None,
icSourceCat=None):
"""!Calibrate an exposure (science image or coadd)
@param[in,out] exposure exposure to calibrate (an
lsst.afw.image.ExposureF or similar);
in:
- MaskedImage
- Psf
out:
- MaskedImage has background subtracted
- Wcs is replaced
- PhotoCalib is replaced
@param[in] exposureIdInfo ID info for exposure (an
lsst.obs.base.ExposureIdInfo) If not provided, returned
SourceCatalog IDs will not be globally unique.
@param[in,out] background background model already subtracted from
exposure (an lsst.afw.math.BackgroundList). May be None if no
background has been subtracted, though that is unusual for
calibration. A refined background model is output.
@param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
from which we can copy some fields.
@return pipe_base Struct containing these fields:
- exposure calibrate science exposure with refined WCS and PhotoCalib
- background model of background subtracted from exposure (an
lsst.afw.math.BackgroundList)
- sourceCat catalog of measured sources
- astromMatches list of source/refObj matches from the astrometry
solver
"""
# detect, deblend and measure sources
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
if background is None:
background = BackgroundList()
sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
exposureIdInfo.unusedBits)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
detRes = self.detection.run(table=table, exposure=exposure,
doSmooth=True)
sourceCat = detRes.sources
if detRes.fpSets.background:
for bg in detRes.fpSets.background:
background.append(bg)
if self.config.doSkySources:
skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId)
if skySourceFootprints:
for foot in skySourceFootprints:
s = sourceCat.addNew()
s.setFootprint(foot)
s.set(self.skySourceKey, True)
if self.config.doDeblend:
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(
measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId
)
if self.config.doApCorr:
self.applyApCorr.run(
catalog=sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap()
)
self.catalogCalculation.run(sourceCat)
self.setPrimaryFlags.run(sourceCat, includeDeblend=self.config.doDeblend)
if icSourceCat is not None and \
len(self.config.icSourceFieldsToCopy) > 0:
self.copyIcSourceFields(icSourceCat=icSourceCat,
sourceCat=sourceCat)
# TODO DM-11568: this contiguous check-and-copy could go away if we
# reserve enough space during SourceDetection and/or SourceDeblend.
# NOTE: sourceSelectors require contiguous catalogs, so ensure
# contiguity now, so views are preserved from here on.
if not sourceCat.isContiguous():
sourceCat = sourceCat.copy(deep=True)
# perform astrometry calibration:
# fit an improved WCS and update the exposure's WCS in place
astromMatches = None
matchMeta = None
if self.config.doAstrometry:
try:
astromRes = self.astrometry.run(
exposure=exposure,
sourceCat=sourceCat,
)
astromMatches = astromRes.matches
matchMeta = astromRes.matchMeta
except Exception as e:
if self.config.requireAstrometry:
raise
self.log.warn("Unable to perform astrometric calibration "
"(%s): attempting to proceed" % e)
# compute photometric calibration
if self.config.doPhotoCal:
try:
photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
exposure.setPhotoCalib(photoRes.photoCalib)
# TODO: reword this to phrase it in terms of the calibration factor?
self.log.info("Photometric zero-point: %f" %
photoRes.photoCalib.instFluxToMagnitude(1.0))
self.setMetadata(exposure=exposure, photoRes=photoRes)
except Exception as e:
if self.config.requirePhotoCal:
raise
self.log.warn("Unable to perform photometric calibration "
"(%s): attempting to proceed" % e)
self.setMetadata(exposure=exposure, photoRes=None)
if self.config.doInsertFakes:
self.insertFakes.run(exposure, background=background)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
detRes = self.detection.run(table=table, exposure=exposure,
doSmooth=True)
sourceCat = detRes.sources
if detRes.fpSets.background:
for bg in detRes.fpSets.background:
background.append(bg)
if self.config.doDeblend:
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(
measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId
)
if self.config.doApCorr:
self.applyApCorr.run(
catalog=sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap()
)
self.catalogCalculation.run(sourceCat)
if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
self.copyIcSourceFields(icSourceCat=icSourceCat,
sourceCat=sourceCat)
frame = getDebugFrame(self._display, "calibrate")
if frame:
displayAstrometry(
sourceCat=sourceCat,
exposure=exposure,
matches=astromMatches,
frame=frame,
pause=False,
)
return pipeBase.Struct(
exposure=exposure,
background=background,
sourceCat=sourceCat,
astromMatches=astromMatches,
matchMeta=matchMeta,
# These are duplicate entries with different names for use with
# gen3 middleware
outputExposure=exposure,
outputCat=sourceCat,
outputBackground=background,
)
def calibrate(self,
exposure,
exposureIdInfo=None,
background=None,
icSourceCat=None):
"""!Calibrate an exposure (science image or coadd)
@param[in,out] exposure exposure to calibrate (an
lsst.afw.image.ExposureF or similar);
in:
- MaskedImage
- Psf
out:
- MaskedImage has background subtracted
- Wcs is replaced
- Calib zero-point is set
@param[in] exposureIdInfo ID info for exposure (an
lsst.obs.base.ExposureIdInfo) If not provided, returned
SourceCatalog IDs will not be globally unique.
@param[in,out] background background model already subtracted from
exposure (an lsst.afw.math.BackgroundList). May be None if no
background has been subtracted, though that is unusual for
calibration. A refined background model is output.
@param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
from which we can copy some fields.
@return pipe_base Struct containing these fields:
- exposure calibrate science exposure with refined WCS and Calib
- background model of background subtracted from exposure (an
lsst.afw.math.BackgroundList)
- sourceCat catalog of measured sources
- astromMatches list of source/refObj matches from the astrometry
solver
"""
# detect, deblend and measure sources
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
if background is None:
background = BackgroundList()
sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
exposureIdInfo.unusedBits)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
if self.config.doInsertFakes:
self.insertFakes.run(exposure, background=background)
detRes = self.detection.run(table=table,
exposure=exposure,
doSmooth=True)
sourceCat = detRes.sources
if detRes.fpSets.background:
background.append(detRes.fpSets.background)
if self.config.doDeblend:
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId)
if self.config.doApCorr:
self.applyApCorr.run(catalog=sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap())
self.catalogCalculation.run(sourceCat)
if icSourceCat is not None and \
len(self.config.icSourceFieldsToCopy) > 0:
self.copyIcSourceFields(icSourceCat=icSourceCat,
sourceCat=sourceCat)
# perform astrometry calibration:
# fit an improved WCS and update the exposure's WCS in place
astromMatches = None
if self.config.doAstrometry:
try:
astromRes = self.astrometry.run(
exposure=exposure,
sourceCat=sourceCat,
)
astromMatches = astromRes.matches
except Exception as e:
if self.config.requireAstrometry:
raise
self.log.warn("Unable to perform astrometric calibration "
"(%s): attempting to proceed" % e)
# compute photometric calibration
if self.config.doPhotoCal:
try:
photoRes = self.photoCal.run(exposure, sourceCat=sourceCat)
exposure.getCalib().setFluxMag0(photoRes.calib.getFluxMag0())
self.log.info("Photometric zero-point: %f" %
photoRes.calib.getMagnitude(1.0))
self.setMetadata(exposure=exposure, photoRes=photoRes)
except Exception as e:
if self.config.requirePhotoCal:
raise
self.log.warn("Unable to perform photometric calibration "
"(%s): attempting to proceed" % e)
self.setMetadata(exposure=exposure, photoRes=None)
frame = getDebugFrame(self._display, "calibrate")
if frame:
displayAstrometry(
sourceCat=sourceCat,
exposure=exposure,
matches=astromMatches,
frame=frame,
pause=False,
)
return pipeBase.Struct(
exposure=exposure,
background=background,
sourceCat=sourceCat,
astromMatches=astromMatches,
)
def run(self,
fakeCat,
exposure,
wcs=None,
photoCalib=None,
exposureIdInfo=None,
icSourceCat=None,
sfdSourceCat=None):
"""Add fake sources to a calexp and then run detection, deblending and measurement.
Parameters
----------
fakeCat : `pandas.core.frame.DataFrame`
The catalog of fake sources to add to the exposure
exposure : `lsst.afw.image.exposure.exposure.ExposureF`
The exposure to add the fake sources to
wcs : `lsst.afw.geom.SkyWcs`
WCS to use to add fake sources
photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
Photometric calibration to be used to calibrate the fake sources
exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
icSourceCat : `lsst.afw.table.SourceCatalog`
Default : None
Catalog to take the information about which sources were used for calibration from.
sfdSourceCat : `lsst.afw.table.SourceCatalog`
Default : None
Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
Returns
-------
resultStruct : `lsst.pipe.base.struct.Struct`
contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
outputCat : `lsst.afw.table.source.source.SourceCatalog`
Notes
-----
Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
pixels.
Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
the calexp and the calexp with fakes included returned.
The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
this is then convolved with the PSF at that point.
If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
"""
if wcs is None:
wcs = exposure.getWcs()
if photoCalib is None:
photoCalib = exposure.getPhotoCalib()
self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
# detect, deblend and measure sources
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
exposureIdInfo.unusedBits)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
detRes = self.detection.run(table=table,
exposure=exposure,
doSmooth=True)
sourceCat = detRes.sources
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId)
self.applyApCorr.run(catalog=sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap())
self.catalogCalculation.run(sourceCat)
sourceCat = self.copyCalibrationFields(
icSourceCat, sourceCat, self.config.calibrationFieldsToCopy)
sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat,
self.config.srcFieldsToCopy)
resultStruct = pipeBase.Struct(outputExposure=exposure,
outputCat=sourceCat)
return resultStruct
def run(self, ccdExposure):
"""Mask negative pixels"""
ccd = ccdExposure.getDetector()
ccdExposure = self.convertIntToFloat(ccdExposure)
self.updateVariance(ccdExposure, ccd[0]) # Treating as having only a single amplifier
image = ccdExposure.getMaskedImage().getImage()
mask = ccdExposure.getMaskedImage().getMask()
bad = mask.getPlaneBitMask("BAD")
if False:
mask.getArray()[:] = numpy.where(image <= 0, bad, 0) # XXX this causes bad things to happen
#from lsst.afw.image.utils import clipImage
#clipImage(image,0,10)
#exit()
""" transfer wcs system to TAN """
matches = ReferenceMatchVector()
md = ccdExposure.getMetadata()
wcs = ccdExposure.getWcs()
refSchema = SimpleTable.makeMinimalSchema()
Point2DKey.addFields(refSchema, "centroid", "centroid position", "pixel")
refCatalog = SimpleCatalog(refSchema)
schema = SourceTable.makeMinimalSchema()
centroidKey = Point2DKey.addFields(schema, "centroid", "centroid position", "pixel")
imgCatalog = SourceCatalog(schema)
imgCatalog.defineCentroid("centroid")
# for i in numpy.linspace(10, md.get("ZNAXIS1")-10, 20):
# for j in numpy.linspace(10, md.get("ZNAXIS2")-10, 20):
for i in numpy.linspace(10, 4000, 10):
for j in numpy.linspace(10, 4000, 10):
imgcrd = Point2D(i,j)
skycrd = wcs.pixelToSky(afwGeom.Point2D(i, j))
# Create the reference catalog (with coordinates on the sky)
refSrc = refCatalog.addNew()
refSrc.setCoord(skycrd)
# Create the position catalog (with positions on the image)
imgSrc = imgCatalog.addNew()
imgSrc.set(centroidKey, imgcrd)
# matches
matches.push_back(ReferenceMatch(refSrc, imgSrc, float("NaN")))
# make initial wcs
refPix = afwGeom.Point2D(0.5*ccdExposure.getWidth(), 0.5*ccdExposure.getHeight())
refSky = wcs.pixelToSky(refPix)
xPixelScale = yPixelScale = (0.2*afwGeom.arcseconds).asDegrees()
initanWcs = afwImage.makeWcs(refSky, refPix, xPixelScale, 0.0, 0.0, yPixelScale)
# obtain modified wcs with matched catalogs
fitter = FitTanSipWcsTask()
fitRes = fitter.fitWcs(
matches = matches,
initWcs = initanWcs,
refCat = refCatalog,
sourceCat = imgCatalog,
)
ccdExposure.setWcs(fitRes.wcs)
""" Set zero point, ZP error, exptime """
zp = md.get("MAGZPT")
ccdExposure.getCalib().setFluxMag0(10.0**(0.4*zp))
return lsst.pipe.base.Struct(exposure=ccdExposure)
def run(self,
dataRef,
exposure=None,
background=None,
doUnpersist=True,
doPsf=True,
doApCorr=True,
doWrite=True,
doCalc=True):
self.log.info("gotoCharTask Processing %s" % (dataRef.dataId))
if doUnpersist:
if exposure is not None or background is not None:
raise RuntimeError(
"doUnpersist true; exposure and background must be None")
exposure = dataRef.get("postISRCCD", immediate=True)
elif exposure is None:
raise RuntimeError("doUnpersist false; exposure must be provided")
exposureIdInfo = dataRef.get("expIdInfo")
if not exposure.hasPsf():
self.log.warn("Using SimplePsf for astrometry source detection")
self.installSimplePsf.run(exposure=exposure)
#Repair cosmic rays
self.repair.run(exposure=exposure, keepCRs=True)
# subtract an initial estimate of background level
background = self.background.run(exposure).background
#Table schema needs to be set up prior to detection:
sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
exposureIdInfo.unusedBits)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
#Perform detection
sourceCat = self.detection.run(table=table,
exposure=exposure,
doSmooth=True).sources
#Perform measurement
self.measurement.run(measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId)
if self.config.doEarlyAstrometry:
astromRes = self.astrometry.run(exposure=exposure,
sourceCat=sourceCat)
measPsfRes = pipeBase.Struct(cellSet=None)
if doPsf and self.config.doMeasurePsf:
if self.measurePsf.usesMatches:
matches = self.ref_match.loadAndMatch(
exposure=exposure, sourceCat=sourceCat).matches
else:
matches = None
psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1
for i in range(psfIterations):
measPsfRes = self.measurePsf.run(exposure=exposure,
sources=sourceCat,
matches=matches,
expId=exposureIdInfo.expId)
# perform final repair with final PSF
self.repair.run(exposure=exposure)
if doApCorr and self.config.doApCorr:
apCorrMap = self.measureApCorr.run(exposure=exposure,
catalog=sourceCat).apCorrMap
exposure.getInfo().setApCorrMap(apCorrMap)
self.applyApCorr.run(catalog=sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap())
if doCalc:
self.catalogCalculation.run(sourceCat)
if doWrite and self.config.doWrite:
dataRef.put(sourceCat, "icSrc")
if self.config.doWriteExposure:
dataRef.put(exposure, "icExp")
dataRef.put(background, "icExpBackground")
return pipeBase.Struct(
exposure=exposure,
sourceCat=sourceCat,
background=background,
psfCellSet=measPsfRes.cellSet,
)
def measure(self, exposure, exposureIdInfo, sourceCat, allowApCorr=True):
"""Measure sources
@param[in,out] exposure exposure to process. Background must already be subtracted
to reasonable accuracy, else detection will fail.
Set apCorrMap if measuring aperture correction.
@param[in] exposureIdInfo ID info for exposure (an lsst.daf.butlerUtils.ExposureIdInfo)
@param[in,out] background background model to be modified (an lsst.afw.math.BackgroundList),
or None to create a new background model
@param[in] allowApCorr allow measuring and applying aperture correction?
"""
if self.config.doMeasureApCorr or allowApCorr:
# NOTE: The measurement task has a serious misfeature when it comes to applying
# aperture correction (one that will be fixed in DM-5877): it applies aperture
# correction after performing ALL measurements, regardless of their execution
# order. As a result, no plugins, in particular those whose measurements rely
# on aperture corrected fluxes, were getting aperture-corrected fluxes.
#
# To work around this, we measure fluxes and apply aperture correction where
# appropriate in three phases (as described step-by-step below):
# 1) run plugins up to order APCORR_ORDER to measure all fluxes
# 2) apply aperture correction to all appropriate fluxes
# 3) run the remaining plugins with aperture correction disabled
# (to avoid applying the correction twice)
# 1) run plugins with order up to APCORR_ORDER to measure all fluxes
self.measurement.run(
measCat = sourceCat,
exposure = exposure,
exposureId = exposureIdInfo.expId,
endOrder = BasePlugin.APCORR_ORDER,
)
sourceCat.sort(SourceTable.getParentKey())
if self.config.doMeasureApCorr and allowApCorr:
# measure the aperture correction map
apCorrMap = self.measureApCorr.run(exposure=exposure, catalog=sourceCat).apCorrMap
exposure.getInfo().setApCorrMap(apCorrMap)
# 2) run APCORR_ORDER only to apply the aperture correction to the measured
# fluxes. The effect of this step is simply to apply the aperture correction
# (using the apCorrMap measured above or perhaps in a previous stage) to any
# flux measurements present whose plugins were registered with shouldApCorr=True
# (no actual plugins are run in this step)
self.measurement.run(
measCat = sourceCat,
exposure = exposure,
exposureId = exposureIdInfo.expId,
beginOrder = BasePlugin.APCORR_ORDER,
endOrder = BasePlugin.APCORR_ORDER + 1,
allowApCorr = allowApCorr,
)
# 3) run the remaining APCORR_ORDER+1 plugins, whose measurements should be
# performed on aperture corrected fluxes, disallowing apCorr (to avoid applying it
# more than once, noting that, when allowApCorr=True, self._applyApCorrIfWanted()
# in meas_base's SingleFrameMeasurement runs with no regard to beginOrder, so
# running with allowApCorr=True in any call to measurement.run subsequent to step 2)
# would erroneously re-apply the aperture correction to already-corrected fluxes)
self.measurement.run(
measCat = sourceCat,
exposure = exposure,
exposureId = exposureIdInfo.expId,
beginOrder = BasePlugin.APCORR_ORDER + 1,
allowApCorr = False,
)
else:
self.measurement.run(
measCat = sourceCat,
exposure = exposure,
exposureId = exposureIdInfo.expId,
allowApCorr = False,
)
def calculateThreshold(self, exposure, seed, sigma=None):
"""Calculate new threshold
This is the main functional addition to the vanilla
`SourceDetectionTask`.
We identify sky objects and perform forced PSF photometry on
them. Using those PSF flux measurements and estimated errors,
we set the threshold so that the stdev of the measurements
matches the median estimated error.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
Exposure on which we're detecting sources.
seed : `int`
RNG seed to use for finding sky objects.
sigma : `float`, optional
Gaussian sigma of smoothing kernel; if not provided,
will be deduced from the exposure's PSF.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with components:
- ``multiplicative``: multiplicative factor to be applied to the
configured detection threshold (`float`).
- ``additive``: additive factor to be applied to the background
level (`float`).
"""
# Make a catalog of sky objects
fp = self.skyObjects.run(exposure.maskedImage.mask, seed)
skyFootprints = FootprintSet(exposure.getBBox())
skyFootprints.setFootprints(fp)
table = SourceTable.make(self.skyMeasurement.schema)
catalog = SourceCatalog(table)
catalog.reserve(len(skyFootprints.getFootprints()))
skyFootprints.makeSources(catalog)
key = catalog.getCentroidKey()
for source in catalog:
peaks = source.getFootprint().getPeaks()
assert len(peaks) == 1
source.set(key, peaks[0].getF())
source.updateCoord(exposure.getWcs())
# Forced photometry on sky objects
self.skyMeasurement.run(catalog, exposure, catalog, exposure.getWcs())
# Calculate new threshold
fluxes = catalog["base_PsfFlux_instFlux"]
area = catalog["base_PsfFlux_area"]
bg = catalog["base_LocalBackground_instFlux"]
good = (~catalog["base_PsfFlux_flag"] & ~catalog["base_LocalBackground_flag"] &
np.isfinite(fluxes) & np.isfinite(area) & np.isfinite(bg))
if good.sum() < self.config.minNumSources:
self.log.warn("Insufficient good flux measurements (%d < %d) for dynamic threshold calculation",
good.sum(), self.config.minNumSources)
return Struct(multiplicative=1.0, additive=0.0)
bgMedian = np.median((fluxes/area)[good])
lq, uq = np.percentile((fluxes - bg*area)[good], [25.0, 75.0])
stdevMeas = 0.741*(uq - lq)
medianError = np.median(catalog["base_PsfFlux_instFluxErr"][good])
return Struct(multiplicative=medianError/stdevMeas, additive=bgMedian)
def calculateThreshold(self, exposure, seed, sigma=None):
"""Calculate new threshold
This is the main functional addition to the vanilla
`SourceDetectionTask`.
We identify sky objects and perform forced PSF photometry on
them. Using those PSF flux measurements and estimated errors,
we set the threshold so that the stdev of the measurements
matches the median estimated error.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
Exposure on which we're detecting sources.
seed : `int`
RNG seed to use for finding sky objects.
sigma : `float`, optional
Gaussian sigma of smoothing kernel; if not provided,
will be deduced from the exposure's PSF.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with components:
- ``multiplicative``: multiplicative factor to be applied to the
configured detection threshold (`float`).
- ``additive``: additive factor to be applied to the background
level (`float`).
"""
# Make a catalog of sky objects
fp = self.skyObjects.run(exposure.maskedImage.mask, seed)
skyFootprints = FootprintSet(exposure.getBBox())
skyFootprints.setFootprints(fp)
table = SourceTable.make(self.skyMeasurement.schema)
catalog = SourceCatalog(table)
catalog.reserve(len(skyFootprints.getFootprints()))
skyFootprints.makeSources(catalog)
key = catalog.getCentroidKey()
for source in catalog:
peaks = source.getFootprint().getPeaks()
assert len(peaks) == 1
source.set(key, peaks[0].getF())
source.updateCoord(exposure.getWcs())
# Forced photometry on sky objects
self.skyMeasurement.run(catalog, exposure, catalog, exposure.getWcs())
# Calculate new threshold
fluxes = catalog["base_PsfFlux_instFlux"]
area = catalog["base_PsfFlux_area"]
bg = catalog["base_LocalBackground_instFlux"]
good = (~catalog["base_PsfFlux_flag"] & ~catalog["base_LocalBackground_flag"] &
np.isfinite(fluxes) & np.isfinite(area) & np.isfinite(bg))
if good.sum() < self.config.minNumSources:
self.log.warn("Insufficient good flux measurements (%d < %d) for dynamic threshold calculation",
good.sum(), self.config.minNumSources)
return Struct(multiplicative=1.0, additive=0.0)
bgMedian = np.median((fluxes/area)[good])
lq, uq = np.percentile((fluxes - bg*area)[good], [25.0, 75.0])
stdevMeas = 0.741*(uq - lq)
medianError = np.median(catalog["base_PsfFlux_instFluxErr"][good])
return Struct(multiplicative=medianError/stdevMeas, additive=bgMedian)
def getClumps(self, sigma=1.0, display=False):
if self._num <= 0:
raise RuntimeError("No candidate PSF sources")
psfImage = self.getImage()
#
# Embed psfImage into a larger image so we can smooth when measuring it
#
width, height = psfImage.getWidth(), psfImage.getHeight()
largeImg = psfImage.Factory(afwGeom.ExtentI(2 * width, 2 * height))
largeImg.set(0)
bbox = afwGeom.BoxI(afwGeom.PointI(width, height),
afwGeom.ExtentI(width, height))
largeImg.assign(psfImage, bbox, afwImage.LOCAL)
#
# Now measure that image, looking for the highest peak. Start by building an Exposure
#
msk = afwImage.MaskU(largeImg.getDimensions())
msk.set(0)
var = afwImage.ImageF(largeImg.getDimensions())
var.set(1)
mpsfImage = afwImage.MaskedImageF(largeImg, msk, var)
mpsfImage.setXY0(afwGeom.PointI(-width, -height))
del msk
del var
exposure = afwImage.makeExposure(mpsfImage)
#
# Next run an object detector
#
maxVal = afwMath.makeStatistics(psfImage, afwMath.MAX).getValue()
threshold = maxVal - sigma * math.sqrt(maxVal)
if threshold <= 0.0:
threshold = maxVal
threshold = afwDetection.Threshold(threshold)
ds = afwDetection.FootprintSet(mpsfImage, threshold, "DETECTED")
#
# And measure it. This policy isn't the one we use to measure
# Sources, it's only used to characterize this PSF histogram
#
schema = SourceTable.makeMinimalSchema()
psfImageConfig = SingleFrameMeasurementConfig()
psfImageConfig.slots.centroid = "base_SdssCentroid"
psfImageConfig.plugins["base_SdssCentroid"].doFootprintCheck = False
psfImageConfig.slots.psfFlux = None # "base_PsfFlux"
psfImageConfig.slots.apFlux = "base_CircularApertureFlux_3_0"
psfImageConfig.slots.modelFlux = None
psfImageConfig.slots.instFlux = None
psfImageConfig.slots.calibFlux = None
psfImageConfig.slots.shape = "base_SdssShape"
# Formerly, this code had centroid.sdss, flux.psf, flux.naive,
# flags.pixel, and shape.sdss
psfImageConfig.algorithms.names = [
"base_SdssCentroid", "base_CircularApertureFlux", "base_SdssShape"
]
psfImageConfig.algorithms["base_CircularApertureFlux"].radii = [3.0]
psfImageConfig.validate()
task = SingleFrameMeasurementTask(schema, config=psfImageConfig)
sourceCat = SourceCatalog(schema)
gaussianWidth = 1.5 # Gaussian sigma for detection convolution
exposure.setPsf(algorithmsLib.DoubleGaussianPsf(11, 11, gaussianWidth))
ds.makeSources(sourceCat)
#
# Show us the Histogram
#
if display:
frame = 1
dispImage = mpsfImage.Factory(
mpsfImage,
afwGeom.BoxI(afwGeom.PointI(width, height),
afwGeom.ExtentI(width, height)), afwImage.LOCAL)
ds9.mtv(dispImage, title="PSF Selection Image", frame=frame)
clumps = list() # List of clumps, to return
e = None # thrown exception
IzzMin = 1.0 # Minimum value for second moments
IzzMax = (
self._xSize /
8.0)**2 # Max value ... clump radius should be < clumpImgSize/8
apFluxes = []
task.run(
sourceCat,
exposure) # notes that this is backwards for the new framework
for i, source in enumerate(sourceCat):
if source.getCentroidFlag():
continue
x, y = source.getX(), source.getY()
apFluxes.append(source.getApFlux())
val = mpsfImage.getImage().get(int(x) + width, int(y) + height)
psfClumpIxx = source.getIxx()
psfClumpIxy = source.getIxy()
psfClumpIyy = source.getIyy()
if display:
if i == 0:
ds9.pan(x, y, frame=frame)
ds9.dot("+", x, y, ctype=ds9.YELLOW, frame=frame)
ds9.dot("@:%g,%g,%g" % (psfClumpIxx, psfClumpIxy, psfClumpIyy),
x,
y,
ctype=ds9.YELLOW,
frame=frame)
if psfClumpIxx < IzzMin or psfClumpIyy < IzzMin:
psfClumpIxx = max(psfClumpIxx, IzzMin)
psfClumpIyy = max(psfClumpIyy, IzzMin)
if display:
ds9.dot("@:%g,%g,%g" %
(psfClumpIxx, psfClumpIxy, psfClumpIyy),
x,
y,
ctype=ds9.RED,
frame=frame)
det = psfClumpIxx * psfClumpIyy - psfClumpIxy * psfClumpIxy
try:
a, b, c = psfClumpIyy / det, -psfClumpIxy / det, psfClumpIxx / det
except ZeroDivisionError:
a, b, c = 1e4, 0, 1e4
clumps.append(
Clump(peak=val,
x=x,
y=y,
a=a,
b=b,
c=c,
ixx=psfClumpIxx,
ixy=psfClumpIxy,
iyy=psfClumpIyy))
if len(clumps) == 0:
msg = "Failed to determine center of PSF clump"
if e:
msg += ": %s" % e
raise RuntimeError(msg)
# if it's all we got return it
if len(clumps) == 1:
return clumps
# which clump is the best?
# if we've undistorted the moments, stars should only have 1 clump
# use the apFlux from the clump measurement, and take the highest
# ... this clump has more psf star candidate neighbours than the others.
# get rid of any that are huge, and thus poorly defined
goodClumps = []
for clump in clumps:
if clump.ixx < IzzMax and clump.iyy < IzzMax:
goodClumps.append(clump)
# if culling > IzzMax cost us all clumps, we'll have to take what we have
if len(goodClumps) == 0:
goodClumps = clumps
# use the 'brightest' clump
iBestClump = numpy.argsort(apFluxes)[0]
clumps = [clumps[iBestClump]]
return clumps
def run(self,
exposure=None,
exposureIdInfo=None,
background=None,
doPsf=True,
doApCorr=True,
doWrite=True,
doCalc=True):
if not exposure.hasPsf():
self.log.warn("Using SimplePsf for astrometry source detection")
self.installSimplePsf.run(exposure=exposure)
#Repair cosmic rays
#self.repair.run(exposure=exposure, keepCRs=True)
# subtract an initial estimate of background level
background = self.background.run(exposure).background
#Table schema needs to be set up prior to detection:
sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
exposureIdInfo.unusedBits)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
#Perform detection
sourceCat = self.detection.run(table=table,
exposure=exposure,
doSmooth=True).sources
#Perform measurement
self.measurement.run(measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId)
if self.config.doEarlyAstrometry:
astromRes = self.astrometry.run(exposure=exposure,
sourceCat=sourceCat)
measPsfRes = pipeBase.Struct(cellSet=None)
if doPsf and self.config.doMeasurePsf:
if self.measurePsf.usesMatches:
matches = self.ref_match.loadAndMatch(
exposure=exposure, sourceCat=sourceCat).matches
else:
matches = None
psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1
for i in range(psfIterations):
measPsfRes = self.measurePsf.run(exposure=exposure,
sources=sourceCat,
matches=matches,
expId=exposureIdInfo.expId)
# perform final repair with final PSF
self.repair.run(exposure=exposure)
if doApCorr and self.config.doApCorr:
apCorrMap = self.measureApCorr.run(exposure=exposure,
catalog=sourceCat).apCorrMap
exposure.getInfo().setApCorrMap(apCorrMap)
self.applyApCorr.run(catalog=sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap())
if doCalc:
self.catalogCalculation.run(sourceCat)
return pipeBase.Struct(
exposure=exposure,
sourceCat=sourceCat,
background=background,
psfCellSet=measPsfRes.cellSet,
)
def detectMeasureAndEstimatePsf(self, exposure, exposureIdInfo,
background):
"""!Perform one iteration of detect, measure and estimate PSF
Performs the following operations:
- if config.doMeasurePsf or not exposure.hasPsf():
- install a simple PSF model (replacing the existing one, if need be)
- interpolate over cosmic rays with keepCRs=True
- estimate background and subtract it from the exposure
- detect, deblend and measure sources, and subtract a refined background model;
- if config.doMeasurePsf:
- measure PSF
@param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar)
The following changes are made:
- update or set psf
- update detection and cosmic ray mask planes
- subtract background
@param[in] exposureIdInfo ID info for exposure (an lsst.obs_base.ExposureIdInfo)
@param[in,out] background initial model of background already subtracted from exposure
(an lsst.afw.math.BackgroundList).
@return pipe_base Struct containing these fields, all from the final iteration
of detect sources, measure sources and estimate PSF:
- exposure characterized exposure; image is repaired by interpolating over cosmic rays,
mask is updated accordingly, and the PSF model is set
- sourceCat detected sources (an lsst.afw.table.SourceCatalog)
- background model of background subtracted from exposure (an lsst.afw.math.BackgroundList)
- psfCellSet spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet)
"""
# install a simple PSF model, if needed or wanted
if not exposure.hasPsf() or (self.config.doMeasurePsf
and self.config.useSimplePsf):
self.log.warn(
"Source catalog detected and measured with placeholder or default PSF"
)
self.installSimplePsf.run(exposure=exposure)
# run repair, but do not interpolate over cosmic rays (do that elsewhere, with the final PSF model)
self.repair.run(exposure=exposure, keepCRs=True)
self.display("repair_iter", exposure=exposure)
if background is None:
background = BackgroundList()
sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
exposureIdInfo.unusedBits)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
detRes = self.detection.run(table=table,
exposure=exposure,
doSmooth=True)
sourceCat = detRes.sources
if detRes.fpSets.background:
for bg in detRes.fpSets.background:
background.append(bg)
if self.config.doDeblend:
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId)
measPsfRes = pipeBase.Struct(cellSet=None)
if self.config.doMeasurePsf:
if self.measurePsf.usesMatches:
matches = self.ref_match.loadAndMatch(
exposure=exposure, sourceCat=sourceCat).matches
else:
matches = None
measPsfRes = self.measurePsf.run(exposure=exposure,
sources=sourceCat,
matches=matches,
expId=exposureIdInfo.expId)
self.display("measure_iter", exposure=exposure, sourceCat=sourceCat)
return pipeBase.Struct(
exposure=exposure,
sourceCat=sourceCat,
background=background,
psfCellSet=measPsfRes.cellSet,
)
def execute(self, dataRef):
"""!Characterize a science image
@param dataRef: butler data reference
@return a pipeBase Struct containing the results
"""
self.log.info("Performing Super CharacterizeImage on sensor data ID %s" % (dataRef.dataId,))
self.log.info("Reading input data using dataRef")
inputData = self.read_input_data(dataRef)
self.log.info("Running operations. The run() method should not take anything Butler")
result = CharacterizeImageTask.characterize(CharacterizeImageTask(config=self.config, log=self.log, schema=SourceTable.makeMinimalSchema()),
**inputData.getDict())
self.log.info("Writing output data using dataRef")
self.write_output_data(dataRef, result)
return result