def makeIdFactory(self, dataRef):
"""Return an IdFactory for setting the detection identifiers
The actual parameters used in the IdFactory are provided by
the butler (through the provided data reference.
"""
info = ExposureIdInfo(
int(dataRef.get(self.config.coaddName + datasetName)),
dataRef.get(self.config.coaddName + datasetName + "_bits"))
return info.makeSourceIdFactory()
def makeIdFactory(self, dataRef):
"""Return an IdFactory for setting the detection identifiers
The actual parameters used in the IdFactory are provided by
the butler (through the provided data reference.
"""
expId = getGen3CoaddExposureId(dataRef, coaddName=self.config.coaddName, includeBand=includeBand,
log=self.log)
info = ExposureIdInfo(expId, dataRef.get(self.config.coaddName + datasetName + "_bits"))
return info.makeSourceIdFactory()
def makeIdFactory(self, dataRef, exposureId):
"""Create an object that generates globally unique source IDs.
Source IDs are created based on a per-CCD ID and the ID of the CCD
itself.
Parameters
----------
dataRef : `lsst.daf.persistence.ButlerDataRef`
Butler data reference. The "CoaddId_bits" and "CoaddId" datasets
are accessed. The data ID must have tract and patch keys.
"""
# With the default configuration, this IdFactory doesn't do anything,
# because the IDs it generates are immediately overwritten by the ID
# from the reference catalog (since that's in
# config.measurement.copyColumns). But we create one here anyway, to
# allow us to revert back to the old behavior of generating new forced
# source IDs, just by renaming the ID in config.copyColumns to
# "object_id".
exposureIdInfo = ExposureIdInfo(
exposureId, dataRef.get(self.config.coaddName + "CoaddId_bits"))
return exposureIdInfo.makeSourceIdFactory()
def run(self, exposure, exposureIdInfo=None):
if not exposure.hasPsf():
self.installSimplePsf.run(exposure=exposure)
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
try:
self.repair.run(exposure=exposure, keepCRs=True)
except LengthError:
self.log.info("Skipping cosmic ray detection: Too many CR pixels (max %0.f)" % self.repair.cosmicray.nCrPixelMax)
sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
filtered = maximum_filter(exposure.getImage().array, size=self.config.maximumFilterBoxWidth)
detected = (filtered == exposure.getImage().getArray()) & (filtered > self.config.thresholdValue)
detectedImage = afwImage.ImageF(detected.astype(np.float32))
fps = afwDetect.FootprintSet(detectedImage, afwDetect.Threshold(0.5))
fp_ctrl = afwDetect.FootprintControl(True, True)
fps = afwDetect.FootprintSet(fps, self.config.footprintGrowValue, fp_ctrl)
sources = afwTable.SourceCatalog(table)
fps.makeSources(sources)
self.measurement.run(measCat=sources, exposure=exposure, exposureId=exposureIdInfo.expId)
self.catalogCalculation.run(sources)
## Add metadata to source catalog
md = exposure.getMetadata()
sources.getMetadata().add("BOTXCAM", md["BOTXCAM"])
sources.getMetadata().add("BOTYCAM", md["BOTYCAM"])
self.display("measure", exposure=exposure, sourceCat=sources)
return pipeBase.Struct(sourceCat=sources)
def run(self, diaSources, tractPatchId, skymapBits):
"""Associate DiaSources into a collection of DiaObjects using a
brute force matching algorithm.
Reproducible is for the same input data is assured by ordering the
DiaSource data by ccdVisit ordering.
Parameters
----------
diaSources : `pandas.DataFrame`
DiaSources grouped by CcdVisitId to spatially associate into
DiaObjects.
tractPatchId : `int`
Unique identifier for the tract patch.
skymapBits : `int`
Maximum number of bits used the ``tractPatchId`` integer
identifier.
Returns
-------
results : `lsst.pipe.base.Struct`
Results struct with attributes:
``assocDiaSources``
Table of DiaSources with updated values for the DiaObjects
they are spatially associated to (`pandas.DataFrame`).
``diaObjects``
Table of DiaObjects from matching DiaSources
(`pandas.DataFrame`).
"""
# Expected indexes include diaSourceId or meaningless range index
# If meaningless range index, drop it, else keep it.
doDropIndex = diaSources.index.names[0] is None
diaSources.reset_index(inplace=True, drop=doDropIndex)
# Sort by ccdVisit and diaSourceId to get a reproducible ordering for
# the association.
diaSources.set_index(["ccdVisitId", "diaSourceId"], inplace=True)
# Empty lists to store matching and location data.
diaObjectCat = []
diaObjectCoords = []
healPixIndices = []
# Create Id factory and catalog for creating DiaObjectIds.
exposureIdInfo = ExposureIdInfo(tractPatchId, skymapBits)
idFactory = exposureIdInfo.makeSourceIdFactory()
idCat = afwTable.SourceCatalog(
afwTable.SourceTable.make(afwTable.SourceTable.makeMinimalSchema(),
idFactory))
for ccdVisit in diaSources.index.levels[0]:
# For the first ccdVisit, just copy the DiaSource info into the
# diaObject data to create the first set of Objects.
ccdVisitSources = diaSources.loc[ccdVisit]
if len(diaObjectCat) == 0:
for diaSourceId, diaSrc in ccdVisitSources.iterrows():
self.addNewDiaObject(diaSrc,
diaSources,
ccdVisit,
diaSourceId,
diaObjectCat,
idCat,
diaObjectCoords,
healPixIndices)
continue
# Temp list to store DiaObjects already used for this ccdVisit.
usedMatchIndicies = []
# Run over subsequent data.
for diaSourceId, diaSrc in ccdVisitSources.iterrows():
# Find matches.
matchResult = self.findMatches(diaSrc["ra"],
diaSrc["decl"],
2*self.config.tolerance,
healPixIndices,
diaObjectCat)
dists = matchResult.dists
matches = matchResult.matches
# Create a new DiaObject if no match found.
if dists is None:
self.addNewDiaObject(diaSrc,
diaSources,
ccdVisit,
diaSourceId,
diaObjectCat,
idCat,
diaObjectCoords,
healPixIndices)
continue
# If matched, update catalogs and arrays.
if np.min(dists) < np.deg2rad(self.config.tolerance/3600):
matchDistArg = np.argmin(dists)
matchIndex = matches[matchDistArg]
# Test to see if the DiaObject has been used.
if np.isin([matchIndex], usedMatchIndicies).sum() < 1:
self.updateCatalogs(matchIndex,
diaSrc,
diaSources,
ccdVisit,
diaSourceId,
diaObjectCat,
diaObjectCoords,
healPixIndices)
usedMatchIndicies.append(matchIndex)
# If the matched DiaObject has already been used, create a
# new DiaObject for this DiaSource.
else:
self.addNewDiaObject(diaSrc,
diaSources,
ccdVisit,
diaSourceId,
diaObjectCat,
idCat,
diaObjectCoords,
healPixIndices)
# Create new DiaObject if no match found within the matching
# tolerance.
else:
self.addNewDiaObject(diaSrc,
diaSources,
ccdVisit,
diaSourceId,
diaObjectCat,
idCat,
diaObjectCoords,
healPixIndices)
# Drop indices before returning associated diaSource catalog.
diaSources.reset_index(inplace=True)
diaSources.set_index("diaSourceId", inplace=True, verify_integrity=True)
objs = diaObjectCat if diaObjectCat else np.array([], dtype=[('diaObjectId', 'int64'),
('ra', 'float64'),
('decl', 'float64'),
('nDiaSources', 'int64')])
diaObjects = pd.DataFrame(data=objs)
if "diaObjectId" in diaObjects.columns:
diaObjects.set_index("diaObjectId", inplace=True, verify_integrity=True)
return pipeBase.Struct(
assocDiaSources=diaSources,
diaObjects=diaObjects)
def run(self,
exposure,
exposureIdInfo=None,
background=None,
icSourceCat=None):
"""!Calibrate an exposure (science image or coadd)
@param[in,out] exposure exposure to calibrate (an
lsst.afw.image.ExposureF or similar);
in:
- MaskedImage
- Psf
out:
- MaskedImage has background subtracted
- Wcs is replaced
- PhotoCalib is replaced
@param[in] exposureIdInfo ID info for exposure (an
lsst.obs.base.ExposureIdInfo) If not provided, returned
SourceCatalog IDs will not be globally unique.
@param[in,out] background background model already subtracted from
exposure (an lsst.afw.math.BackgroundList). May be None if no
background has been subtracted, though that is unusual for
calibration. A refined background model is output.
@param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
from which we can copy some fields.
@return pipe_base Struct containing these fields:
- exposure calibrate science exposure with refined WCS and PhotoCalib
- background model of background subtracted from exposure (an
lsst.afw.math.BackgroundList)
- sourceCat catalog of measured sources
- astromMatches list of source/refObj matches from the astrometry
solver
"""
# detect, deblend and measure sources
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
if background is None:
background = BackgroundList()
sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
detRes = self.detection.run(table=table,
exposure=exposure,
doSmooth=True)
sourceCat = detRes.sources
if detRes.fpSets.background:
for bg in detRes.fpSets.background:
background.append(bg)
if self.config.doSkySources:
skySourceFootprints = self.skySources.run(
mask=exposure.mask, seed=exposureIdInfo.expId)
if skySourceFootprints:
for foot in skySourceFootprints:
s = sourceCat.addNew()
s.setFootprint(foot)
s.set(self.skySourceKey, True)
if self.config.doDeblend:
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId)
if self.config.doApCorr:
self.applyApCorr.run(catalog=sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap())
self.catalogCalculation.run(sourceCat)
self.setPrimaryFlags.run(sourceCat)
if icSourceCat is not None and \
len(self.config.icSourceFieldsToCopy) > 0:
self.copyIcSourceFields(icSourceCat=icSourceCat,
sourceCat=sourceCat)
# TODO DM-11568: this contiguous check-and-copy could go away if we
# reserve enough space during SourceDetection and/or SourceDeblend.
# NOTE: sourceSelectors require contiguous catalogs, so ensure
# contiguity now, so views are preserved from here on.
if not sourceCat.isContiguous():
sourceCat = sourceCat.copy(deep=True)
# perform astrometry calibration:
# fit an improved WCS and update the exposure's WCS in place
astromMatches = None
matchMeta = None
if self.config.doAstrometry:
try:
astromRes = self.astrometry.run(
exposure=exposure,
sourceCat=sourceCat,
)
astromMatches = astromRes.matches
matchMeta = astromRes.matchMeta
except Exception as e:
if self.config.requireAstrometry:
raise
self.log.warn("Unable to perform astrometric calibration "
"(%s): attempting to proceed" % e)
# compute photometric calibration
if self.config.doPhotoCal:
try:
photoRes = self.photoCal.run(exposure,
sourceCat=sourceCat,
expId=exposureIdInfo.expId)
exposure.setPhotoCalib(photoRes.photoCalib)
# TODO: reword this to phrase it in terms of the calibration factor?
self.log.info("Photometric zero-point: %f" %
photoRes.photoCalib.instFluxToMagnitude(1.0))
self.setMetadata(exposure=exposure, photoRes=photoRes)
except Exception as e:
if self.config.requirePhotoCal:
raise
self.log.warn("Unable to perform photometric calibration "
"(%s): attempting to proceed" % e)
self.setMetadata(exposure=exposure, photoRes=None)
if self.config.doInsertFakes:
self.insertFakes.run(exposure, background=background)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
detRes = self.detection.run(table=table,
exposure=exposure,
doSmooth=True)
sourceCat = detRes.sources
if detRes.fpSets.background:
for bg in detRes.fpSets.background:
background.append(bg)
if self.config.doDeblend:
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId)
if self.config.doApCorr:
self.applyApCorr.run(
catalog=sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap())
self.catalogCalculation.run(sourceCat)
if icSourceCat is not None and len(
self.config.icSourceFieldsToCopy) > 0:
self.copyIcSourceFields(icSourceCat=icSourceCat,
sourceCat=sourceCat)
frame = getDebugFrame(self._display, "calibrate")
if frame:
displayAstrometry(
sourceCat=sourceCat,
exposure=exposure,
matches=astromMatches,
frame=frame,
pause=False,
)
return pipeBase.Struct(
exposure=exposure,
background=background,
sourceCat=sourceCat,
astromMatches=astromMatches,
matchMeta=matchMeta,
# These are duplicate entries with different names for use with
# gen3 middleware
outputExposure=exposure,
outputCat=sourceCat,
outputBackground=background,
)
