def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
returnMaxBits=True)
inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
if self.config.doAstrometry:
refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
for ref in inputRefs.astromRefCat],
refCats=inputs.pop('astromRefCat'),
config=self.config.astromRefObjLoader, log=self.log)
self.pixelMargin = refObjLoader.config.pixelMargin
self.astrometry.setRefObjLoader(refObjLoader)
if self.config.doPhotoCal:
photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
for ref in inputRefs.photoRefCat],
refCats=inputs.pop('photoRefCat'),
config=self.config.photoRefObjLoader,
log=self.log)
self.pixelMargin = photoRefObjLoader.config.pixelMargin
self.photoCal.match.setRefObjLoader(photoRefObjLoader)
outputs = self.run(**inputs)
if self.config.doWriteMatches and self.config.doAstrometry:
normalizedMatches = afwTable.packMatches(outputs.astromMatches)
normalizedMatches.table.setMetadata(outputs.matchMeta)
if self.config.doWriteMatchesDenormalized:
denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
outputs.matchesDenormalized = denormMatches
outputs.matches = normalizedMatches
butlerQC.put(outputs, outputRefs)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
expId, expBits = butler.registry.packDataId("visit_detector",
inputDataIds['exposure'],
returnMaxBits=True)
inputData['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
if self.config.doAstrometry:
refObjLoader = ReferenceObjectLoader(dataIds=inputDataIds['astromRefCat'],
butler=butler,
config=self.config.astromRefObjLoader,
log=self.log)
self.pixelMargin = refObjLoader.config.pixelMargin
self.astrometry.setRefObjLoader(refObjLoader)
if self.config.doPhotoCal:
photoRefObjLoader = ReferenceObjectLoader(inputDataIds['photoRefCat'],
butler,
self.config.photoRefObjLoader,
self.log)
self.pixelMargin = photoRefObjLoader.config.pixelMargin
self.photoCal.match.setRefObjLoader(photoRefObjLoader)
results = self.run(**inputData)
if self.config.doWriteMatches:
normalizedMatches = afwTable.packMatches(results.astromMatches)
normalizedMatches.table.setMetadata(results.matchMeta)
if self.config.doWriteMatchesDenormalized:
denormMatches = denormalizeMatches(results.astromMatches, results.matchMeta)
results.matchesDenormalized = denormMatches
results.matches = normalizedMatches
return results
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
if 'exposureIdInfo' not in inputs.keys():
exposureIdInfo = ExposureIdInfo()
exposureIdInfo.expId, exposureIdInfo.expBits = butlerQC.quantum.dataId.pack(
"visit_detector", returnMaxBits=True)
inputs['exposureIdInfo'] = exposureIdInfo
outputs = self.run(**inputs)
butlerQC.put(outputs, outputRefs)
def makeIdFactory(self, dataRef):
"""Return an IdFactory for setting the detection identifiers
The actual parameters used in the IdFactory are provided by
the butler (through the provided data reference.
"""
info = ExposureIdInfo(
int(dataRef.get(self.config.coaddName + datasetName)),
dataRef.get(self.config.coaddName + datasetName + "_bits"))
return info.makeSourceIdFactory()
def makeIdFactory(self, dataRef):
"""Return an IdFactory for setting the detection identifiers
The actual parameters used in the IdFactory are provided by
the butler (through the provided data reference.
"""
expId = getGen3CoaddExposureId(dataRef, coaddName=self.config.coaddName, includeBand=includeBand,
log=self.log)
info = ExposureIdInfo(expId, dataRef.get(self.config.coaddName + datasetName + "_bits"))
return info.makeSourceIdFactory()
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
if 'exposureIdInfo' not in inputData.keys():
packer = butler.registry.makeDataIdPacker("visit_detector",
inputDataIds['exposure'])
exposureIdInfo = ExposureIdInfo()
exposureIdInfo.expId = packer.pack(inputDataIds['exposure'])
exposureIdInfo.expBits = packer.maxBits
inputData['exposureIdInfo'] = exposureIdInfo
return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds,
butler)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
if 'exposureIdInfo' not in inputs.keys():
expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
returnMaxBits=True)
inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
if inputs["wcs"] is None:
inputs["wcs"] = inputs["image"].getWcs()
if inputs["photoCalib"] is None:
inputs["photoCalib"] = inputs["image"].getPhotoCalib()
outputs = self.run(**inputs)
butlerQC.put(outputs, outputRefs)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
if 'exposureIdInfo' not in inputData.keys():
packer = butler.registry.makeDataIdPacker("VisitDetector",
inputDataIds['exposure'])
exposureIdInfo = ExposureIdInfo()
exposureIdInfo.expId = packer.pack(inputDataIds['exposure'])
exposureIdInfo.expBits = packer.maxBits
inputData['exposureIdInfo'] = exposureIdInfo
if inputData["wcs"] is None:
inputData["wcs"] = inputData["image"].getWcs()
if inputData["photoCalib"] is None:
inputData["photoCalib"] = inputData["image"].getCalib()
return self.run(**inputData)
def run(self, exposure, exposureIdInfo=None):
if not exposure.hasPsf():
self.installSimplePsf.run(exposure=exposure)
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
try:
self.repair.run(exposure=exposure, keepCRs=True)
except LengthError:
self.log.info("Skipping cosmic ray detection: Too many CR pixels (max %0.f)" % self.repair.cosmicray.nCrPixelMax)
sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
filtered = maximum_filter(exposure.getImage().array, size=self.config.maximumFilterBoxWidth)
detected = (filtered == exposure.getImage().getArray()) & (filtered > self.config.thresholdValue)
detectedImage = afwImage.ImageF(detected.astype(np.float32))
fps = afwDetect.FootprintSet(detectedImage, afwDetect.Threshold(0.5))
fp_ctrl = afwDetect.FootprintControl(True, True)
fps = afwDetect.FootprintSet(fps, self.config.footprintGrowValue, fp_ctrl)
sources = afwTable.SourceCatalog(table)
fps.makeSources(sources)
self.measurement.run(measCat=sources, exposure=exposure, exposureId=exposureIdInfo.expId)
self.catalogCalculation.run(sources)
## Add metadata to source catalog
md = exposure.getMetadata()
sources.getMetadata().add("BOTXCAM", md["BOTXCAM"])
sources.getMetadata().add("BOTYCAM", md["BOTYCAM"])
self.display("measure", exposure=exposure, sourceCat=sources)
return pipeBase.Struct(sourceCat=sources)
def makeIdFactory(expId, expBits):
"""Create IdFactory instance for unique 64 bit diaSource id-s.
Parameters
----------
expId : `int`
Exposure id.
expBits: `int`
Number of used bits in ``expId``.
Notes
-----
The diasource id-s consists of the ``expId`` stored fixed in the highest value
``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
low value end of the integer.
Returns
-------
idFactory: `lsst.afw.table.IdFactory`
"""
return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
def makeIdFactory(self, dataRef, exposureId):
"""Create an object that generates globally unique source IDs.
Source IDs are created based on a per-CCD ID and the ID of the CCD
itself.
Parameters
----------
dataRef : `lsst.daf.persistence.ButlerDataRef`
Butler data reference. The "CoaddId_bits" and "CoaddId" datasets
are accessed. The data ID must have tract and patch keys.
"""
# With the default configuration, this IdFactory doesn't do anything,
# because the IDs it generates are immediately overwritten by the ID
# from the reference catalog (since that's in
# config.measurement.copyColumns). But we create one here anyway, to
# allow us to revert back to the old behavior of generating new forced
# source IDs, just by renaming the ID in config.copyColumns to
# "object_id".
exposureIdInfo = ExposureIdInfo(
exposureId, dataRef.get(self.config.coaddName + "CoaddId_bits"))
return exposureIdInfo.makeSourceIdFactory()
def run(self, exposure, exposureIdInfo=None, background=None):
"""!Characterize a science image
Peforms the following operations:
- Iterate the following config.psfIterations times, or once if config.doMeasurePsf false:
- detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details)
- interpolate over cosmic rays
- perform final measurement
@param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar).
The following changes are made:
- update or set psf
- set apCorrMap
- update detection and cosmic ray mask planes
- subtract background and interpolate over cosmic rays
@param[in] exposureIdInfo ID info for exposure (an lsst.obs.base.ExposureIdInfo).
If not provided, returned SourceCatalog IDs will not be globally unique.
@param[in,out] background initial model of background already subtracted from exposure
(an lsst.afw.math.BackgroundList). May be None if no background has been subtracted,
which is typical for image characterization.
@return pipe_base Struct containing these fields, all from the final iteration
of detectMeasureAndEstimatePsf:
- exposure: characterized exposure; image is repaired by interpolating over cosmic rays,
mask is updated accordingly, and the PSF model is set
- sourceCat: detected sources (an lsst.afw.table.SourceCatalog)
- background: model of background subtracted from exposure (an lsst.afw.math.BackgroundList)
- psfCellSet: spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet)
"""
self._frame = self._initialFrame # reset debug display frame
if not self.config.doMeasurePsf and not exposure.hasPsf():
self.log.warn(
"Source catalog detected and measured with placeholder or default PSF"
)
self.installSimplePsf.run(exposure=exposure)
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
# subtract an initial estimate of background level
background = self.background.run(exposure).background
psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1
for i in range(psfIterations):
dmeRes = self.detectMeasureAndEstimatePsf(
exposure=exposure,
exposureIdInfo=exposureIdInfo,
background=background,
)
psf = dmeRes.exposure.getPsf()
psfSigma = psf.computeShape().getDeterminantRadius()
psfDimensions = psf.computeImage().getDimensions()
medBackground = np.median(dmeRes.background.getImage().getArray())
self.log.info(
"iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f"
% (i + 1, psfSigma, psfDimensions, medBackground))
self.display("psf",
exposure=dmeRes.exposure,
sourceCat=dmeRes.sourceCat)
# perform final repair with final PSF
self.repair.run(exposure=dmeRes.exposure)
self.display("repair",
exposure=dmeRes.exposure,
sourceCat=dmeRes.sourceCat)
# perform final measurement with final PSF, including measuring and applying aperture correction,
# if wanted
self.measurement.run(measCat=dmeRes.sourceCat,
exposure=dmeRes.exposure,
exposureId=exposureIdInfo.expId)
if self.config.doApCorr:
apCorrMap = self.measureApCorr.run(
exposure=dmeRes.exposure, catalog=dmeRes.sourceCat).apCorrMap
dmeRes.exposure.getInfo().setApCorrMap(apCorrMap)
self.applyApCorr.run(catalog=dmeRes.sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap())
self.catalogCalculation.run(dmeRes.sourceCat)
self.display("measure",
exposure=dmeRes.exposure,
sourceCat=dmeRes.sourceCat)
return pipeBase.Struct(exposure=dmeRes.exposure,
sourceCat=dmeRes.sourceCat,
background=dmeRes.background,
psfCellSet=dmeRes.psfCellSet,
characterized=dmeRes.exposure,
backgroundModel=dmeRes.background)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
detectorId = inputs["exposure"].getInfo().getDetector().getId()
if 'exposureIdInfo' not in inputs.keys():
expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
returnMaxBits=True)
inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
expWcs = inputs["exposure"].getWcs()
tractId = inputs["skyMap"].findTract(
expWcs.pixelToSky(
inputs["exposure"].getBBox().getCenter())).tract_id
if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs:
inputs["wcs"] = expWcs
elif self.config.doApplyExternalGlobalSkyWcs:
externalSkyWcsCatalog = inputs["externalSkyWcsGlobalCatalog"]
row = externalSkyWcsCatalog.find(detectorId)
inputs["wcs"] = row.getWcs()
elif self.config.doApplyExternalTractSkyWcs:
externalSkyWcsCatalogList = inputs["externalSkyWcsTractCatalog"]
externalSkyWcsCatalog = None
for externalSkyWcsCatalogRef in externalSkyWcsCatalogList:
if externalSkyWcsCatalogRef.dataId["tract"] == tractId:
externalSkyWcsCatalog = externalSkyWcsCatalogRef.get(
datasetType=self.config.connections.
externalSkyWcsTractCatalog)
break
if externalSkyWcsCatalog is None:
usedTract = externalSkyWcsCatalogList[-1].dataId["tract"]
self.log.warn(
f"Warning, external SkyWcs for tract {tractId} not found. Using tract {usedTract} "
"instead.")
externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get(
datasetType=self.config.connections.
externalSkyWcsTractCatalog)
row = externalSkyWcsCatalog.find(detectorId)
inputs["wcs"] = row.getWcs()
if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib:
inputs["photoCalib"] = inputs["exposure"].getPhotoCalib()
elif self.config.doApplyExternalGlobalPhotoCalib:
externalPhotoCalibCatalog = inputs[
"externalPhotoCalibGlobalCatalog"]
row = externalPhotoCalibCatalog.find(detectorId)
inputs["photoCalib"] = row.getPhotoCalib()
elif self.config.doApplyExternalTractPhotoCalib:
externalPhotoCalibCatalogList = inputs[
"externalPhotoCalibTractCatalog"]
externalPhotoCalibCatalog = None
for externalPhotoCalibCatalogRef in externalPhotoCalibCatalogList:
if externalPhotoCalibCatalogRef.dataId["tract"] == tractId:
externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get(
datasetType=self.config.connections.
externalPhotoCalibTractCatalog)
break
if externalPhotoCalibCatalog is None:
usedTract = externalPhotoCalibCatalogList[-1].dataId["tract"]
self.log.warn(
f"Warning, external PhotoCalib for tract {tractId} not found. Using tract {usedTract} "
"instead.")
externalPhotoCalibCatalog = externalPhotoCalibCatalogList[
-1].get(datasetType=self.config.connections.
externalPhotoCalibTractCatalog)
row = externalPhotoCalibCatalog.find(detectorId)
inputs["photoCalib"] = row.getPhotoCalib()
outputs = self.run(**inputs)
butlerQC.put(outputs, outputRefs)
def run(self, diaSources, tractPatchId, skymapBits):
"""Associate DiaSources into a collection of DiaObjects using a
brute force matching algorithm.
Reproducible is for the same input data is assured by ordering the
DiaSource data by ccdVisit ordering.
Parameters
----------
diaSources : `pandas.DataFrame`
DiaSources grouped by CcdVisitId to spatially associate into
DiaObjects.
tractPatchId : `int`
Unique identifier for the tract patch.
skymapBits : `int`
Maximum number of bits used the ``tractPatchId`` integer
identifier.
Returns
-------
results : `lsst.pipe.base.Struct`
Results struct with attributes:
``assocDiaSources``
Table of DiaSources with updated values for the DiaObjects
they are spatially associated to (`pandas.DataFrame`).
``diaObjects``
Table of DiaObjects from matching DiaSources
(`pandas.DataFrame`).
"""
# Expected indexes include diaSourceId or meaningless range index
# If meaningless range index, drop it, else keep it.
doDropIndex = diaSources.index.names[0] is None
diaSources.reset_index(inplace=True, drop=doDropIndex)
# Sort by ccdVisit and diaSourceId to get a reproducible ordering for
# the association.
diaSources.set_index(["ccdVisitId", "diaSourceId"], inplace=True)
# Empty lists to store matching and location data.
diaObjectCat = []
diaObjectCoords = []
healPixIndices = []
# Create Id factory and catalog for creating DiaObjectIds.
exposureIdInfo = ExposureIdInfo(tractPatchId, skymapBits)
idFactory = exposureIdInfo.makeSourceIdFactory()
idCat = afwTable.SourceCatalog(
afwTable.SourceTable.make(afwTable.SourceTable.makeMinimalSchema(),
idFactory))
for ccdVisit in diaSources.index.levels[0]:
# For the first ccdVisit, just copy the DiaSource info into the
# diaObject data to create the first set of Objects.
ccdVisitSources = diaSources.loc[ccdVisit]
if len(diaObjectCat) == 0:
for diaSourceId, diaSrc in ccdVisitSources.iterrows():
self.addNewDiaObject(diaSrc,
diaSources,
ccdVisit,
diaSourceId,
diaObjectCat,
idCat,
diaObjectCoords,
healPixIndices)
continue
# Temp list to store DiaObjects already used for this ccdVisit.
usedMatchIndicies = []
# Run over subsequent data.
for diaSourceId, diaSrc in ccdVisitSources.iterrows():
# Find matches.
matchResult = self.findMatches(diaSrc["ra"],
diaSrc["decl"],
2*self.config.tolerance,
healPixIndices,
diaObjectCat)
dists = matchResult.dists
matches = matchResult.matches
# Create a new DiaObject if no match found.
if dists is None:
self.addNewDiaObject(diaSrc,
diaSources,
ccdVisit,
diaSourceId,
diaObjectCat,
idCat,
diaObjectCoords,
healPixIndices)
continue
# If matched, update catalogs and arrays.
if np.min(dists) < np.deg2rad(self.config.tolerance/3600):
matchDistArg = np.argmin(dists)
matchIndex = matches[matchDistArg]
# Test to see if the DiaObject has been used.
if np.isin([matchIndex], usedMatchIndicies).sum() < 1:
self.updateCatalogs(matchIndex,
diaSrc,
diaSources,
ccdVisit,
diaSourceId,
diaObjectCat,
diaObjectCoords,
healPixIndices)
usedMatchIndicies.append(matchIndex)
# If the matched DiaObject has already been used, create a
# new DiaObject for this DiaSource.
else:
self.addNewDiaObject(diaSrc,
diaSources,
ccdVisit,
diaSourceId,
diaObjectCat,
idCat,
diaObjectCoords,
healPixIndices)
# Create new DiaObject if no match found within the matching
# tolerance.
else:
self.addNewDiaObject(diaSrc,
diaSources,
ccdVisit,
diaSourceId,
diaObjectCat,
idCat,
diaObjectCoords,
healPixIndices)
# Drop indices before returning associated diaSource catalog.
diaSources.reset_index(inplace=True)
diaSources.set_index("diaSourceId", inplace=True, verify_integrity=True)
objs = diaObjectCat if diaObjectCat else np.array([], dtype=[('diaObjectId', 'int64'),
('ra', 'float64'),
('decl', 'float64'),
('nDiaSources', 'int64')])
diaObjects = pd.DataFrame(data=objs)
if "diaObjectId" in diaObjects.columns:
diaObjects.set_index("diaObjectId", inplace=True, verify_integrity=True)
return pipeBase.Struct(
assocDiaSources=diaSources,
diaObjects=diaObjects)
def run(self, exposure, exposureIdInfo=None, background=None,
icSourceCat=None):
"""!Calibrate an exposure (science image or coadd)
@param[in,out] exposure exposure to calibrate (an
lsst.afw.image.ExposureF or similar);
in:
- MaskedImage
- Psf
out:
- MaskedImage has background subtracted
- Wcs is replaced
- PhotoCalib is replaced
@param[in] exposureIdInfo ID info for exposure (an
lsst.obs.base.ExposureIdInfo) If not provided, returned
SourceCatalog IDs will not be globally unique.
@param[in,out] background background model already subtracted from
exposure (an lsst.afw.math.BackgroundList). May be None if no
background has been subtracted, though that is unusual for
calibration. A refined background model is output.
@param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
from which we can copy some fields.
@return pipe_base Struct containing these fields:
- exposure calibrate science exposure with refined WCS and PhotoCalib
- background model of background subtracted from exposure (an
lsst.afw.math.BackgroundList)
- sourceCat catalog of measured sources
- astromMatches list of source/refObj matches from the astrometry
solver
"""
# detect, deblend and measure sources
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
if background is None:
background = BackgroundList()
sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
exposureIdInfo.unusedBits)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
detRes = self.detection.run(table=table, exposure=exposure,
doSmooth=True)
sourceCat = detRes.sources
if detRes.fpSets.background:
for bg in detRes.fpSets.background:
background.append(bg)
if self.config.doSkySources:
skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId)
if skySourceFootprints:
for foot in skySourceFootprints:
s = sourceCat.addNew()
s.setFootprint(foot)
s.set(self.skySourceKey, True)
if self.config.doDeblend:
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(
measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId
)
if self.config.doApCorr:
self.applyApCorr.run(
catalog=sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap()
)
self.catalogCalculation.run(sourceCat)
self.setPrimaryFlags.run(sourceCat, includeDeblend=self.config.doDeblend)
if icSourceCat is not None and \
len(self.config.icSourceFieldsToCopy) > 0:
self.copyIcSourceFields(icSourceCat=icSourceCat,
sourceCat=sourceCat)
# TODO DM-11568: this contiguous check-and-copy could go away if we
# reserve enough space during SourceDetection and/or SourceDeblend.
# NOTE: sourceSelectors require contiguous catalogs, so ensure
# contiguity now, so views are preserved from here on.
if not sourceCat.isContiguous():
sourceCat = sourceCat.copy(deep=True)
# perform astrometry calibration:
# fit an improved WCS and update the exposure's WCS in place
astromMatches = None
matchMeta = None
if self.config.doAstrometry:
try:
astromRes = self.astrometry.run(
exposure=exposure,
sourceCat=sourceCat,
)
astromMatches = astromRes.matches
matchMeta = astromRes.matchMeta
except Exception as e:
if self.config.requireAstrometry:
raise
self.log.warn("Unable to perform astrometric calibration "
"(%s): attempting to proceed" % e)
# compute photometric calibration
if self.config.doPhotoCal:
try:
photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
exposure.setPhotoCalib(photoRes.photoCalib)
# TODO: reword this to phrase it in terms of the calibration factor?
self.log.info("Photometric zero-point: %f" %
photoRes.photoCalib.instFluxToMagnitude(1.0))
self.setMetadata(exposure=exposure, photoRes=photoRes)
except Exception as e:
if self.config.requirePhotoCal:
raise
self.log.warn("Unable to perform photometric calibration "
"(%s): attempting to proceed" % e)
self.setMetadata(exposure=exposure, photoRes=None)
if self.config.doInsertFakes:
self.insertFakes.run(exposure, background=background)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
detRes = self.detection.run(table=table, exposure=exposure,
doSmooth=True)
sourceCat = detRes.sources
if detRes.fpSets.background:
for bg in detRes.fpSets.background:
background.append(bg)
if self.config.doDeblend:
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(
measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId
)
if self.config.doApCorr:
self.applyApCorr.run(
catalog=sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap()
)
self.catalogCalculation.run(sourceCat)
if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
self.copyIcSourceFields(icSourceCat=icSourceCat,
sourceCat=sourceCat)
frame = getDebugFrame(self._display, "calibrate")
if frame:
displayAstrometry(
sourceCat=sourceCat,
exposure=exposure,
matches=astromMatches,
frame=frame,
pause=False,
)
return pipeBase.Struct(
exposure=exposure,
background=background,
sourceCat=sourceCat,
astromMatches=astromMatches,
matchMeta=matchMeta,
# These are duplicate entries with different names for use with
# gen3 middleware
outputExposure=exposure,
outputCat=sourceCat,
outputBackground=background,
)
def run(self,
fakeCats,
exposure,
skyMap,
wcs=None,
photoCalib=None,
exposureIdInfo=None,
icSourceCat=None,
sfdSourceCat=None,
externalSkyWcsGlobalCatalog=None,
externalSkyWcsTractCatalog=None,
externalPhotoCalibGlobalCatalog=None,
externalPhotoCalibTractCatalog=None):
"""Add fake sources to a calexp and then run detection, deblending and measurement.
Parameters
----------
fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle`
Set of tract level fake catalogs that potentially cover
this detectorVisit.
exposure : `lsst.afw.image.exposure.exposure.ExposureF`
The exposure to add the fake sources to
skyMap : `lsst.skymap.SkyMap`
SkyMap defining the tracts and patches the fakes are stored over.
wcs : `lsst.afw.geom.SkyWcs`
WCS to use to add fake sources
photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
Photometric calibration to be used to calibrate the fake sources
exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
icSourceCat : `lsst.afw.table.SourceCatalog`
Default : None
Catalog to take the information about which sources were used for calibration from.
sfdSourceCat : `lsst.afw.table.SourceCatalog`
Default : None
Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
Returns
-------
resultStruct : `lsst.pipe.base.struct.Struct`
contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
outputCat : `lsst.afw.table.source.source.SourceCatalog`
Notes
-----
Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
pixels.
Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
the calexp and the calexp with fakes included returned.
The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
this is then convolved with the PSF at that point.
If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
"""
fakeCat = self.composeFakeCat(fakeCats, skyMap)
if wcs is None:
wcs = exposure.getWcs()
if photoCalib is None:
photoCalib = exposure.getPhotoCalib()
if self.config.doMatchVisit:
fakeCat = self.getVisitMatchedFakeCat(fakeCat, exposure)
self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
# detect, deblend and measure sources
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
returnedStruct = self.calibrate.run(exposure,
exposureIdInfo=exposureIdInfo)
sourceCat = returnedStruct.sourceCat
sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat,
self.config.srcFieldsToCopy)
resultStruct = pipeBase.Struct(outputExposure=exposure,
outputCat=sourceCat)
return resultStruct
def calibrate(self,
exposure,
exposureIdInfo=None,
background=None,
icSourceCat=None):
"""!Calibrate an exposure (science image or coadd)
@param[in,out] exposure exposure to calibrate (an
lsst.afw.image.ExposureF or similar);
in:
- MaskedImage
- Psf
out:
- MaskedImage has background subtracted
- Wcs is replaced
- Calib zero-point is set
@param[in] exposureIdInfo ID info for exposure (an
lsst.obs.base.ExposureIdInfo) If not provided, returned
SourceCatalog IDs will not be globally unique.
@param[in,out] background background model already subtracted from
exposure (an lsst.afw.math.BackgroundList). May be None if no
background has been subtracted, though that is unusual for
calibration. A refined background model is output.
@param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
from which we can copy some fields.
@return pipe_base Struct containing these fields:
- exposure calibrate science exposure with refined WCS and Calib
- background model of background subtracted from exposure (an
lsst.afw.math.BackgroundList)
- sourceCat catalog of measured sources
- astromMatches list of source/refObj matches from the astrometry
solver
"""
# detect, deblend and measure sources
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
if background is None:
background = BackgroundList()
sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
exposureIdInfo.unusedBits)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
if self.config.doInsertFakes:
self.insertFakes.run(exposure, background=background)
detRes = self.detection.run(table=table,
exposure=exposure,
doSmooth=True)
sourceCat = detRes.sources
if detRes.fpSets.background:
background.append(detRes.fpSets.background)
if self.config.doDeblend:
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId)
if self.config.doApCorr:
self.applyApCorr.run(catalog=sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap())
self.catalogCalculation.run(sourceCat)
if icSourceCat is not None and \
len(self.config.icSourceFieldsToCopy) > 0:
self.copyIcSourceFields(icSourceCat=icSourceCat,
sourceCat=sourceCat)
# perform astrometry calibration:
# fit an improved WCS and update the exposure's WCS in place
astromMatches = None
if self.config.doAstrometry:
try:
astromRes = self.astrometry.run(
exposure=exposure,
sourceCat=sourceCat,
)
astromMatches = astromRes.matches
except Exception as e:
if self.config.requireAstrometry:
raise
self.log.warn("Unable to perform astrometric calibration "
"(%s): attempting to proceed" % e)
# compute photometric calibration
if self.config.doPhotoCal:
try:
photoRes = self.photoCal.run(exposure, sourceCat=sourceCat)
exposure.getCalib().setFluxMag0(photoRes.calib.getFluxMag0())
self.log.info("Photometric zero-point: %f" %
photoRes.calib.getMagnitude(1.0))
self.setMetadata(exposure=exposure, photoRes=photoRes)
except Exception as e:
if self.config.requirePhotoCal:
raise
self.log.warn("Unable to perform photometric calibration "
"(%s): attempting to proceed" % e)
self.setMetadata(exposure=exposure, photoRes=None)
frame = getDebugFrame(self._display, "calibrate")
if frame:
displayAstrometry(
sourceCat=sourceCat,
exposure=exposure,
matches=astromMatches,
frame=frame,
pause=False,
)
return pipeBase.Struct(
exposure=exposure,
background=background,
sourceCat=sourceCat,
astromMatches=astromMatches,
)
def run(self,
fakeCats,
exposure,
skyMap,
wcs=None,
photoCalib=None,
exposureIdInfo=None,
icSourceCat=None,
sfdSourceCat=None):
"""Add fake sources to a calexp and then run detection, deblending and measurement.
Parameters
----------
fakeCat : `pandas.core.frame.DataFrame`
The catalog of fake sources to add to the exposure
exposure : `lsst.afw.image.exposure.exposure.ExposureF`
The exposure to add the fake sources to
skyMap : `lsst.skymap.SkyMap`
SkyMap defining the tracts and patches the fakes are stored over.
wcs : `lsst.afw.geom.SkyWcs`
WCS to use to add fake sources
photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
Photometric calibration to be used to calibrate the fake sources
exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
icSourceCat : `lsst.afw.table.SourceCatalog`
Default : None
Catalog to take the information about which sources were used for calibration from.
sfdSourceCat : `lsst.afw.table.SourceCatalog`
Default : None
Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
Returns
-------
resultStruct : `lsst.pipe.base.struct.Struct`
Results Strcut containing:
- outputExposure : Exposure with added fakes
(`lsst.afw.image.exposure.exposure.ExposureF`)
- outputCat : Catalog with detected fakes
(`lsst.afw.table.source.source.SourceCatalog`)
- ccdVisitFakeMagnitudes : Magnitudes that these fakes were
inserted with after being scattered (`pandas.DataFrame`)
Notes
-----
Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
pixels.
Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
the calexp and the calexp with fakes included returned.
The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
this is then convolved with the PSF at that point.
If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
"""
fakeCat = self.composeFakeCat(fakeCats, skyMap)
if wcs is None:
wcs = exposure.getWcs()
if photoCalib is None:
photoCalib = exposure.getPhotoCalib()
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
band = exposure.getFilter().bandLabel
ccdVisitMagnitudes = self.addVariablity(fakeCat, band, exposure,
photoCalib, exposureIdInfo)
self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
# detect, deblend and measure sources
returnedStruct = self.calibrate.run(exposure,
exposureIdInfo=exposureIdInfo)
sourceCat = returnedStruct.sourceCat
sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat,
self.config.srcFieldsToCopy)
resultStruct = pipeBase.Struct(
outputExposure=exposure,
outputCat=sourceCat,
ccdVisitFakeMagnitudes=ccdVisitMagnitudes)
return resultStruct
def run(self,
fakeCat,
exposure,
wcs=None,
photoCalib=None,
exposureIdInfo=None,
icSourceCat=None,
sfdSourceCat=None):
"""Add fake sources to a calexp and then run detection, deblending and measurement.
Parameters
----------
fakeCat : `pandas.core.frame.DataFrame`
The catalog of fake sources to add to the exposure
exposure : `lsst.afw.image.exposure.exposure.ExposureF`
The exposure to add the fake sources to
wcs : `lsst.afw.geom.SkyWcs`
WCS to use to add fake sources
photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
Photometric calibration to be used to calibrate the fake sources
exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
icSourceCat : `lsst.afw.table.SourceCatalog`
Default : None
Catalog to take the information about which sources were used for calibration from.
sfdSourceCat : `lsst.afw.table.SourceCatalog`
Default : None
Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
Returns
-------
resultStruct : `lsst.pipe.base.struct.Struct`
contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
outputCat : `lsst.afw.table.source.source.SourceCatalog`
Notes
-----
Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
pixels.
Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
the calexp and the calexp with fakes included returned.
The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
this is then convolved with the PSF at that point.
If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
"""
if wcs is None:
wcs = exposure.getWcs()
if photoCalib is None:
photoCalib = exposure.getPhotoCalib()
self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
# detect, deblend and measure sources
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
exposureIdInfo.unusedBits)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
detRes = self.detection.run(table=table,
exposure=exposure,
doSmooth=True)
sourceCat = detRes.sources
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId)
self.applyApCorr.run(catalog=sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap())
self.catalogCalculation.run(sourceCat)
sourceCat = self.copyCalibrationFields(
icSourceCat, sourceCat, self.config.calibrationFieldsToCopy)
sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat,
self.config.srcFieldsToCopy)
resultStruct = pipeBase.Struct(outputExposure=exposure,
outputCat=sourceCat)
return resultStruct