def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
if 'exposureIdInfo' not in inputs.keys():
exposureIdInfo = ExposureIdInfo()
exposureIdInfo.expId, exposureIdInfo.expBits = butlerQC.quantum.dataId.pack(
"visit_detector", returnMaxBits=True)
inputs['exposureIdInfo'] = exposureIdInfo
outputs = self.run(**inputs)
butlerQC.put(outputs, outputRefs)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
if 'exposureIdInfo' not in inputData.keys():
packer = butler.registry.makeDataIdPacker("visit_detector",
inputDataIds['exposure'])
exposureIdInfo = ExposureIdInfo()
exposureIdInfo.expId = packer.pack(inputDataIds['exposure'])
exposureIdInfo.expBits = packer.maxBits
inputData['exposureIdInfo'] = exposureIdInfo
return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds, butler)
def makeIdFactory(self, dataRef):
"""Return an IdFactory for setting the detection identifiers
The actual parameters used in the IdFactory are provided by
the butler (through the provided data reference.
"""
expId = getGen3CoaddExposureId(dataRef, coaddName=self.config.coaddName, includeBand=includeBand,
log=self.log)
info = ExposureIdInfo(expId, dataRef.get(self.config.coaddName + datasetName + "_bits"))
return info.makeSourceIdFactory()
def makeIdFactory(self, dataRef):
"""Return an IdFactory for setting the detection identifiers
The actual parameters used in the IdFactory are provided by
the butler (through the provided data reference.
"""
info = ExposureIdInfo(
int(dataRef.get(self.config.coaddName + datasetName)),
dataRef.get(self.config.coaddName + datasetName + "_bits"))
return info.makeSourceIdFactory()
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
if 'exposureIdInfo' not in inputData.keys():
packer = butler.registry.makeDataIdPacker("visit_detector",
inputDataIds['exposure'])
exposureIdInfo = ExposureIdInfo()
exposureIdInfo.expId = packer.pack(inputDataIds['exposure'])
exposureIdInfo.expBits = packer.maxBits
inputData['exposureIdInfo'] = exposureIdInfo
return super().adaptArgsAndRun(inputData, inputDataIds, outputDataIds,
butler)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
if 'exposureIdInfo' not in inputData.keys():
packer = butler.registry.makeDataIdPacker("VisitDetector",
inputDataIds['exposure'])
exposureIdInfo = ExposureIdInfo()
exposureIdInfo.expId = packer.pack(inputDataIds['exposure'])
exposureIdInfo.expBits = packer.maxBits
inputData['exposureIdInfo'] = exposureIdInfo
if inputData["wcs"] is None:
inputData["wcs"] = inputData["image"].getWcs()
if inputData["photoCalib"] is None:
inputData["photoCalib"] = inputData["image"].getCalib()
return self.run(**inputData)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
id_tp = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"tract_patch").expId
input_refs_objs = [(inputRefs.cats_meas, inputs['cats_meas']),
(inputRefs.coadds, inputs['coadds'])]
cats, exps = [{dRef.dataId: obj
for dRef, obj in zip(refs, objs)}
for refs, objs in input_refs_objs]
dataIds = set(cats).union(set(exps))
catexps = [
CatalogExposure(
catalog=cats.get(dataId),
exposure=exps.get(dataId),
dataId=dataId,
id_tract_patch=id_tp,
) for dataId in dataIds
]
outputs = self.run(catexps=catexps, cat_ref=inputs['cat_ref'])
butlerQC.put(outputs, outputRefs)
# Validate the output catalog's schema and raise if inconsistent (after output to allow debugging)
if outputs.cat_output.schema != self.cat_output_schema.schema:
raise RuntimeError(
f'{__class__}.config.fit_multiband.run schema != initOutput schema:'
f' {outputs.cat_output.schema} vs {self.cat_output_schema.schema}'
)
def adaptArgsAndRun(self, inputData, inputDataIds, outputDataIds, butler):
expId, expBits = butler.registry.packDataId("visit_detector",
inputDataIds['exposure'],
returnMaxBits=True)
inputData['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
if self.config.doAstrometry:
refObjLoader = ReferenceObjectLoader(dataIds=inputDataIds['astromRefCat'],
butler=butler,
config=self.config.astromRefObjLoader,
log=self.log)
self.pixelMargin = refObjLoader.config.pixelMargin
self.astrometry.setRefObjLoader(refObjLoader)
if self.config.doPhotoCal:
photoRefObjLoader = ReferenceObjectLoader(inputDataIds['photoRefCat'],
butler,
self.config.photoRefObjLoader,
self.log)
self.pixelMargin = photoRefObjLoader.config.pixelMargin
self.photoCal.match.setRefObjLoader(photoRefObjLoader)
results = self.run(**inputData)
if self.config.doWriteMatches:
normalizedMatches = afwTable.packMatches(results.astromMatches)
normalizedMatches.table.setMetadata(results.matchMeta)
if self.config.doWriteMatchesDenormalized:
denormMatches = denormalizeMatches(results.astromMatches, results.matchMeta)
results.matchesDenormalized = denormMatches
results.matches = normalizedMatches
return results
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
exposureIdInfo = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"tract_patch")
inputs["skySeed"] = exposureIdInfo.expId
inputs["idFactory"] = exposureIdInfo.makeSourceIdFactory()
catalogDict = {
ref.dataId['band']: cat
for ref, cat in zip(inputRefs.catalogs, inputs['catalogs'])
}
inputs['catalogs'] = catalogDict
skyMap = inputs.pop('skyMap')
# Can use the first dataId to find the tract and patch being worked on
tractNumber = inputRefs.catalogs[0].dataId['tract']
tractInfo = skyMap[tractNumber]
patchInfo = tractInfo.getPatchInfo(
inputRefs.catalogs[0].dataId['patch'])
skyInfo = Struct(skyMap=skyMap,
tractInfo=tractInfo,
patchInfo=patchInfo,
wcs=tractInfo.getWcs(),
bbox=patchInfo.getOuterBBox())
inputs['skyInfo'] = skyInfo
outputs = self.run(**inputs)
butlerQC.put(outputs, outputRefs)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
returnMaxBits=True)
inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
if self.config.doAstrometry:
refObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
for ref in inputRefs.astromRefCat],
refCats=inputs.pop('astromRefCat'),
config=self.config.astromRefObjLoader, log=self.log)
self.pixelMargin = refObjLoader.config.pixelMargin
self.astrometry.setRefObjLoader(refObjLoader)
if self.config.doPhotoCal:
photoRefObjLoader = ReferenceObjectLoader(dataIds=[ref.datasetRef.dataId
for ref in inputRefs.photoRefCat],
refCats=inputs.pop('photoRefCat'),
config=self.config.photoRefObjLoader,
log=self.log)
self.pixelMargin = photoRefObjLoader.config.pixelMargin
self.photoCal.match.setRefObjLoader(photoRefObjLoader)
outputs = self.run(**inputs)
if self.config.doWriteMatches and self.config.doAstrometry:
normalizedMatches = afwTable.packMatches(outputs.astromMatches)
normalizedMatches.table.setMetadata(outputs.matchMeta)
if self.config.doWriteMatchesDenormalized:
denormMatches = denormalizeMatches(outputs.astromMatches, outputs.matchMeta)
outputs.matchesDenormalized = denormMatches
outputs.matches = normalizedMatches
butlerQC.put(outputs, outputRefs)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
if 'exposureIdInfo' not in inputs.keys():
inputs['exposureIdInfo'] = ExposureIdInfo.fromDataId(
butlerQC.quantum.dataId, "visit_detector")
outputs = self.run(**inputs)
butlerQC.put(outputs, outputRefs)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
exposureIdInfo = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"tract_patch")
inputs["idFactory"] = exposureIdInfo.makeSourceIdFactory()
inputs["filters"] = [dRef.dataId["band"] for dRef in inputRefs.coadds]
outputs = self.run(**inputs)
sortedTemplateCatalogs = []
for outRef in outputRefs.templateCatalogs:
band = outRef.dataId['band']
sortedTemplateCatalogs.append(outputs.templateCatalogs[band])
outputs.templateCatalogs = sortedTemplateCatalogs
butlerQC.put(outputs, outputRefs)
def makeIdFactory(self, dataRef, exposureId):
"""Create an object that generates globally unique source IDs.
Source IDs are created based on a per-CCD ID and the ID of the CCD
itself.
Parameters
----------
dataRef : `lsst.daf.persistence.ButlerDataRef`
Butler data reference. The "CoaddId_bits" and "CoaddId" datasets
are accessed. The data ID must have tract and patch keys.
"""
# With the default configuration, this IdFactory doesn't do anything,
# because the IDs it generates are immediately overwritten by the ID
# from the reference catalog (since that's in
# config.measurement.copyColumns). But we create one here anyway, to
# allow us to revert back to the old behavior of generating new forced
# source IDs, just by renaming the ID in config.copyColumns to
# "object_id".
exposureIdInfo = ExposureIdInfo(
exposureId, dataRef.get(self.config.coaddName + "CoaddId_bits"))
return exposureIdInfo.makeSourceIdFactory()
def run(self, exposure, exposureIdInfo=None):
if not exposure.hasPsf():
self.installSimplePsf.run(exposure=exposure)
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
try:
self.repair.run(exposure=exposure, keepCRs=True)
except LengthError:
self.log.info("Skipping cosmic ray detection: Too many CR pixels (max %0.f)" % self.repair.cosmicray.nCrPixelMax)
sourceIdFactory = exposureIdInfo.makeSourceIdFactory()
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
filtered = maximum_filter(exposure.getImage().array, size=self.config.maximumFilterBoxWidth)
detected = (filtered == exposure.getImage().getArray()) & (filtered > self.config.thresholdValue)
detectedImage = afwImage.ImageF(detected.astype(np.float32))
fps = afwDetect.FootprintSet(detectedImage, afwDetect.Threshold(0.5))
fp_ctrl = afwDetect.FootprintControl(True, True)
fps = afwDetect.FootprintSet(fps, self.config.footprintGrowValue, fp_ctrl)
sources = afwTable.SourceCatalog(table)
fps.makeSources(sources)
self.measurement.run(measCat=sources, exposure=exposure, exposureId=exposureIdInfo.expId)
self.catalogCalculation.run(sources)
## Add metadata to source catalog
md = exposure.getMetadata()
sources.getMetadata().add("BOTXCAM", md["BOTXCAM"])
sources.getMetadata().add("BOTYCAM", md["BOTYCAM"])
self.display("measure", exposure=exposure, sourceCat=sources)
return pipeBase.Struct(sourceCat=sources)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
# Obtain the list of bands, sort them (alphabetically), then reorder
# all input lists to match this band order.
bandOrder = [dRef.dataId["band"] for dRef in inputRefs.coadds]
bandOrder.sort()
inputRefs = reorderRefs(inputRefs, bandOrder, dataIdKey="band")
inputs = butlerQC.get(inputRefs)
exposureIdInfo = ExposureIdInfo.fromDataId(butlerQC.quantum.dataId,
"tract_patch")
inputs["idFactory"] = exposureIdInfo.makeSourceIdFactory()
inputs["filters"] = [dRef.dataId["band"] for dRef in inputRefs.coadds]
outputs = self.run(**inputs)
butlerQC.put(outputs, outputRefs)
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
if 'exposureIdInfo' not in inputs.keys():
expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
returnMaxBits=True)
inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
if inputs["wcs"] is None:
inputs["wcs"] = inputs["image"].getWcs()
if inputs["photoCalib"] is None:
inputs["photoCalib"] = inputs["image"].getPhotoCalib()
outputs = self.run(**inputs)
butlerQC.put(outputs, outputRefs)
def makeIdFactory(expId, expBits):
"""Create IdFactory instance for unique 64 bit diaSource id-s.
Parameters
----------
expId : `int`
Exposure id.
expBits: `int`
Number of used bits in ``expId``.
Notes
-----
The diasource id-s consists of the ``expId`` stored fixed in the highest value
``expBits`` of the 64-bit integer plus (bitwise or) a generated sequence number in the
low value end of the integer.
Returns
-------
idFactory: `lsst.afw.table.IdFactory`
"""
return ExposureIdInfo(expId, expBits).makeSourceIdFactory()
def run(self, exposure, exposureIdInfo=None, background=None):
"""!Characterize a science image
Peforms the following operations:
- Iterate the following config.psfIterations times, or once if config.doMeasurePsf false:
- detect and measure sources and estimate PSF (see detectMeasureAndEstimatePsf for details)
- interpolate over cosmic rays
- perform final measurement
@param[in,out] exposure exposure to characterize (an lsst.afw.image.ExposureF or similar).
The following changes are made:
- update or set psf
- set apCorrMap
- update detection and cosmic ray mask planes
- subtract background and interpolate over cosmic rays
@param[in] exposureIdInfo ID info for exposure (an lsst.obs.base.ExposureIdInfo).
If not provided, returned SourceCatalog IDs will not be globally unique.
@param[in,out] background initial model of background already subtracted from exposure
(an lsst.afw.math.BackgroundList). May be None if no background has been subtracted,
which is typical for image characterization.
@return pipe_base Struct containing these fields, all from the final iteration
of detectMeasureAndEstimatePsf:
- exposure: characterized exposure; image is repaired by interpolating over cosmic rays,
mask is updated accordingly, and the PSF model is set
- sourceCat: detected sources (an lsst.afw.table.SourceCatalog)
- background: model of background subtracted from exposure (an lsst.afw.math.BackgroundList)
- psfCellSet: spatial cells of PSF candidates (an lsst.afw.math.SpatialCellSet)
"""
self._frame = self._initialFrame # reset debug display frame
if not self.config.doMeasurePsf and not exposure.hasPsf():
self.log.warn(
"Source catalog detected and measured with placeholder or default PSF"
)
self.installSimplePsf.run(exposure=exposure)
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
# subtract an initial estimate of background level
background = self.background.run(exposure).background
psfIterations = self.config.psfIterations if self.config.doMeasurePsf else 1
for i in range(psfIterations):
dmeRes = self.detectMeasureAndEstimatePsf(
exposure=exposure,
exposureIdInfo=exposureIdInfo,
background=background,
)
psf = dmeRes.exposure.getPsf()
psfSigma = psf.computeShape().getDeterminantRadius()
psfDimensions = psf.computeImage().getDimensions()
medBackground = np.median(dmeRes.background.getImage().getArray())
self.log.info(
"iter %s; PSF sigma=%0.2f, dimensions=%s; median background=%0.2f"
% (i + 1, psfSigma, psfDimensions, medBackground))
self.display("psf",
exposure=dmeRes.exposure,
sourceCat=dmeRes.sourceCat)
# perform final repair with final PSF
self.repair.run(exposure=dmeRes.exposure)
self.display("repair",
exposure=dmeRes.exposure,
sourceCat=dmeRes.sourceCat)
# perform final measurement with final PSF, including measuring and applying aperture correction,
# if wanted
self.measurement.run(measCat=dmeRes.sourceCat,
exposure=dmeRes.exposure,
exposureId=exposureIdInfo.expId)
if self.config.doApCorr:
apCorrMap = self.measureApCorr.run(
exposure=dmeRes.exposure, catalog=dmeRes.sourceCat).apCorrMap
dmeRes.exposure.getInfo().setApCorrMap(apCorrMap)
self.applyApCorr.run(catalog=dmeRes.sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap())
self.catalogCalculation.run(dmeRes.sourceCat)
self.display("measure",
exposure=dmeRes.exposure,
sourceCat=dmeRes.sourceCat)
return pipeBase.Struct(exposure=dmeRes.exposure,
sourceCat=dmeRes.sourceCat,
background=dmeRes.background,
psfCellSet=dmeRes.psfCellSet,
characterized=dmeRes.exposure,
backgroundModel=dmeRes.background)
def run(self,
fakeCats,
exposure,
skyMap,
wcs=None,
photoCalib=None,
exposureIdInfo=None,
icSourceCat=None,
sfdSourceCat=None,
externalSkyWcsGlobalCatalog=None,
externalSkyWcsTractCatalog=None,
externalPhotoCalibGlobalCatalog=None,
externalPhotoCalibTractCatalog=None):
"""Add fake sources to a calexp and then run detection, deblending and measurement.
Parameters
----------
fakeCats : `list` of `lsst.daf.butler.DeferredDatasetHandle`
Set of tract level fake catalogs that potentially cover
this detectorVisit.
exposure : `lsst.afw.image.exposure.exposure.ExposureF`
The exposure to add the fake sources to
skyMap : `lsst.skymap.SkyMap`
SkyMap defining the tracts and patches the fakes are stored over.
wcs : `lsst.afw.geom.SkyWcs`
WCS to use to add fake sources
photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
Photometric calibration to be used to calibrate the fake sources
exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
icSourceCat : `lsst.afw.table.SourceCatalog`
Default : None
Catalog to take the information about which sources were used for calibration from.
sfdSourceCat : `lsst.afw.table.SourceCatalog`
Default : None
Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
Returns
-------
resultStruct : `lsst.pipe.base.struct.Struct`
contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
outputCat : `lsst.afw.table.source.source.SourceCatalog`
Notes
-----
Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
pixels.
Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
the calexp and the calexp with fakes included returned.
The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
this is then convolved with the PSF at that point.
If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
"""
fakeCat = self.composeFakeCat(fakeCats, skyMap)
if wcs is None:
wcs = exposure.getWcs()
if photoCalib is None:
photoCalib = exposure.getPhotoCalib()
if self.config.doMatchVisit:
fakeCat = self.getVisitMatchedFakeCat(fakeCat, exposure)
self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
# detect, deblend and measure sources
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
returnedStruct = self.calibrate.run(exposure,
exposureIdInfo=exposureIdInfo)
sourceCat = returnedStruct.sourceCat
sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat,
self.config.srcFieldsToCopy)
resultStruct = pipeBase.Struct(outputExposure=exposure,
outputCat=sourceCat)
return resultStruct
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
detectorId = inputs["exposure"].getInfo().getDetector().getId()
if 'exposureIdInfo' not in inputs.keys():
expId, expBits = butlerQC.quantum.dataId.pack("visit_detector",
returnMaxBits=True)
inputs['exposureIdInfo'] = ExposureIdInfo(expId, expBits)
expWcs = inputs["exposure"].getWcs()
tractId = inputs["skyMap"].findTract(
expWcs.pixelToSky(
inputs["exposure"].getBBox().getCenter())).tract_id
if not self.config.doApplyExternalGlobalSkyWcs and not self.config.doApplyExternalTractSkyWcs:
inputs["wcs"] = expWcs
elif self.config.doApplyExternalGlobalSkyWcs:
externalSkyWcsCatalog = inputs["externalSkyWcsGlobalCatalog"]
row = externalSkyWcsCatalog.find(detectorId)
inputs["wcs"] = row.getWcs()
elif self.config.doApplyExternalTractSkyWcs:
externalSkyWcsCatalogList = inputs["externalSkyWcsTractCatalog"]
externalSkyWcsCatalog = None
for externalSkyWcsCatalogRef in externalSkyWcsCatalogList:
if externalSkyWcsCatalogRef.dataId["tract"] == tractId:
externalSkyWcsCatalog = externalSkyWcsCatalogRef.get(
datasetType=self.config.connections.
externalSkyWcsTractCatalog)
break
if externalSkyWcsCatalog is None:
usedTract = externalSkyWcsCatalogList[-1].dataId["tract"]
self.log.warn(
f"Warning, external SkyWcs for tract {tractId} not found. Using tract {usedTract} "
"instead.")
externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get(
datasetType=self.config.connections.
externalSkyWcsTractCatalog)
row = externalSkyWcsCatalog.find(detectorId)
inputs["wcs"] = row.getWcs()
if not self.config.doApplyExternalGlobalPhotoCalib and not self.config.doApplyExternalTractPhotoCalib:
inputs["photoCalib"] = inputs["exposure"].getPhotoCalib()
elif self.config.doApplyExternalGlobalPhotoCalib:
externalPhotoCalibCatalog = inputs[
"externalPhotoCalibGlobalCatalog"]
row = externalPhotoCalibCatalog.find(detectorId)
inputs["photoCalib"] = row.getPhotoCalib()
elif self.config.doApplyExternalTractPhotoCalib:
externalPhotoCalibCatalogList = inputs[
"externalPhotoCalibTractCatalog"]
externalPhotoCalibCatalog = None
for externalPhotoCalibCatalogRef in externalPhotoCalibCatalogList:
if externalPhotoCalibCatalogRef.dataId["tract"] == tractId:
externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get(
datasetType=self.config.connections.
externalPhotoCalibTractCatalog)
break
if externalPhotoCalibCatalog is None:
usedTract = externalPhotoCalibCatalogList[-1].dataId["tract"]
self.log.warn(
f"Warning, external PhotoCalib for tract {tractId} not found. Using tract {usedTract} "
"instead.")
externalPhotoCalibCatalog = externalPhotoCalibCatalogList[
-1].get(datasetType=self.config.connections.
externalPhotoCalibTractCatalog)
row = externalPhotoCalibCatalog.find(detectorId)
inputs["photoCalib"] = row.getPhotoCalib()
outputs = self.run(**inputs)
butlerQC.put(outputs, outputRefs)
def run(self, diaSources, tractPatchId, skymapBits):
"""Associate DiaSources into a collection of DiaObjects using a
brute force matching algorithm.
Reproducible is for the same input data is assured by ordering the
DiaSource data by ccdVisit ordering.
Parameters
----------
diaSources : `pandas.DataFrame`
DiaSources grouped by CcdVisitId to spatially associate into
DiaObjects.
tractPatchId : `int`
Unique identifier for the tract patch.
skymapBits : `int`
Maximum number of bits used the ``tractPatchId`` integer
identifier.
Returns
-------
results : `lsst.pipe.base.Struct`
Results struct with attributes:
``assocDiaSources``
Table of DiaSources with updated values for the DiaObjects
they are spatially associated to (`pandas.DataFrame`).
``diaObjects``
Table of DiaObjects from matching DiaSources
(`pandas.DataFrame`).
"""
# Expected indexes include diaSourceId or meaningless range index
# If meaningless range index, drop it, else keep it.
doDropIndex = diaSources.index.names[0] is None
diaSources.reset_index(inplace=True, drop=doDropIndex)
# Sort by ccdVisit and diaSourceId to get a reproducible ordering for
# the association.
diaSources.set_index(["ccdVisitId", "diaSourceId"], inplace=True)
# Empty lists to store matching and location data.
diaObjectCat = []
diaObjectCoords = []
healPixIndices = []
# Create Id factory and catalog for creating DiaObjectIds.
exposureIdInfo = ExposureIdInfo(tractPatchId, skymapBits)
idFactory = exposureIdInfo.makeSourceIdFactory()
idCat = afwTable.SourceCatalog(
afwTable.SourceTable.make(afwTable.SourceTable.makeMinimalSchema(),
idFactory))
for ccdVisit in diaSources.index.levels[0]:
# For the first ccdVisit, just copy the DiaSource info into the
# diaObject data to create the first set of Objects.
ccdVisitSources = diaSources.loc[ccdVisit]
if len(diaObjectCat) == 0:
for diaSourceId, diaSrc in ccdVisitSources.iterrows():
self.addNewDiaObject(diaSrc,
diaSources,
ccdVisit,
diaSourceId,
diaObjectCat,
idCat,
diaObjectCoords,
healPixIndices)
continue
# Temp list to store DiaObjects already used for this ccdVisit.
usedMatchIndicies = []
# Run over subsequent data.
for diaSourceId, diaSrc in ccdVisitSources.iterrows():
# Find matches.
matchResult = self.findMatches(diaSrc["ra"],
diaSrc["decl"],
2*self.config.tolerance,
healPixIndices,
diaObjectCat)
dists = matchResult.dists
matches = matchResult.matches
# Create a new DiaObject if no match found.
if dists is None:
self.addNewDiaObject(diaSrc,
diaSources,
ccdVisit,
diaSourceId,
diaObjectCat,
idCat,
diaObjectCoords,
healPixIndices)
continue
# If matched, update catalogs and arrays.
if np.min(dists) < np.deg2rad(self.config.tolerance/3600):
matchDistArg = np.argmin(dists)
matchIndex = matches[matchDistArg]
# Test to see if the DiaObject has been used.
if np.isin([matchIndex], usedMatchIndicies).sum() < 1:
self.updateCatalogs(matchIndex,
diaSrc,
diaSources,
ccdVisit,
diaSourceId,
diaObjectCat,
diaObjectCoords,
healPixIndices)
usedMatchIndicies.append(matchIndex)
# If the matched DiaObject has already been used, create a
# new DiaObject for this DiaSource.
else:
self.addNewDiaObject(diaSrc,
diaSources,
ccdVisit,
diaSourceId,
diaObjectCat,
idCat,
diaObjectCoords,
healPixIndices)
# Create new DiaObject if no match found within the matching
# tolerance.
else:
self.addNewDiaObject(diaSrc,
diaSources,
ccdVisit,
diaSourceId,
diaObjectCat,
idCat,
diaObjectCoords,
healPixIndices)
# Drop indices before returning associated diaSource catalog.
diaSources.reset_index(inplace=True)
diaSources.set_index("diaSourceId", inplace=True, verify_integrity=True)
objs = diaObjectCat if diaObjectCat else np.array([], dtype=[('diaObjectId', 'int64'),
('ra', 'float64'),
('decl', 'float64'),
('nDiaSources', 'int64')])
diaObjects = pd.DataFrame(data=objs)
if "diaObjectId" in diaObjects.columns:
diaObjects.set_index("diaObjectId", inplace=True, verify_integrity=True)
return pipeBase.Struct(
assocDiaSources=diaSources,
diaObjects=diaObjects)
def generateMeasCat(self, exposureDataId, exposure, refCat, refCatInBand,
refWcs, idPackerName, footprintData):
"""Generate a measurement catalog for Gen3.
Parameters
----------
exposureDataId : `DataId`
Butler dataId for this exposure.
exposure : `lsst.afw.image.exposure.Exposure`
Exposure to generate the catalog for.
refCat : `lsst.afw.table.SourceCatalog`
Catalog of shapes and positions at which to force photometry.
refCatInBand : `lsst.afw.table.SourceCatalog`
Catalog of shapes and position in the band forced photometry is
currently being performed
refWcs : `lsst.afw.image.SkyWcs`
Reference world coordinate system.
idPackerName : `str`
Type of ID packer to construct from the registry.
footprintData : `ScarletDataModel` or `lsst.afw.table.SourceCatalog`
Either the scarlet data models or the deblended catalog
containing footprints.
If `footprintData` is `None` then the footprints contained
in `refCatInBand` are used.
Returns
-------
measCat : `lsst.afw.table.SourceCatalog`
Catalog of forced sources to measure.
expId : `int`
Unique binary id associated with the input exposure
Raises
------
LookupError
Raised if a footprint with a given source id was in the reference
catalog but not in the reference catalog in band (meaning there
was some sort of mismatch in the two input catalogs)
"""
exposureIdInfo = ExposureIdInfo.fromDataId(exposureDataId,
idPackerName)
idFactory = exposureIdInfo.makeSourceIdFactory()
measCat = self.measurement.generateMeasCat(exposure,
refCat,
refWcs,
idFactory=idFactory)
# attach footprints here, as the attachFootprints method is geared for gen2
# and is not worth modifying, as this can naturally live inside this method
if self.config.footprintDatasetName == "ScarletModelData":
# Load the scarlet models
self._attachScarletFootprints(catalog=measCat,
modelData=footprintData,
exposure=exposure,
band=exposureDataId["band"])
else:
if self.config.footprintDatasetName is None:
footprintCat = refCatInBand
else:
footprintCat = footprintData
for srcRecord in measCat:
fpRecord = footprintCat.find(srcRecord.getId())
if fpRecord is None:
raise LookupError(
"Cannot find Footprint for source {}; please check that {} "
"IDs are compatible with reference source IDs".format(
srcRecord.getId(), footprintCat))
srcRecord.setFootprint(fpRecord.getFootprint())
return measCat, exposureIdInfo.expId
def run(self, exposure, exposureIdInfo=None, background=None,
icSourceCat=None):
"""!Calibrate an exposure (science image or coadd)
@param[in,out] exposure exposure to calibrate (an
lsst.afw.image.ExposureF or similar);
in:
- MaskedImage
- Psf
out:
- MaskedImage has background subtracted
- Wcs is replaced
- PhotoCalib is replaced
@param[in] exposureIdInfo ID info for exposure (an
lsst.obs.base.ExposureIdInfo) If not provided, returned
SourceCatalog IDs will not be globally unique.
@param[in,out] background background model already subtracted from
exposure (an lsst.afw.math.BackgroundList). May be None if no
background has been subtracted, though that is unusual for
calibration. A refined background model is output.
@param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
from which we can copy some fields.
@return pipe_base Struct containing these fields:
- exposure calibrate science exposure with refined WCS and PhotoCalib
- background model of background subtracted from exposure (an
lsst.afw.math.BackgroundList)
- sourceCat catalog of measured sources
- astromMatches list of source/refObj matches from the astrometry
solver
"""
# detect, deblend and measure sources
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
if background is None:
background = BackgroundList()
sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
exposureIdInfo.unusedBits)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
detRes = self.detection.run(table=table, exposure=exposure,
doSmooth=True)
sourceCat = detRes.sources
if detRes.fpSets.background:
for bg in detRes.fpSets.background:
background.append(bg)
if self.config.doSkySources:
skySourceFootprints = self.skySources.run(mask=exposure.mask, seed=exposureIdInfo.expId)
if skySourceFootprints:
for foot in skySourceFootprints:
s = sourceCat.addNew()
s.setFootprint(foot)
s.set(self.skySourceKey, True)
if self.config.doDeblend:
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(
measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId
)
if self.config.doApCorr:
self.applyApCorr.run(
catalog=sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap()
)
self.catalogCalculation.run(sourceCat)
self.setPrimaryFlags.run(sourceCat, includeDeblend=self.config.doDeblend)
if icSourceCat is not None and \
len(self.config.icSourceFieldsToCopy) > 0:
self.copyIcSourceFields(icSourceCat=icSourceCat,
sourceCat=sourceCat)
# TODO DM-11568: this contiguous check-and-copy could go away if we
# reserve enough space during SourceDetection and/or SourceDeblend.
# NOTE: sourceSelectors require contiguous catalogs, so ensure
# contiguity now, so views are preserved from here on.
if not sourceCat.isContiguous():
sourceCat = sourceCat.copy(deep=True)
# perform astrometry calibration:
# fit an improved WCS and update the exposure's WCS in place
astromMatches = None
matchMeta = None
if self.config.doAstrometry:
try:
astromRes = self.astrometry.run(
exposure=exposure,
sourceCat=sourceCat,
)
astromMatches = astromRes.matches
matchMeta = astromRes.matchMeta
except Exception as e:
if self.config.requireAstrometry:
raise
self.log.warn("Unable to perform astrometric calibration "
"(%s): attempting to proceed" % e)
# compute photometric calibration
if self.config.doPhotoCal:
try:
photoRes = self.photoCal.run(exposure, sourceCat=sourceCat, expId=exposureIdInfo.expId)
exposure.setPhotoCalib(photoRes.photoCalib)
# TODO: reword this to phrase it in terms of the calibration factor?
self.log.info("Photometric zero-point: %f" %
photoRes.photoCalib.instFluxToMagnitude(1.0))
self.setMetadata(exposure=exposure, photoRes=photoRes)
except Exception as e:
if self.config.requirePhotoCal:
raise
self.log.warn("Unable to perform photometric calibration "
"(%s): attempting to proceed" % e)
self.setMetadata(exposure=exposure, photoRes=None)
if self.config.doInsertFakes:
self.insertFakes.run(exposure, background=background)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
detRes = self.detection.run(table=table, exposure=exposure,
doSmooth=True)
sourceCat = detRes.sources
if detRes.fpSets.background:
for bg in detRes.fpSets.background:
background.append(bg)
if self.config.doDeblend:
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(
measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId
)
if self.config.doApCorr:
self.applyApCorr.run(
catalog=sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap()
)
self.catalogCalculation.run(sourceCat)
if icSourceCat is not None and len(self.config.icSourceFieldsToCopy) > 0:
self.copyIcSourceFields(icSourceCat=icSourceCat,
sourceCat=sourceCat)
frame = getDebugFrame(self._display, "calibrate")
if frame:
displayAstrometry(
sourceCat=sourceCat,
exposure=exposure,
matches=astromMatches,
frame=frame,
pause=False,
)
return pipeBase.Struct(
exposure=exposure,
background=background,
sourceCat=sourceCat,
astromMatches=astromMatches,
matchMeta=matchMeta,
# These are duplicate entries with different names for use with
# gen3 middleware
outputExposure=exposure,
outputCat=sourceCat,
outputBackground=background,
)
def run(self,
fakeCats,
exposure,
skyMap,
wcs=None,
photoCalib=None,
exposureIdInfo=None,
icSourceCat=None,
sfdSourceCat=None):
"""Add fake sources to a calexp and then run detection, deblending and measurement.
Parameters
----------
fakeCat : `pandas.core.frame.DataFrame`
The catalog of fake sources to add to the exposure
exposure : `lsst.afw.image.exposure.exposure.ExposureF`
The exposure to add the fake sources to
skyMap : `lsst.skymap.SkyMap`
SkyMap defining the tracts and patches the fakes are stored over.
wcs : `lsst.afw.geom.SkyWcs`
WCS to use to add fake sources
photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
Photometric calibration to be used to calibrate the fake sources
exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
icSourceCat : `lsst.afw.table.SourceCatalog`
Default : None
Catalog to take the information about which sources were used for calibration from.
sfdSourceCat : `lsst.afw.table.SourceCatalog`
Default : None
Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
Returns
-------
resultStruct : `lsst.pipe.base.struct.Struct`
Results Strcut containing:
- outputExposure : Exposure with added fakes
(`lsst.afw.image.exposure.exposure.ExposureF`)
- outputCat : Catalog with detected fakes
(`lsst.afw.table.source.source.SourceCatalog`)
- ccdVisitFakeMagnitudes : Magnitudes that these fakes were
inserted with after being scattered (`pandas.DataFrame`)
Notes
-----
Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
pixels.
Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
the calexp and the calexp with fakes included returned.
The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
this is then convolved with the PSF at that point.
If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
"""
fakeCat = self.composeFakeCat(fakeCats, skyMap)
if wcs is None:
wcs = exposure.getWcs()
if photoCalib is None:
photoCalib = exposure.getPhotoCalib()
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
band = exposure.getFilter().bandLabel
ccdVisitMagnitudes = self.addVariablity(fakeCat, band, exposure,
photoCalib, exposureIdInfo)
self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
# detect, deblend and measure sources
returnedStruct = self.calibrate.run(exposure,
exposureIdInfo=exposureIdInfo)
sourceCat = returnedStruct.sourceCat
sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat,
self.config.srcFieldsToCopy)
resultStruct = pipeBase.Struct(
outputExposure=exposure,
outputCat=sourceCat,
ccdVisitFakeMagnitudes=ccdVisitMagnitudes)
return resultStruct
def calibrate(self,
exposure,
exposureIdInfo=None,
background=None,
icSourceCat=None):
"""!Calibrate an exposure (science image or coadd)
@param[in,out] exposure exposure to calibrate (an
lsst.afw.image.ExposureF or similar);
in:
- MaskedImage
- Psf
out:
- MaskedImage has background subtracted
- Wcs is replaced
- Calib zero-point is set
@param[in] exposureIdInfo ID info for exposure (an
lsst.obs.base.ExposureIdInfo) If not provided, returned
SourceCatalog IDs will not be globally unique.
@param[in,out] background background model already subtracted from
exposure (an lsst.afw.math.BackgroundList). May be None if no
background has been subtracted, though that is unusual for
calibration. A refined background model is output.
@param[in] icSourceCat A SourceCatalog from CharacterizeImageTask
from which we can copy some fields.
@return pipe_base Struct containing these fields:
- exposure calibrate science exposure with refined WCS and Calib
- background model of background subtracted from exposure (an
lsst.afw.math.BackgroundList)
- sourceCat catalog of measured sources
- astromMatches list of source/refObj matches from the astrometry
solver
"""
# detect, deblend and measure sources
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
if background is None:
background = BackgroundList()
sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
exposureIdInfo.unusedBits)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
if self.config.doInsertFakes:
self.insertFakes.run(exposure, background=background)
detRes = self.detection.run(table=table,
exposure=exposure,
doSmooth=True)
sourceCat = detRes.sources
if detRes.fpSets.background:
background.append(detRes.fpSets.background)
if self.config.doDeblend:
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId)
if self.config.doApCorr:
self.applyApCorr.run(catalog=sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap())
self.catalogCalculation.run(sourceCat)
if icSourceCat is not None and \
len(self.config.icSourceFieldsToCopy) > 0:
self.copyIcSourceFields(icSourceCat=icSourceCat,
sourceCat=sourceCat)
# perform astrometry calibration:
# fit an improved WCS and update the exposure's WCS in place
astromMatches = None
if self.config.doAstrometry:
try:
astromRes = self.astrometry.run(
exposure=exposure,
sourceCat=sourceCat,
)
astromMatches = astromRes.matches
except Exception as e:
if self.config.requireAstrometry:
raise
self.log.warn("Unable to perform astrometric calibration "
"(%s): attempting to proceed" % e)
# compute photometric calibration
if self.config.doPhotoCal:
try:
photoRes = self.photoCal.run(exposure, sourceCat=sourceCat)
exposure.getCalib().setFluxMag0(photoRes.calib.getFluxMag0())
self.log.info("Photometric zero-point: %f" %
photoRes.calib.getMagnitude(1.0))
self.setMetadata(exposure=exposure, photoRes=photoRes)
except Exception as e:
if self.config.requirePhotoCal:
raise
self.log.warn("Unable to perform photometric calibration "
"(%s): attempting to proceed" % e)
self.setMetadata(exposure=exposure, photoRes=None)
frame = getDebugFrame(self._display, "calibrate")
if frame:
displayAstrometry(
sourceCat=sourceCat,
exposure=exposure,
matches=astromMatches,
frame=frame,
pause=False,
)
return pipeBase.Struct(
exposure=exposure,
background=background,
sourceCat=sourceCat,
astromMatches=astromMatches,
)
def run(self,
fakeCat,
exposure,
wcs=None,
photoCalib=None,
exposureIdInfo=None,
icSourceCat=None,
sfdSourceCat=None):
"""Add fake sources to a calexp and then run detection, deblending and measurement.
Parameters
----------
fakeCat : `pandas.core.frame.DataFrame`
The catalog of fake sources to add to the exposure
exposure : `lsst.afw.image.exposure.exposure.ExposureF`
The exposure to add the fake sources to
wcs : `lsst.afw.geom.SkyWcs`
WCS to use to add fake sources
photoCalib : `lsst.afw.image.photoCalib.PhotoCalib`
Photometric calibration to be used to calibrate the fake sources
exposureIdInfo : `lsst.obs.base.ExposureIdInfo`
icSourceCat : `lsst.afw.table.SourceCatalog`
Default : None
Catalog to take the information about which sources were used for calibration from.
sfdSourceCat : `lsst.afw.table.SourceCatalog`
Default : None
Catalog produced by singleFrameDriver, needed to copy some calibration flags from.
Returns
-------
resultStruct : `lsst.pipe.base.struct.Struct`
contains : outputExposure : `lsst.afw.image.exposure.exposure.ExposureF`
outputCat : `lsst.afw.table.source.source.SourceCatalog`
Notes
-----
Adds pixel coordinates for each source to the fakeCat and removes objects with bulge or disk half
light radius = 0 (if ``config.cleanCat = True``). These columns are called ``x`` and ``y`` and are in
pixels.
Adds the ``Fake`` mask plane to the exposure which is then set by `addFakeSources` to mark where fake
sources have been added. Uses the information in the ``fakeCat`` to make fake galaxies (using galsim)
and fake stars, using the PSF models from the PSF information for the calexp. These are then added to
the calexp and the calexp with fakes included returned.
The galsim galaxies are made using a double sersic profile, one for the bulge and one for the disk,
this is then convolved with the PSF at that point.
If exposureIdInfo is not provided then the SourceCatalog IDs will not be globally unique.
"""
if wcs is None:
wcs = exposure.getWcs()
if photoCalib is None:
photoCalib = exposure.getPhotoCalib()
self.insertFakes.run(fakeCat, exposure, wcs, photoCalib)
# detect, deblend and measure sources
if exposureIdInfo is None:
exposureIdInfo = ExposureIdInfo()
sourceIdFactory = IdFactory.makeSource(exposureIdInfo.expId,
exposureIdInfo.unusedBits)
table = SourceTable.make(self.schema, sourceIdFactory)
table.setMetadata(self.algMetadata)
detRes = self.detection.run(table=table,
exposure=exposure,
doSmooth=True)
sourceCat = detRes.sources
self.deblend.run(exposure=exposure, sources=sourceCat)
self.measurement.run(measCat=sourceCat,
exposure=exposure,
exposureId=exposureIdInfo.expId)
self.applyApCorr.run(catalog=sourceCat,
apCorrMap=exposure.getInfo().getApCorrMap())
self.catalogCalculation.run(sourceCat)
sourceCat = self.copyCalibrationFields(
icSourceCat, sourceCat, self.config.calibrationFieldsToCopy)
sourceCat = self.copyCalibrationFields(sfdSourceCat, sourceCat,
self.config.srcFieldsToCopy)
resultStruct = pipeBase.Struct(outputExposure=exposure,
outputCat=sourceCat)
return resultStruct
def runQuantum(self, butlerQC, inputRefs, outputRefs):
inputs = butlerQC.get(inputRefs)
inputs["idFactory"] = ExposureIdInfo.fromDataId(
butlerQC.quantum.dataId, "tract_patch").makeSourceIdFactory()
outputs = self.run(**inputs)
butlerQC.put(outputs, outputRefs)
