def run(self, dataRef):
"""!Compute a few statistics on the image plane of an exposure
@param dataRef: data reference for a calibrated science exposure ("calexp")
@return a pipeBase Struct containing:
- mean: mean of image plane
- meanErr: uncertainty in mean
- stdDev: standard deviation of image plane
- stdDevErr: uncertainty in standard deviation
"""
self.log.info("Processing data ID %s" % (dataRef.dataId, ))
if self.config.doFail:
raise pipeBase.TaskError(
"Raising TaskError by request (config.doFail=True)")
# Unpersist the raw exposure pointed to by the data reference
rawExp = dataRef.get("raw")
maskedImage = rawExp.getMaskedImage()
# Support extra debug output.
# -
import lsstDebug
display = lsstDebug.Info(__name__).display
if display:
frame = 1
mtv(rawExp, frame=frame, title="exposure")
# return the pipe_base Struct that is returned by self.stats.run
return self.stats.run(maskedImage)
def runDataRef(self, dataRefList, camera, butler, tract, debug, diagDir=".",
diagnostics=False, snapshots=False, numCoresForReadSource=1,
readTimeout=9999, verbose=False):
self.log.info("Running self-calibration for tract %d" % tract)
skyMap = butler.get("deepCoadd_skyMap", immediate=True)
tractInfo = skyMap[tract]
filters = set(dataRef.dataId['filter'] for dataRef in dataRefList)
if len(filters) != 1:
self.log.warn("There are %d filters in input frames: %s" % (len(filters), ", ".join(filters)))
if not self.config.allowMixedFilters:
raise pipeBase.TaskError("Multiple filters found: %s" % (filters,))
if self.config.doColorTerms and self.config.photoCatName:
filterName = sorted(filters)[0]
self.log.info("Using color terms for filter %s" % filterName)
ct = self.config.colorterms.getColorterm(filterName, self.config.photoCatName)
self.log.info("color term: " + str(ct))
elif self.config.doColorTerms:
ct = None
self.log.warn("Cannot apply color term: reference catalog not specified")
else:
ct = None
self.log.info("Not applying color term")
return self.run(dataRefList, tractInfo, ct, debug, diagDir, diagnostics, snapshots,
numCoresForReadSource, readTimeout, verbose)
def runDataRef(self, dataRef):
if self.config.doFail:
raise pipeBase.TaskError(
"Failed by request: config.doFail is true")
self.dataRefList.append(dataRef)
self.numProcessed += 1
self.metadata["numProcessed"] = self.numProcessed
return pipeBase.Struct(numProcessed=self.numProcessed, )
def fetchInPatches(self, butler, exposure, tract, patchList):
"""!
Get the reference catalogs from a given tract,patchlist
@param[in] butler A Butler used to get the reference catalogs
@param[in] exposure A deepDiff_exposure on which to run the measurements
@param[in] tract The tract
@param[in] patchList A list of patches that need to be checked
@return Combined SourceCatalog from all the patches
"""
dataset = f"{self.config.coaddName}Diff_diaObject"
catalog = None
for patch in patchList:
dataId = {'tract': tract.getId(), 'patch': "%d,%d" % patch.getIndex()}
self.log.info("Getting references in %s" % (dataId,))
if not butler.datasetExists(dataset, dataId):
if self.config.skipMissing:
self.log.info("Could not find %s for dataset %s" % (dataId, dataset))
continue
raise pipeBase.TaskError("Reference %s doesn't exist" % (dataId,))
new_catalog = butler.get(dataset, dataId, immediate=True)
patchBox = geom.Box2D(patch.getInnerBBox())
tractBox = tract.getInnerSkyPolygon()
tractWcs = tract.getWcs()
expBox = geom.Box2D(exposure.getBBox())
expWcs = exposure.getWcs()
# only use objects that overlap inner patch bounding box and overlap exposure
validPatch = np.array([
patchBox.contains(tractWcs.skyToPixel(s.getCoord())) for s in new_catalog])
# There doesn't seem to be a inner bounding box so I have to use the sphgeom stuff
validTract = np.array(
[tractBox.contains(sphgeom.UnitVector3d(sphgeom.LonLat.fromRadians(s.getRa().asRadians(),
s.getDec().asRadians())))
for s in new_catalog])
validExposure = np.array([
expBox.contains(expWcs.skyToPixel(s.getCoord())) for s in new_catalog])
if validPatch.size == 0 or validExposure.size == 0 or validTract.size == 0:
self.log.debug("No valid sources %s for dataset %s" % (dataId, dataset))
continue
if catalog is None:
catalog = new_catalog[validPatch & validExposure & validTract]
else:
catalog.extend(new_catalog[validPatch & validExposure & validTract])
return catalog
def run(self, exposure, sensorRef, templateIdList):
"""Return a calexp exposure with based on input sensorRef.
Construct a dataId based on the sensorRef.dataId combined
with the specifications from the first dataId in templateIdList
Parameters
----------
exposure : `lsst.afw.image.Exposure`
exposure (unused)
sensorRef : `list` of `lsst.daf.persistence.ButlerDataRef`
Data reference of the calexp(s) to subtract from.
templateIdList : `list` of `lsst.daf.persistence.ButlerDataRef`
Data reference of the template calexp to be subtraced.
Can be incomplete, fields are initialized from `sensorRef`.
If there are multiple items, only the first one is used.
Returns
-------
result : `struct`
return a pipeBase.Struct:
- ``exposure`` : a template calexp
- ``sources`` : source catalog measured on the template
"""
if len(templateIdList) == 0:
raise RuntimeError("No template data reference supplied.")
if len(templateIdList) > 1:
self.log.warning(
"Multiple template data references supplied. Using the first one only."
)
templateId = sensorRef.dataId.copy()
templateId.update(templateIdList[0])
self.log.info("Fetching calexp (%s) as template.", templateId)
butler = sensorRef.getButler()
template = butler.get(datasetType="calexp", dataId=templateId)
if self.config.doAddCalexpBackground:
templateBg = butler.get(datasetType="calexpBackground",
dataId=templateId)
mi = template.getMaskedImage()
mi += templateBg.getImage()
if not template.hasPsf():
raise pipeBase.TaskError("Template has no psf")
templateSources = butler.get(datasetType="src", dataId=templateId)
return pipeBase.Struct(exposure=template, sources=templateSources)
def run(self, exposure, sensorRef, templateIdList):
"""!Return a calexp exposure with same ccd as input sensorRef.
\param[in] exposure -- exposure (unused)
\param[in] sensorRef -- a Butler data reference
\param[in] templateIdList -- list of data ids. Only visit used.
\return a pipeBase.Struct
- exposure: a template calexp
- sources: source catalog measured on the template
"""
#if len(templateIdList):
# self.log.warn("Multiple template visits supplied. Getting template from first visit: %s" %
# (templateIdList[0]['visit']))
#print "refList",templateIdList.refList
#print "dataId", templateIdList.dataId
print "....."
#calexps = templateIdList.getButler().get("calexp")
for elem in templateIdList:
print elem
elem["visit"] = int(elem["visit"])
elem["ccd"] = int(elem["ccd"])
# print elem.dataId["visit"]
print "....."
templateId = templateIdList[0]
# templateId = templateIdList[0].dataId
#dataRefList = templateIdList.idList
#templateId["visit"] = dataRefList[0]['visit']
#templateId = templateIdList.refList
self.log.info("Fetching calexp (%s) as template." % (templateId))
#print templateId
butler = sensorRef.getButler()
template = butler.get(datasetType="calexp", dataId=templateId)
if self.config.doAddCalexpBackground:
templateBg = butler.get(datasetType="calexpBackground",
dataId=templateId)
mi = template.getMaskedImage()
mi += templateBg.getImage()
if not template.hasPsf():
raise pipeBase.TaskError("Template has no psf")
templateSources = butler.get(datasetType="src", dataId=templateId)
return pipeBase.Struct(exposure=template, sources=templateSources)
def run(self, exposure, sensorRef, templateIdList):
"""!Return a calexp exposure with based on input sensorRef.
Construct a dataId based on the sensorRef.dataId combined
with the specifications from the first dataId in templateIdList
\param[in] exposure -- exposure (unused)
\param[in] sensorRef -- a Butler data reference
\param[in] templateIdList -- list of data ids, which should contain a single item.
If there are multiple items, only the first is used.
\return a pipeBase.Struct
- exposure: a template calexp
- sources: source catalog measured on the template
"""
if len(templateIdList) == 0:
raise RuntimeError(
"No template supplied! Please supply a template visit id.")
if len(templateIdList) > 1:
self.log.warn(
"Multiple template visits supplied. Getting template from first visit: %s"
% (templateIdList[0]['visit']))
templateId = sensorRef.dataId.copy()
templateId.update(templateIdList[0])
self.log.info("Fetching calexp (%s) as template." % (templateId))
butler = sensorRef.getButler()
template = butler.get(datasetType="calexp", dataId=templateId)
if self.config.doAddCalexpBackground:
templateBg = butler.get(datasetType="calexpBackground",
dataId=templateId)
mi = template.getMaskedImage()
mi += templateBg.getImage()
if not template.hasPsf():
raise pipeBase.TaskError("Template has no psf")
templateSources = butler.get(datasetType="src", dataId=templateId)
return pipeBase.Struct(exposure=template, sources=templateSources)
def fetchInPatches(self, dataRef, patchList):
"""!
Copied from CoaddSrcReferencesTask and modified to allow loading deepDiff_diaObjects.
The given dataRef must include the tract in its dataId.
"""
dataset = "deepDiff_diaObject"
tract = dataRef.dataId["tract"]
butler = dataRef.butlerSubset.butler
for patch in patchList:
dataId = {'tract': tract, 'patch': "%d,%d" % patch.getIndex()}
if not butler.datasetExists(dataset, dataId):
if self.config.skipMissing:
continue
raise pipeBase.TaskError("Reference %s doesn't exist" %
(dataId, ))
self.log.info("Getting references in %s" % (dataId, ))
catalog = butler.get(dataset, dataId, immediate=True)
for source in catalog:
yield source
def selectRefExposure(self, expRefList, imageScalerList, expDatasetType):
"""Find best exposure to use as the reference exposure
Calculate an appropriate reference exposure by minimizing a cost function that penalizes
high variance, high background level, and low coverage. Use the following config parameters:
- bestRefWeightCoverage
- bestRefWeightVariance
- bestRefWeightLevel
@param[in] expRefList: list of data references to exposures.
Retrieves dataset type specified by expDatasetType.
If an exposure is not found, it is skipped with a warning.
@param[in] imageScalerList: list of image scalers (coaddUtils.ImageScaler);
must be the same length as expRefList
@param[in] expDatasetType: dataset type of exposure: e.g. 'goodSeeingCoadd_tempExp'
@return: index of best exposure
@raise pipeBase.TaskError if none of the exposures in expRefList are found.
"""
self.log.info("Calculating best reference visit")
varList = []
meanBkgdLevelList = []
coverageList = []
if len(expRefList) != len(imageScalerList):
raise RuntimeError(
"len(expRefList) = %s != %s = len(imageScalerList)" %
(len(expRefList), len(imageScalerList)))
for expRef, imageScaler in zip(expRefList, imageScalerList):
exposure = expRef.get(expDatasetType, immediate=True)
maskedImage = exposure.getMaskedImage()
if imageScaler is not None:
try:
imageScaler.scaleMaskedImage(maskedImage)
except:
# need to put a place holder in Arr
varList.append(numpy.nan)
meanBkgdLevelList.append(numpy.nan)
coverageList.append(numpy.nan)
continue
statObjIm = afwMath.makeStatistics(
maskedImage.getImage(), maskedImage.getMask(),
afwMath.MEAN | afwMath.NPOINT | afwMath.VARIANCE, self.sctrl)
meanVar, meanVarErr = statObjIm.getResult(afwMath.VARIANCE)
meanBkgdLevel, meanBkgdLevelErr = statObjIm.getResult(afwMath.MEAN)
npoints, npointsErr = statObjIm.getResult(afwMath.NPOINT)
varList.append(meanVar)
meanBkgdLevelList.append(meanBkgdLevel)
coverageList.append(npoints)
if not coverageList:
raise pipeBase.TaskError(
"None of the candidate %s exist; cannot select best reference exposure"
% (expDatasetType, ))
# Normalize metrics to range from 0 to 1
varArr = numpy.array(varList) / numpy.nanmax(varList)
meanBkgdLevelArr = numpy.array(meanBkgdLevelList) / numpy.nanmax(
meanBkgdLevelList)
coverageArr = numpy.nanmin(coverageList) / numpy.array(coverageList)
costFunctionArr = self.config.bestRefWeightVariance * varArr
costFunctionArr += self.config.bestRefWeightLevel * meanBkgdLevelArr
costFunctionArr += self.config.bestRefWeightCoverage * coverageArr
return numpy.nanargmin(costFunctionArr)
def run(self,
expRefList,
expDatasetType,
imageScalerList=None,
refExpDataRef=None,
refImageScaler=None):
"""Match the backgrounds of a list of coadd temp exposures to a reference coadd temp exposure.
Choose a refExpDataRef automatically if none supplied.
@param[in] expRefList: list of data references to science exposures to be background-matched;
all exposures must exist.
@param[in] expDatasetType: dataset type of exposures, e.g. 'goodSeeingCoadd_tempExp'
@param[in] imageScalerList: list of image scalers (coaddUtils.ImageScaler);
if None then the images are not scaled
@param[in] refExpDataRef: data reference for the reference exposure.
If None, then this task selects the best exposures from expRefList.
if not None then must be one of the exposures in expRefList.
@param[in] refImageScaler: image scaler for reference image;
ignored if refExpDataRef is None, else scaling is not performed if None
@return: a pipBase.Struct containing these fields:
- backgroundInfoList: a list of pipeBase.Struct, one per exposure in expRefList,
each of which contains these fields:
- isReference: this is the reference exposure (only one returned Struct will
contain True for this value, unless the ref exposure is listed multiple times)
- backgroundModel: differential background model (afw.Math.Background or afw.Math.Approximate).
Add this to the science exposure to match the reference exposure.
- fitRMS: rms of the fit. This is the sqrt(mean(residuals**2)).
- matchedMSE: the MSE of the reference and matched images: mean((refImage - matchedSciImage)**2);
should be comparable to difference image's mean variance.
- diffImVar: the mean variance of the difference image.
All fields except isReference will be None if isReference True or the fit failed.
@warning: all exposures must exist on disk
"""
numExp = len(expRefList)
if numExp < 1:
raise pipeBase.TaskError("No exposures to match")
if expDatasetType is None:
raise pipeBase.TaskError("Must specify expDatasetType")
if imageScalerList is None:
self.log.info(
"imageScalerList is None; no scaling will be performed")
imageScalerList = [None] * numExp
if len(expRefList) != len(imageScalerList):
raise RuntimeError(
"len(expRefList) = %s != %s = len(imageScalerList)" %
(len(expRefList), len(imageScalerList)))
refInd = None
if refExpDataRef is None:
# select the best reference exposure from expRefList
refInd = self.selectRefExposure(
expRefList=expRefList,
imageScalerList=imageScalerList,
expDatasetType=expDatasetType,
)
refExpDataRef = expRefList[refInd]
refImageScaler = imageScalerList[refInd]
# refIndSet is the index of all exposures in expDataList that match the reference.
# It is used to avoid background-matching an exposure to itself. It is a list
# because it is possible (though unlikely) that expDataList will contain duplicates.
expKeyList = refExpDataRef.butlerSubset.butler.getKeys(expDatasetType)
refMatcher = DataRefMatcher(refExpDataRef.butlerSubset.butler,
expDatasetType)
refIndSet = set(
refMatcher.matchList(ref0=refExpDataRef, refList=expRefList))
if refInd is not None and refInd not in refIndSet:
raise RuntimeError(
"Internal error: selected reference %s not found in expRefList"
)
refExposure = refExpDataRef.get(expDatasetType, immediate=True)
if refImageScaler is not None:
refMI = refExposure.getMaskedImage()
refImageScaler.scaleMaskedImage(refMI)
debugIdKeyList = tuple(set(expKeyList) - set(['tract', 'patch']))
self.log.info("Matching %d Exposures" % (numExp))
backgroundInfoList = []
for ind, (toMatchRef,
imageScaler) in enumerate(zip(expRefList, imageScalerList)):
if ind in refIndSet:
backgroundInfoStruct = pipeBase.Struct(
isReference=True,
backgroundModel=None,
fitRMS=0.0,
matchedMSE=None,
diffImVar=None,
)
else:
self.log.info("Matching background of %s to %s" %
(toMatchRef.dataId, refExpDataRef.dataId))
try:
toMatchExposure = toMatchRef.get(expDatasetType,
immediate=True)
if imageScaler is not None:
toMatchMI = toMatchExposure.getMaskedImage()
imageScaler.scaleMaskedImage(toMatchMI)
# store a string specifying the visit to label debug plot
self.debugDataIdString = ''.join(
[str(toMatchRef.dataId[vk]) for vk in debugIdKeyList])
backgroundInfoStruct = self.matchBackgrounds(
refExposure=refExposure,
sciExposure=toMatchExposure,
)
backgroundInfoStruct.isReference = False
except Exception as e:
self.log.warn("Failed to fit background %s: %s" %
(toMatchRef.dataId, e))
backgroundInfoStruct = pipeBase.Struct(
isReference=False,
backgroundModel=None,
fitRMS=None,
matchedMSE=None,
diffImVar=None,
)
backgroundInfoList.append(backgroundInfoStruct)
return pipeBase.Struct(backgroundInfoList=backgroundInfoList)
def run(self, sensorRef, templateIdList=None):
"""Subtract an image from a template coadd and measure the result
Steps include:
- warp template coadd to match WCS of image
- PSF match image to warped template
- subtract image from PSF-matched, warped template
- persist difference image
- detect sources
- measure sources
@param sensorRef: sensor-level butler data reference, used for the following data products:
Input only:
- calexp
- psf
- ccdExposureId
- ccdExposureId_bits
- self.config.coaddName + "Coadd_skyMap"
- self.config.coaddName + "Coadd"
Input or output, depending on config:
- self.config.coaddName + "Diff_subtractedExp"
Output, depending on config:
- self.config.coaddName + "Diff_matchedExp"
- self.config.coaddName + "Diff_src"
@return pipe_base Struct containing these fields:
- subtractedExposure: exposure after subtracting template;
the unpersisted version if subtraction not run but detection run
None if neither subtraction nor detection run (i.e. nothing useful done)
- subtractRes: results of subtraction task; None if subtraction not run
- sources: detected and possibly measured sources; None if detection not run
"""
self.log.info("Processing %s" % (sensorRef.dataId))
# initialize outputs and some intermediate products
subtractedExposure = None
subtractRes = None
selectSources = None
kernelSources = None
controlSources = None
diaSources = None
# We make one IdFactory that will be used by both icSrc and src datasets;
# I don't know if this is the way we ultimately want to do things, but at least
# this ensures the source IDs are fully unique.
expBits = sensorRef.get("ccdExposureId_bits")
expId = int(sensorRef.get("ccdExposureId"))
idFactory = afwTable.IdFactory.makeSource(expId, 64 - expBits)
# Retrieve the science image we wish to analyze
exposure = sensorRef.get("calexp", immediate=True)
if self.config.doAddCalexpBackground:
mi = exposure.getMaskedImage()
mi += sensorRef.get("calexpBackground").getImage()
if not exposure.hasPsf():
raise pipeBase.TaskError("Exposure has no psf")
sciencePsf = exposure.getPsf()
subtractedExposureName = self.config.coaddName + "Diff_differenceExp"
templateExposure = None # Stitched coadd exposure
templateSources = None # Sources on the template image
if self.config.doSubtract:
template = self.getTemplate.run(exposure,
sensorRef,
templateIdList=templateIdList)
templateExposure = template.exposure
templateSources = template.sources
# compute scienceSigmaOrig: sigma of PSF of science image before pre-convolution
scienceSigmaOrig = sciencePsf.computeShape().getDeterminantRadius()
# sigma of PSF of template image before warping
templateSigma = templateExposure.getPsf().computeShape(
).getDeterminantRadius()
# if requested, convolve the science exposure with its PSF
# (properly, this should be a cross-correlation, but our code does not yet support that)
# compute scienceSigmaPost: sigma of science exposure with pre-convolution, if done,
# else sigma of original science exposure
if self.config.doPreConvolve:
convControl = afwMath.ConvolutionControl()
# cannot convolve in place, so make a new MI to receive convolved image
srcMI = exposure.getMaskedImage()
destMI = srcMI.Factory(srcMI.getDimensions())
srcPsf = sciencePsf
if self.config.useGaussianForPreConvolution:
# convolve with a simplified PSF model: a double Gaussian
kWidth, kHeight = sciencePsf.getLocalKernel(
).getDimensions()
preConvPsf = SingleGaussianPsf(kWidth, kHeight,
scienceSigmaOrig)
else:
# convolve with science exposure's PSF model
preConvPsf = srcPsf
afwMath.convolve(destMI, srcMI, preConvPsf.getLocalKernel(),
convControl)
exposure.setMaskedImage(destMI)
scienceSigmaPost = scienceSigmaOrig * math.sqrt(2)
else:
scienceSigmaPost = scienceSigmaOrig
# If requested, find sources in the image
if self.config.doSelectSources:
if not sensorRef.datasetExists("src"):
self.log.warn(
"Src product does not exist; running detection, measurement, selection"
)
# Run own detection and measurement; necessary in nightly processing
selectSources = self.subtract.getSelectSources(
exposure,
sigma=scienceSigmaPost,
doSmooth=not self.doPreConvolve,
idFactory=idFactory,
)
else:
self.log.info("Source selection via src product")
# Sources already exist; for data release processing
selectSources = sensorRef.get("src")
# Number of basis functions
nparam = len(
makeKernelBasisList(
self.subtract.config.kernel.active,
referenceFwhmPix=scienceSigmaPost * FwhmPerSigma,
targetFwhmPix=templateSigma * FwhmPerSigma))
if self.config.doAddMetrics:
# Modify the schema of all Sources
kcQa = KernelCandidateQa(nparam)
selectSources = kcQa.addToSchema(selectSources)
if self.config.kernelSourcesFromRef:
# match exposure sources to reference catalog
astromRet = self.astrometer.loadAndMatch(
exposure=exposure, sourceCat=selectSources)
matches = astromRet.matches
elif templateSources:
# match exposure sources to template sources
mc = afwTable.MatchControl()
mc.findOnlyClosest = False
matches = afwTable.matchRaDec(templateSources,
selectSources,
1.0 * afwGeom.arcseconds, mc)
else:
raise RuntimeError(
"doSelectSources=True and kernelSourcesFromRef=False,"
+
"but template sources not available. Cannot match science "
+
"sources with template sources. Run process* on data from "
+ "which templates are built.")
kernelSources = self.sourceSelector.selectStars(
exposure, selectSources, matches=matches).starCat
random.shuffle(kernelSources, random.random)
controlSources = kernelSources[::self.config.controlStepSize]
kernelSources = [
k for i, k in enumerate(kernelSources)
if i % self.config.controlStepSize
]
if self.config.doSelectDcrCatalog:
redSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(
grMin=self.sourceSelector.config.grMax,
grMax=99.999))
redSources = redSelector.selectStars(
exposure, selectSources, matches=matches).starCat
controlSources.extend(redSources)
blueSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(
grMin=-99.999,
grMax=self.sourceSelector.config.grMin))
blueSources = blueSelector.selectStars(
exposure, selectSources, matches=matches).starCat
controlSources.extend(blueSources)
if self.config.doSelectVariableCatalog:
varSelector = DiaCatalogSourceSelectorTask(
DiaCatalogSourceSelectorConfig(includeVariable=True))
varSources = varSelector.selectStars(
exposure, selectSources, matches=matches).starCat
controlSources.extend(varSources)
self.log.info(
"Selected %d / %d sources for Psf matching (%d for control sample)"
% (len(kernelSources), len(selectSources),
len(controlSources)))
allresids = {}
if self.config.doUseRegister:
self.log.info("Registering images")
if templateSources is None:
# Run detection on the template, which is
# temporarily background-subtracted
templateSources = self.subtract.getSelectSources(
templateExposure,
sigma=templateSigma,
doSmooth=True,
idFactory=idFactory)
# Third step: we need to fit the relative astrometry.
#
wcsResults = self.fitAstrometry(templateSources,
templateExposure,
selectSources)
warpedExp = self.register.warpExposure(templateExposure,
wcsResults.wcs,
exposure.getWcs(),
exposure.getBBox())
templateExposure = warpedExp
# Create debugging outputs on the astrometric
# residuals as a function of position. Persistence
# not yet implemented; expected on (I believe) #2636.
if self.config.doDebugRegister:
# Grab matches to reference catalog
srcToMatch = {x.second.getId(): x.first for x in matches}
refCoordKey = wcsResults.matches[0].first.getTable(
).getCoordKey()
inCentroidKey = wcsResults.matches[0].second.getTable(
).getCentroidKey()
sids = [m.first.getId() for m in wcsResults.matches]
positions = [
m.first.get(refCoordKey) for m in wcsResults.matches
]
residuals = [
m.first.get(refCoordKey).getOffsetFrom(
wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey)))
for m in wcsResults.matches
]
allresids = dict(zip(sids, zip(positions, residuals)))
cresiduals = [
m.first.get(refCoordKey).getTangentPlaneOffset(
wcsResults.wcs.pixelToSky(
m.second.get(inCentroidKey)))
for m in wcsResults.matches
]
colors = numpy.array([
-2.5 * numpy.log10(srcToMatch[x].get("g")) +
2.5 * numpy.log10(srcToMatch[x].get("r")) for x in sids
if x in srcToMatch.keys()
])
dlong = numpy.array([
r[0].asArcseconds() for s, r in zip(sids, cresiduals)
if s in srcToMatch.keys()
])
dlat = numpy.array([
r[1].asArcseconds() for s, r in zip(sids, cresiduals)
if s in srcToMatch.keys()
])
idx1 = numpy.where(
colors < self.sourceSelector.config.grMin)
idx2 = numpy.where(
(colors >= self.sourceSelector.config.grMin)
& (colors <= self.sourceSelector.config.grMax))
idx3 = numpy.where(
colors > self.sourceSelector.config.grMax)
rms1Long = IqrToSigma * \
(numpy.percentile(dlong[idx1], 75)-numpy.percentile(dlong[idx1], 25))
rms1Lat = IqrToSigma * (numpy.percentile(dlat[idx1], 75) -
numpy.percentile(dlat[idx1], 25))
rms2Long = IqrToSigma * \
(numpy.percentile(dlong[idx2], 75)-numpy.percentile(dlong[idx2], 25))
rms2Lat = IqrToSigma * (numpy.percentile(dlat[idx2], 75) -
numpy.percentile(dlat[idx2], 25))
rms3Long = IqrToSigma * \
(numpy.percentile(dlong[idx3], 75)-numpy.percentile(dlong[idx3], 25))
rms3Lat = IqrToSigma * (numpy.percentile(dlat[idx3], 75) -
numpy.percentile(dlat[idx3], 25))
self.log.info("Blue star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx1]), rms1Long,
numpy.median(dlat[idx1]), rms1Lat))
self.log.info(
"Green star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx2]), rms2Long,
numpy.median(dlat[idx2]), rms2Lat))
self.log.info("Red star offsets'': %.3f %.3f, %.3f %.3f" %
(numpy.median(dlong[idx3]), rms3Long,
numpy.median(dlat[idx3]), rms3Lat))
self.metadata.add("RegisterBlueLongOffsetMedian",
numpy.median(dlong[idx1]))
self.metadata.add("RegisterGreenLongOffsetMedian",
numpy.median(dlong[idx2]))
self.metadata.add("RegisterRedLongOffsetMedian",
numpy.median(dlong[idx3]))
self.metadata.add("RegisterBlueLongOffsetStd", rms1Long)
self.metadata.add("RegisterGreenLongOffsetStd", rms2Long)
self.metadata.add("RegisterRedLongOffsetStd", rms3Long)
self.metadata.add("RegisterBlueLatOffsetMedian",
numpy.median(dlat[idx1]))
self.metadata.add("RegisterGreenLatOffsetMedian",
numpy.median(dlat[idx2]))
self.metadata.add("RegisterRedLatOffsetMedian",
numpy.median(dlat[idx3]))
self.metadata.add("RegisterBlueLatOffsetStd", rms1Lat)
self.metadata.add("RegisterGreenLatOffsetStd", rms2Lat)
self.metadata.add("RegisterRedLatOffsetStd", rms3Lat)
# warp template exposure to match exposure,
# PSF match template exposure to exposure,
# then return the difference
# Return warped template... Construct sourceKernelCand list after subtract
self.log.info("Subtracting images")
subtractRes = self.subtract.subtractExposures(
templateExposure=templateExposure,
scienceExposure=exposure,
candidateList=kernelSources,
convolveTemplate=self.config.convolveTemplate,
doWarping=not self.config.doUseRegister)
subtractedExposure = subtractRes.subtractedExposure
if self.config.doWriteMatchedExp:
sensorRef.put(subtractRes.matchedExposure,
self.config.coaddName + "Diff_matchedExp")
if self.config.doDetection:
self.log.info("Computing diffim PSF")
if subtractedExposure is None:
subtractedExposure = sensorRef.get(subtractedExposureName)
# Get Psf from the appropriate input image if it doesn't exist
if not subtractedExposure.hasPsf():
if self.config.convolveTemplate:
subtractedExposure.setPsf(exposure.getPsf())
else:
if templateExposure is None:
template = self.getTemplate.run(
exposure, sensorRef, templateIdList=templateIdList)
subtractedExposure.setPsf(template.exposure.getPsf())
# If doSubtract is False, then subtractedExposure was fetched from disk (above), thus it may have
# already been decorrelated. Thus, we do not do decorrelation if doSubtract is False.
if self.config.doDecorrelation and self.config.doSubtract:
decorrResult = self.decorrelate.run(exposure, templateExposure,
subtractedExposure,
subtractRes.psfMatchingKernel)
subtractedExposure = decorrResult.correctedExposure
if self.config.doDetection:
self.log.info("Running diaSource detection")
# Erase existing detection mask planes
mask = subtractedExposure.getMaskedImage().getMask()
mask &= ~(mask.getPlaneBitMask("DETECTED")
| mask.getPlaneBitMask("DETECTED_NEGATIVE"))
table = afwTable.SourceTable.make(self.schema, idFactory)
table.setMetadata(self.algMetadata)
results = self.detection.makeSourceCatalog(
table=table,
exposure=subtractedExposure,
doSmooth=not self.config.doPreConvolve)
if self.config.doMerge:
fpSet = results.fpSets.positive
fpSet.merge(results.fpSets.negative, self.config.growFootprint,
self.config.growFootprint, False)
diaSources = afwTable.SourceCatalog(table)
fpSet.makeSources(diaSources)
self.log.info("Merging detections into %d sources" %
(len(diaSources)))
else:
diaSources = results.sources
if self.config.doMeasurement:
self.log.info("Running diaSource measurement")
if not self.config.doDipoleFitting:
self.measurement.run(diaSources, subtractedExposure)
else:
if self.config.doSubtract:
self.measurement.run(diaSources, subtractedExposure,
exposure,
subtractRes.matchedExposure)
else:
self.measurement.run(diaSources, subtractedExposure,
exposure)
# Match with the calexp sources if possible
if self.config.doMatchSources:
if sensorRef.datasetExists("src"):
# Create key,val pair where key=diaSourceId and val=sourceId
matchRadAsec = self.config.diaSourceMatchRadius
matchRadPixel = matchRadAsec / exposure.getWcs(
).pixelScale().asArcseconds()
srcMatches = afwTable.matchXy(sensorRef.get("src"),
diaSources, matchRadPixel)
srcMatchDict = dict([(srcMatch.second.getId(),
srcMatch.first.getId())
for srcMatch in srcMatches])
self.log.info("Matched %d / %d diaSources to sources" %
(len(srcMatchDict), len(diaSources)))
else:
self.log.warn(
"Src product does not exist; cannot match with diaSources"
)
srcMatchDict = {}
# Create key,val pair where key=diaSourceId and val=refId
refAstromConfig = AstrometryConfig()
refAstromConfig.matcher.maxMatchDistArcSec = matchRadAsec
refAstrometer = AstrometryTask(refAstromConfig)
astromRet = refAstrometer.run(exposure=exposure,
sourceCat=diaSources)
refMatches = astromRet.matches
if refMatches is None:
self.log.warn(
"No diaSource matches with reference catalog")
refMatchDict = {}
else:
self.log.info(
"Matched %d / %d diaSources to reference catalog" %
(len(refMatches), len(diaSources)))
refMatchDict = dict([(refMatch.second.getId(),
refMatch.first.getId())
for refMatch in refMatches])
# Assign source Ids
for diaSource in diaSources:
sid = diaSource.getId()
if sid in srcMatchDict:
diaSource.set("srcMatchId", srcMatchDict[sid])
if sid in refMatchDict:
diaSource.set("refMatchId", refMatchDict[sid])
if diaSources is not None and self.config.doWriteSources:
sensorRef.put(diaSources,
self.config.coaddName + "Diff_diaSrc")
if self.config.doAddMetrics and self.config.doSelectSources:
self.log.info("Evaluating metrics and control sample")
kernelCandList = []
for cell in subtractRes.kernelCellSet.getCellList():
for cand in cell.begin(False): # include bad candidates
kernelCandList.append(cand)
# Get basis list to build control sample kernels
basisList = kernelCandList[0].getKernel(
KernelCandidateF.ORIG).getKernelList()
controlCandList = \
diffimTools.sourceTableToCandidateList(controlSources,
subtractRes.warpedExposure, exposure,
self.config.subtract.kernel.active,
self.config.subtract.kernel.active.detectionConfig,
self.log, doBuild=True, basisList=basisList)
kcQa.apply(kernelCandList,
subtractRes.psfMatchingKernel,
subtractRes.backgroundModel,
dof=nparam)
kcQa.apply(controlCandList, subtractRes.psfMatchingKernel,
subtractRes.backgroundModel)
if self.config.doDetection:
kcQa.aggregate(selectSources, self.metadata, allresids,
diaSources)
else:
kcQa.aggregate(selectSources, self.metadata, allresids)
sensorRef.put(selectSources,
self.config.coaddName + "Diff_kernelSrc")
if self.config.doWriteSubtractedExp:
sensorRef.put(subtractedExposure, subtractedExposureName)
self.runDebug(exposure, subtractRes, selectSources, kernelSources,
diaSources)
return pipeBase.Struct(
subtractedExposure=subtractedExposure,
subtractRes=subtractRes,
sources=diaSources,
)
def matchObjectsToSources(self, refCat, sourceCat, wcs, sourceFluxField, refFluxField,
match_tolerance=None):
"""Match sources to position reference stars
refCat : `lsst.afw.table.SimpleCatalog`
catalog of reference objects that overlap the exposure; reads
fields for:
- coord
- the specified flux field
sourceCat : `lsst.afw.table.SourceCatalog`
Catalog of sources found on an exposure. This should already be
down-selected to "good"/"usable" sources in the calling Task.
wcs : `lsst.afw.geom.SkyWcs`
estimated WCS
sourceFluxField: `str`
field of sourceCat to use for flux
refFluxField : `str`
field of refCat to use for flux
match_tolerance : `lsst.meas.astrom.MatchTolerancePessimistic`
is a MatchTolerance class object or `None`. This this class is used
to communicate state between AstrometryTask and MatcherTask.
AstrometryTask will also set the MatchTolerance class variable
maxMatchDist based on the scatter AstrometryTask has found after
fitting for the wcs.
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with components:
- ``matches`` : source to reference matches found (`list` of
`lsst.afw.table.ReferenceMatch`)
- ``usableSourceCat`` : a catalog of sources potentially usable for
matching and WCS fitting (`lsst.afw.table.SourceCatalog`).
- ``match_tolerance`` : a MatchTolerance object containing the
resulting state variables from the match
(`lsst.meas.astrom.MatchTolerancePessimistic`).
"""
import lsstDebug
debug = lsstDebug.Info(__name__)
# If we get an empty tolerance struct create the variables we need for
# this matcher.
if match_tolerance is None:
match_tolerance = MatchTolerancePessimistic()
# Make a name alias here for consistency with older code, and to make
# it clear that this is a good/usable (cleaned) source catalog.
goodSourceCat = sourceCat
numUsableSources = len(goodSourceCat)
if len(goodSourceCat) == 0:
raise pipeBase.TaskError("No sources are good")
minMatchedPairs = min(self.config.minMatchedPairs,
int(self.config.minFracMatchedPairs *
min([len(refCat), len(goodSourceCat)])))
if len(refCat) > self.config.maxRefObjects:
self.log.warn(
"WARNING: Reference catalog larger that maximum allowed. "
"Trimming to %i" % self.config.maxRefObjects)
trimmedRefCat = self._filterRefCat(refCat, refFluxField)
else:
trimmedRefCat = refCat
doMatchReturn = self._doMatch(
refCat=trimmedRefCat,
sourceCat=goodSourceCat,
wcs=wcs,
refFluxField=refFluxField,
numUsableSources=numUsableSources,
minMatchedPairs=minMatchedPairs,
match_tolerance=match_tolerance,
sourceFluxField=sourceFluxField,
verbose=debug.verbose,
)
matches = doMatchReturn.matches
match_tolerance = doMatchReturn.match_tolerance
if len(matches) == 0:
raise RuntimeError("Unable to match sources")
self.log.info("Matched %d sources" % len(matches))
if len(matches) < minMatchedPairs:
self.log.warn("Number of matches is smaller than request")
return pipeBase.Struct(
matches=matches,
usableSourceCat=goodSourceCat,
match_tolerance=match_tolerance,
)
def detectFootprints(self,
exposure,
doSmooth=True,
sigma=None,
clearMask=True):
"""!Detect footprints.
\param exposure Exposure to process; DETECTED{,_NEGATIVE} mask plane will be set in-place.
\param doSmooth if True, smooth the image before detection using a Gaussian of width sigma
\param sigma sigma of PSF (pixels); used for smoothing and to grow detections;
if None then measure the sigma of the PSF of the exposure
\param clearMask Clear both DETECTED and DETECTED_NEGATIVE planes before running detection
\return a lsst.pipe.base.Struct with fields:
- positive: lsst.afw.detection.FootprintSet with positive polarity footprints (may be None)
- negative: lsst.afw.detection.FootprintSet with negative polarity footprints (may be None)
- numPos: number of footprints in positive or 0 if detection polarity was negative
- numNeg: number of footprints in negative or 0 if detection polarity was positive
- background: re-estimated background. None if reEstimateBackground==False
\throws lsst.pipe.base.TaskError if sigma=None and the exposure has no PSF
"""
try:
import lsstDebug
display = lsstDebug.Info(__name__).display
except ImportError:
try:
display
except NameError:
display = False
if exposure is None:
raise RuntimeError("No exposure for detection")
maskedImage = exposure.getMaskedImage()
region = maskedImage.getBBox()
if clearMask:
mask = maskedImage.getMask()
mask &= ~(mask.getPlaneBitMask("DETECTED")
| mask.getPlaneBitMask("DETECTED_NEGATIVE"))
del mask
if self.config.doTempLocalBackground:
tempBgRes = self.tempLocalBackground.run(maskedImage)
tempLocalBkgdImage = tempBgRes.background.getImage()
if sigma is None:
psf = exposure.getPsf()
if psf is None:
raise pipeBase.TaskError(
"exposure has no PSF; must specify sigma")
shape = psf.computeShape()
sigma = shape.getDeterminantRadius()
self.metadata.set("sigma", sigma)
self.metadata.set("doSmooth", doSmooth)
if not doSmooth:
convolvedImage = maskedImage.Factory(maskedImage)
middle = convolvedImage
else:
# smooth using a Gaussian (which is separate, hence fast) of width sigma
# make a SingleGaussian (separable) kernel with the 'sigma'
psf = exposure.getPsf()
kWidth = (int(sigma * 7 + 0.5) // 2) * 2 + 1 # make sure it is odd
self.metadata.set("smoothingKernelWidth", kWidth)
gaussFunc = afwMath.GaussianFunction1D(sigma)
gaussKernel = afwMath.SeparableKernel(kWidth, kWidth, gaussFunc,
gaussFunc)
convolvedImage = maskedImage.Factory(maskedImage.getBBox())
afwMath.convolve(convolvedImage, maskedImage, gaussKernel,
afwMath.ConvolutionControl())
#
# Only search psf-smooth part of frame
#
goodBBox = gaussKernel.shrinkBBox(convolvedImage.getBBox())
middle = convolvedImage.Factory(convolvedImage, goodBBox,
afwImage.PARENT, False)
#
# Mark the parts of the image outside goodBBox as EDGE
#
self.setEdgeBits(maskedImage, goodBBox,
maskedImage.getMask().getPlaneBitMask("EDGE"))
fpSets = pipeBase.Struct(positive=None, negative=None)
if self.config.thresholdPolarity != "negative":
fpSets.positive = self.thresholdImage(middle, "positive")
if self.config.reEstimateBackground or self.config.thresholdPolarity != "positive":
fpSets.negative = self.thresholdImage(middle, "negative")
for polarity, maskName in (("positive", "DETECTED"),
("negative", "DETECTED_NEGATIVE")):
fpSet = getattr(fpSets, polarity)
if fpSet is None:
continue
fpSet.setRegion(region)
if self.config.nSigmaToGrow > 0:
nGrow = int((self.config.nSigmaToGrow * sigma) + 0.5)
self.metadata.set("nGrow", nGrow)
fpSet = afwDet.FootprintSet(fpSet, nGrow,
self.config.isotropicGrow)
fpSet.setMask(maskedImage.getMask(), maskName)
if not self.config.returnOriginalFootprints:
setattr(fpSets, polarity, fpSet)
fpSets.numPos = len(fpSets.positive.getFootprints()
) if fpSets.positive is not None else 0
fpSets.numNeg = len(fpSets.negative.getFootprints()
) if fpSets.negative is not None else 0
if self.config.thresholdPolarity != "negative":
self.log.log(
self.log.INFO, "Detected %d positive sources to %g sigma." %
(fpSets.numPos, self.config.thresholdValue *
self.config.includeThresholdMultiplier))
if self.config.doTempLocalBackground:
maskedImage += tempLocalBkgdImage
fpSets.background = None
if self.config.reEstimateBackground:
mi = exposure.getMaskedImage()
bkgd = self.background.fitBackground(mi)
if self.config.adjustBackground:
self.log.log(
self.log.WARN, "Fiddling the background by %g" %
self.config.adjustBackground)
bkgd += self.config.adjustBackground
fpSets.background = bkgd
self.log.log(
self.log.INFO,
"Resubtracting the background after object detection")
mi -= bkgd.getImageF()
del mi
if self.config.thresholdPolarity == "positive":
if self.config.reEstimateBackground:
mask = maskedImage.getMask()
mask &= ~mask.getPlaneBitMask("DETECTED_NEGATIVE")
del mask
fpSets.negative = None
else:
self.log.log(
self.log.INFO, "Detected %d negative sources to %g %s" %
(fpSets.numNeg, self.config.thresholdValue,
("DN" if self.config.thresholdType == "value" else "sigma")))
if display:
ds9.mtv(exposure, frame=0, title="detection")
x0, y0 = exposure.getXY0()
def plotPeaks(fps, ctype):
if fps is None:
return
with ds9.Buffering():
for fp in fps.getFootprints():
for pp in fp.getPeaks():
ds9.dot("+",
pp.getFx() - x0,
pp.getFy() - y0,
ctype=ctype)
plotPeaks(fpSets.positive, "yellow")
plotPeaks(fpSets.negative, "red")
if convolvedImage and display and display > 1:
ds9.mtv(convolvedImage, frame=1, title="PSF smoothed")
return fpSets
class SourceDetectionTask(pipeBase.Task):
"""
Detect positive and negative sources on an exposure and return a new SourceCatalog.
"""
ConfigClass = SourceDetectionConfig
_DefaultName = "sourceDetection"
def __init__(self, schema=None, **kwds):
"""Create the detection task. Most arguments are simply passed onto pipe_base.Task.
If schema is not None, it will be used to register a 'flags.negative' flag field
that will be set for negative detections.
"""
pipeBase.Task.__init__(self, **kwds)
if schema is not None:
self.negativeFlagKey = schema.addField(
"flags.negative",
type="Flag",
doc="set if source was detected as significantly negative")
else:
if self.config.thresholdPolarity == "both":
self.log.log(self.log.WARN, "Detection polarity set to 'both', but no flag will be "\
"set to distinguish between positive and negative detections")
self.negativeFlagKey = None
@pipeBase.timeMethod
def makeSourceCatalog(self,
table,
exposure,
doSmooth=True,
sigma=None,
clearMask=True):
"""Run source detection and create a SourceCatalog.
To avoid dealing with sources and tables, use detectFootprints() to just get the FootprintSets.
@param table lsst.afw.table.SourceTable object that will be used to created the SourceCatalog.
@param exposure Exposure to process; DETECTED mask plane will be set in-place.
@param doSmooth if True, smooth the image before detection using a Gaussian of width sigma
@param sigma sigma of PSF (pixels); used for smoothing and to grow detections;
if None then measure the sigma of the PSF of the exposure
@param clearMask Clear DETECTED{,_NEGATIVE} planes before running detection
@return a Struct with:
sources -- an lsst.afw.table.SourceCatalog object
fpSets --- Struct returned by detectFootprints
@raise pipe_base TaskError if sigma=None, doSmooth=True and the exposure has no PSF
"""
if self.negativeFlagKey is not None and self.negativeFlagKey not in table.getSchema(
):
raise ValueError("Table has incorrect Schema")
fpSets = self.detectFootprints(exposure=exposure,
doSmooth=doSmooth,
sigma=sigma,
clearMask=clearMask)
sources = afwTable.SourceCatalog(table)
table.preallocate(fpSets.numPos +
fpSets.numNeg) # not required, but nice
if fpSets.negative:
fpSets.negative.makeSources(sources)
if self.negativeFlagKey:
for record in sources:
record.set(self.negativeFlagKey, True)
if fpSets.positive:
fpSets.positive.makeSources(sources)
return pipeBase.Struct(sources=sources, fpSets=fpSets)
@pipeBase.timeMethod
def detectFootprints(self,
exposure,
doSmooth=True,
sigma=None,
clearMask=True):
"""Detect footprints.
@param exposure Exposure to process; DETECTED{,_NEGATIVE} mask plane will be set in-place.
@param doSmooth if True, smooth the image before detection using a Gaussian of width sigma
@param sigma sigma of PSF (pixels); used for smoothing and to grow detections;
if None then measure the sigma of the PSF of the exposure
@param clearMask Clear both DETECTED and DETECTED_NEGATIVE planes before running detection
@return a lsst.pipe.base.Struct with fields:
- positive: lsst.afw.detection.FootprintSet with positive polarity footprints (may be None)
- negative: lsst.afw.detection.FootprintSet with negative polarity footprints (may be None)
- numPos: number of footprints in positive or 0 if detection polarity was negative
- numNeg: number of footprints in negative or 0 if detection polarity was positive
- background: re-estimated background. None if reEstimateBackground==False
@raise pipe_base TaskError if sigma=None and the exposure has no PSF
"""
try:
import lsstDebug
display = lsstDebug.Info(__name__).display
except ImportError, e:
try:
display
except NameError:
display = False
if exposure is None:
raise RuntimeException("No exposure for detection")
maskedImage = exposure.getMaskedImage()
region = maskedImage.getBBox(afwImage.PARENT)
if clearMask:
mask = maskedImage.getMask()
mask &= ~(mask.getPlaneBitMask("DETECTED")
| mask.getPlaneBitMask("DETECTED_NEGATIVE"))
del mask
if sigma is None:
psf = exposure.getPsf()
if psf is None:
raise pipeBase.TaskError(
"exposure has no PSF; must specify sigma")
shape = psf.computeShape()
sigma = shape.getDeterminantRadius()
self.metadata.set("sigma", sigma)
self.metadata.set("doSmooth", doSmooth)
if not doSmooth:
convolvedImage = maskedImage.Factory(maskedImage)
middle = convolvedImage
else:
# smooth using a Gaussian (which is separate, hence fast) of width sigma
# make a SingleGaussian (separable) kernel with the 'sigma'
psf = exposure.getPsf()
kWidth = (int(sigma * 7 + 0.5) / 2) * 2 + 1 # make sure it is odd
self.metadata.set("smoothingKernelWidth", kWidth)
gaussFunc = afwMath.GaussianFunction1D(sigma)
gaussKernel = afwMath.SeparableKernel(kWidth, kWidth, gaussFunc,
gaussFunc)
convolvedImage = maskedImage.Factory(
maskedImage.getBBox(afwImage.PARENT))
afwMath.convolve(convolvedImage, maskedImage, gaussKernel,
afwMath.ConvolutionControl())
#
# Only search psf-smooth part of frame
#
goodBBox = gaussKernel.shrinkBBox(
convolvedImage.getBBox(afwImage.PARENT))
middle = convolvedImage.Factory(convolvedImage, goodBBox,
afwImage.PARENT, False)
#
# Mark the parts of the image outside goodBBox as EDGE
#
self.setEdgeBits(maskedImage, goodBBox,
maskedImage.getMask().getPlaneBitMask("EDGE"))
fpSets = pipeBase.Struct(positive=None, negative=None)
if self.config.thresholdPolarity != "negative":
fpSets.positive = self.thresholdImage(middle, "positive")
if self.config.reEstimateBackground or self.config.thresholdPolarity != "positive":
fpSets.negative = self.thresholdImage(middle, "negative")
for polarity, maskName in (("positive", "DETECTED"),
("negative", "DETECTED_NEGATIVE")):
fpSet = getattr(fpSets, polarity)
if fpSet is None:
continue
fpSet.setRegion(region)
if self.config.nSigmaToGrow > 0:
nGrow = int((self.config.nSigmaToGrow * sigma) + 0.5)
self.metadata.set("nGrow", nGrow)
fpSet = afwDet.FootprintSet(fpSet, nGrow, False)
fpSet.setMask(maskedImage.getMask(), maskName)
if not self.config.returnOriginalFootprints:
setattr(fpSets, polarity, fpSet)
fpSets.numPos = len(fpSets.positive.getFootprints()
) if fpSets.positive is not None else 0
fpSets.numNeg = len(fpSets.negative.getFootprints()
) if fpSets.negative is not None else 0
if self.config.thresholdPolarity != "negative":
self.log.log(
self.log.INFO, "Detected %d positive sources to %g sigma." %
(fpSets.numPos, self.config.thresholdValue))
fpSets.background = None
if self.config.reEstimateBackground:
mi = exposure.getMaskedImage()
bkgd = getBackground(mi, self.config.background)
if self.config.adjustBackground:
self.log.log(
self.log.WARN, "Fiddling the background by %g" %
self.config.adjustBackground)
bkgd += self.config.adjustBackground
fpSets.background = bkgd
self.log.log(
self.log.INFO,
"Resubtracting the background after object detection")
mi -= bkgd.getImageF()
del mi
if self.config.thresholdPolarity == "positive":
if self.config.reEstimateBackground:
mask = maskedImage.getMask()
mask &= ~mask.getPlaneBitMask("DETECTED_NEGATIVE")
del mask
fpSets.negative = None
else:
self.log.log(
self.log.INFO, "Detected %d negative sources to %g %s" %
(fpSets.numNeg, self.config.thresholdValue,
("DN" if self.config.thresholdType == "value" else "sigma")))
if display:
ds9.mtv(exposure, frame=0, title="detection")
if convolvedImage and display and display > 1:
ds9.mtv(convolvedImage, frame=1, title="PSF smoothed")
if middle and display and display > 1:
ds9.mtv(middle, frame=2, title="middle")
return fpSets
def matchObjectsToSources(self, refCat, sourceCat, wcs, sourceFluxField, refFluxField,
match_tolerance=None):
"""Match sources to position reference stars.
Parameters
----------
refCat : `lsst.afw.table.SimpleCatalog`
Reference catalog to match.
sourceCat : `lsst.afw.table.SourceCatalog`
Catalog of sources found on an exposure. This should already be
down-selected to "good"/"usable" sources in the calling Task.
wcs : `lsst.afw.geom.SkyWcs`
Current WCS of the exposure containing the sources.
sourceFluxField : `str`
Field of the sourceCat to use for flux
refFluxField : `str`
Field of the refCat to use for flux
match_tolerance : `lsst.meas.astrom.MatchTolerance`
Object containing information from previous
`lsst.meas.astrom.AstrometryTask` match/fit cycles for use in
matching. If `None` is config defaults.
Returns
-------
matchResult : `lsst.pipe.base.Struct`
Result struct with components
- ``matches`` : List of matches with distance below the maximum match
distance (`list` of `lsst.afw.table.ReferenceMatch`).
- ``useableSourceCat`` : Catalog of sources matched and suited for
WCS fitting (`lsst.afw.table.SourceCatalog`).
- ``match_tolerance`` : MatchTolerance object updated from this
match iteration (`lsst.meas.astrom.MatchTolerance`).
"""
import lsstDebug
debug = lsstDebug.Info(__name__)
preNumObj = len(refCat)
refCat = self.filterStars(refCat)
numRefObj = len(refCat)
if self.log:
self.log.info("filterStars purged %d reference stars, leaving %d stars" %
(preNumObj - numRefObj, numRefObj))
if match_tolerance is None:
match_tolerance = MatchTolerance()
# Make a name alias here for consistency with older code, and to make
# it clear that this is a good/usable (cleaned) source catalog.
usableSourceCat = sourceCat
numUsableSources = len(usableSourceCat)
if len(usableSourceCat) == 0:
raise pipeBase.TaskError("No sources are usable")
minMatchedPairs = min(self.config.minMatchedPairs,
int(self.config.minFracMatchedPairs * min([len(refCat), len(usableSourceCat)])))
# match usable (possibly saturated) sources and then purge saturated sources from the match list
usableMatches = self._doMatch(
refCat=refCat,
sourceCat=usableSourceCat,
wcs=wcs,
refFluxField=refFluxField,
numUsableSources=numUsableSources,
minMatchedPairs=minMatchedPairs,
maxMatchDist=match_tolerance.maxMatchDist,
sourceFluxField=sourceFluxField,
verbose=debug.verbose,
)
# cull non-good sources
matches = []
self._getIsGoodKeys(usableSourceCat.schema)
for match in usableMatches:
if self._isGoodTest(match.second):
# Append the isGood match.
matches.append(match)
self.log.debug("Found %d usable matches, of which %d had good sources",
len(usableMatches), len(matches))
if len(matches) == 0:
raise RuntimeError("Unable to match sources")
self.log.info("Matched %d sources" % len(matches))
if len(matches) < minMatchedPairs:
self.log.warn("Number of matches is smaller than request")
return pipeBase.Struct(
matches=matches,
usableSourceCat=usableSourceCat,
match_tolerance=match_tolerance,
)
def matchObjectsToSources(self,
refCat,
sourceCat,
wcs,
refFluxField,
match_tolerance=None):
"""!Match sources to position reference stars
@param[in] refCat catalog of reference objects that overlap the
exposure; reads fields for:
- coord
- the specified flux field
@param[in] sourceCat catalog of sources found on an exposure;
Please check the required fields of your specified source selector
that the correct flags are present.
@param[in] wcs estimated WCS
@param[in] refFluxField field of refCat to use for flux
@param[in] match_tolerance is a MatchTolerance class object or None.
This this class is used to comunicate state between AstrometryTask
and MatcherTask. AstrometryTask will also set the MatchTolerance
class variable maxMatchDist based on the scatter AstrometryTask has
found after fitting for the wcs.
@return an lsst.pipe.base.Struct with fields:
- matches a list of matches, each instance of
lsst.afw.table.ReferenceMatch
- usableSourcCat a catalog of sources potentially usable for
matching.
- match_tolerance a MatchTolerance object containing the resulting
state variables from the match.
"""
import lsstDebug
debug = lsstDebug.Info(__name__)
# If we get an empty tolerance struct create the variables we need for
# this matcher.
if match_tolerance is None:
match_tolerance = MatchTolerancePessimistic()
# usableSourceCat: sources that are good but may be saturated
numSources = len(sourceCat)
selectedSources = self.sourceSelector.selectSources(sourceCat)
goodSourceCat = selectedSources.sourceCat
numUsableSources = len(goodSourceCat)
self.log.info("Purged %d sources, leaving %d good sources" %
(numSources - numUsableSources, numUsableSources))
if len(goodSourceCat) == 0:
raise pipeBase.TaskError("No sources are good")
# avoid accidentally using sourceCat; use goodSourceCat from now on
del sourceCat
minMatchedPairs = min(
self.config.minMatchedPairs,
int(self.config.minFracMatchedPairs *
min([len(refCat), len(goodSourceCat)])))
doMatchReturn = self._doMatch(
refCat=refCat,
sourceCat=goodSourceCat,
wcs=wcs,
refFluxField=refFluxField,
numUsableSources=numUsableSources,
minMatchedPairs=minMatchedPairs,
match_tolerance=match_tolerance,
sourceFluxField=self.sourceSelector.fluxField,
verbose=debug.verbose,
)
matches = doMatchReturn.matches
match_tolerance = doMatchReturn.match_tolerance
if len(matches) == 0:
raise RuntimeError("Unable to match sources")
self.log.info("Matched %d sources" % len(matches))
if len(matches) < minMatchedPairs:
self.log.warn("Number of matches is smaller than request")
return pipeBase.Struct(
matches=matches,
usableSourceCat=goodSourceCat,
match_tolerance=match_tolerance,
)
def solve(self, exposure, sourceCat):
"""Load reference objects overlapping an exposure, match to sources and
fit a WCS
Returns
-------
result : `lsst.pipe.base.Struct`
Result struct with components:
- ``refCat`` : reference object catalog of objects that overlap the
exposure (with some margin) (`lsst::afw::table::SimpleCatalog`).
- ``matches`` : astrometric matches
(`list` of `lsst.afw.table.ReferenceMatch`).
- ``scatterOnSky`` : median on-sky separation between reference
objects and sources in "matches" (`lsst.geom.Angle`)
- ``matchMeta`` : metadata needed to unpersist matches
(`lsst.daf.base.PropertyList`)
Raises
------
TaskError
If the measured mean on-sky distance between the matched source and
reference objects is greater than
``self.config.maxMeanDistanceArcsec``.
Notes
-----
ignores config.forceKnownWcs
"""
if self.refObjLoader is None:
raise RuntimeError(
"Running matcher task with no refObjLoader set in __init__ or setRefObjLoader"
)
import lsstDebug
debug = lsstDebug.Info(__name__)
expMd = self._getExposureMetadata(exposure)
sourceSelection = self.sourceSelector.run(sourceCat)
self.log.info("Purged %d sources, leaving %d good sources",
len(sourceCat) - len(sourceSelection.sourceCat),
len(sourceSelection.sourceCat))
loadRes = self.refObjLoader.loadPixelBox(
bbox=expMd.bbox,
wcs=expMd.wcs,
filterName=expMd.filterName,
epoch=expMd.epoch,
)
refSelection = self.referenceSelector.run(loadRes.refCat)
matchMeta = self.refObjLoader.getMetadataBox(
bbox=expMd.bbox,
wcs=expMd.wcs,
filterName=expMd.filterName,
epoch=expMd.epoch,
)
if debug.display:
frame = int(debug.frame)
displayAstrometry(
refCat=refSelection.sourceCat,
sourceCat=sourceSelection.sourceCat,
exposure=exposure,
bbox=expMd.bbox,
frame=frame,
title="Reference catalog",
)
res = None
wcs = expMd.wcs
match_tolerance = None
for i in range(self.config.maxIter):
iterNum = i + 1
try:
tryRes = self._matchAndFitWcs(
refCat=refSelection.sourceCat,
sourceCat=sourceCat,
goodSourceCat=sourceSelection.sourceCat,
refFluxField=loadRes.fluxField,
bbox=expMd.bbox,
wcs=wcs,
exposure=exposure,
match_tolerance=match_tolerance,
)
except Exception as e:
# if we have had a succeessful iteration then use that; otherwise fail
if i > 0:
self.log.info(
"Fit WCS iter %d failed; using previous iteration: %s",
iterNum, e)
iterNum -= 1
break
else:
raise
match_tolerance = tryRes.match_tolerance
tryMatchDist = self._computeMatchStatsOnSky(tryRes.matches)
self.log.debug(
"Match and fit WCS iteration %d: found %d matches with on-sky distance mean "
"= %0.3f +- %0.3f arcsec; max match distance = %0.3f arcsec",
iterNum, len(tryRes.matches),
tryMatchDist.distMean.asArcseconds(),
tryMatchDist.distStdDev.asArcseconds(),
tryMatchDist.maxMatchDist.asArcseconds())
maxMatchDist = tryMatchDist.maxMatchDist
res = tryRes
wcs = res.wcs
if maxMatchDist.asArcseconds(
) < self.config.minMatchDistanceArcSec:
self.log.debug(
"Max match distance = %0.3f arcsec < %0.3f = config.minMatchDistanceArcSec; "
"that's good enough", maxMatchDist.asArcseconds(),
self.config.minMatchDistanceArcSec)
break
match_tolerance.maxMatchDist = maxMatchDist
self.log.info(
"Matched and fit WCS in %d iterations; "
"found %d matches with on-sky distance mean and scatter = %0.3f +- %0.3f arcsec",
iterNum, len(tryRes.matches), tryMatchDist.distMean.asArcseconds(),
tryMatchDist.distStdDev.asArcseconds())
if tryMatchDist.distMean.asArcseconds(
) > self.config.maxMeanDistanceArcsec:
raise pipeBase.TaskError(
"Fatal astrometry failure detected: mean on-sky distance = %0.3f arcsec > %0.3f "
"(maxMeanDistanceArcsec)" %
(tryMatchDist.distMean.asArcseconds(),
self.config.maxMeanDistanceArcsec))
for m in res.matches:
if self.usedKey:
m.second.set(self.usedKey, True)
exposure.setWcs(res.wcs)
# Record the scatter in the exposure metadata
md = exposure.getMetadata()
md['SFM_ASTROM_OFFSET_MEAN'] = tryMatchDist.distMean.asArcseconds()
md['SFM_ASTROM_OFFSET_STD'] = tryMatchDist.distStdDev.asArcseconds()
return pipeBase.Struct(
refCat=refSelection.sourceCat,
matches=res.matches,
scatterOnSky=res.scatterOnSky,
matchMeta=matchMeta,
)
def fitBackground(self, maskedImage, nx=0, ny=0, algorithm=None):
"""!Estimate the background of a masked image
@param[in] maskedImage masked image whose background is to be computed
@param[in] nx number of x bands; if 0 compute from width and config.binSizeX
@param[in] ny number of y bands; if 0 compute from height and config.binSizeY
@param[in] algorithm name of interpolation algorithm; if None use self.config.algorithm
@return fit background as an lsst.afw.math.Background
@throw RuntimeError if lsst.afw.math.makeBackground returns None,
which is apparently one way it indicates failure
"""
binSizeX = self.config.binSize if self.config.binSizeX == 0 else self.config.binSizeX
binSizeY = self.config.binSize if self.config.binSizeY == 0 else self.config.binSizeY
if not nx:
nx = maskedImage.getWidth() // binSizeX + 1
if not ny:
ny = maskedImage.getHeight() // binSizeY + 1
unsubFrame = getDebugFrame(self._display, "unsubtracted")
if unsubFrame:
unsubDisp = afwDisplay.getDisplay(frame=unsubFrame)
unsubDisp.mtv(maskedImage, title="unsubtracted")
xPosts = numpy.rint(
numpy.linspace(0,
maskedImage.getWidth() + 1,
num=nx,
endpoint=True))
yPosts = numpy.rint(
numpy.linspace(0,
maskedImage.getHeight() + 1,
num=ny,
endpoint=True))
with unsubDisp.Buffering():
for (xMin, xMax), (yMin, yMax) in itertools.product(
zip(xPosts[:-1], xPosts[1:]),
zip(yPosts[:-1], yPosts[1:])):
unsubDisp.line([(xMin, yMin), (xMin, yMax), (xMax, yMax),
(xMax, yMin), (xMin, yMin)])
sctrl = afwMath.StatisticsControl()
badMask = maskedImage.mask.getPlaneBitMask(
self.config.ignoredPixelMask)
sctrl.setAndMask(badMask)
sctrl.setNanSafe(self.config.isNanSafe)
self.log.debug("Ignoring mask planes: %s" %
", ".join(self.config.ignoredPixelMask))
if (maskedImage.mask.getArray() & badMask).all():
raise pipeBase.TaskError(
"All pixels masked. Cannot estimate background")
if algorithm is None:
algorithm = self.config.algorithm
# TODO: DM-22814. This call to a deprecated BackgroundControl constructor
# is necessary to support the algorithm parameter; it # should be replaced with
#
# afwMath.BackgroundControl(nx, ny, sctrl, self.config.statisticsProperty)
#
# when algorithm has been deprecated and removed.
with suppress_deprecations():
bctrl = afwMath.BackgroundControl(algorithm, nx, ny,
self.config.undersampleStyle,
sctrl,
self.config.statisticsProperty)
# TODO: The following check should really be done within lsst.afw.math.
# With the current code structure, it would need to be accounted for in the doGetImage()
# function in BackgroundMI.cc (which currently only checks against the interpolation settings,
# which is not appropriate when useApprox=True)
# and/or the makeApproximate() function in afw/Approximate.cc.
# See ticket DM-2920: "Clean up code in afw for Approximate background
# estimation" (which includes a note to remove the following and the
# similar checks in pipe_tasks/matchBackgrounds.py once implemented)
#
# Check that config setting of approxOrder/binSize make sense
# (i.e. ngrid (= shortDimension/binSize) > approxOrderX) and perform
# appropriate undersampleStlye behavior.
if self.config.useApprox:
if self.config.approxOrderY not in (self.config.approxOrderX, -1):
raise ValueError(
"Error: approxOrderY not in (approxOrderX, -1)")
order = self.config.approxOrderX
minNumberGridPoints = order + 1
if min(nx, ny) <= order:
self.log.warn(
"Too few points in grid to constrain fit: min(nx, ny) < approxOrder) "
"[min(%d, %d) < %d]" % (nx, ny, order))
if self.config.undersampleStyle == "THROW_EXCEPTION":
raise ValueError(
"Too few points in grid (%d, %d) for order (%d) and binSize (%d, %d)"
% (nx, ny, order, binSizeX, binSizeY))
elif self.config.undersampleStyle == "REDUCE_INTERP_ORDER":
if order < 1:
raise ValueError(
"Cannot reduce approxOrder below 0. "
"Try using undersampleStyle = \"INCREASE_NXNYSAMPLE\" instead?"
)
order = min(nx, ny) - 1
self.log.warn("Reducing approxOrder to %d" % order)
elif self.config.undersampleStyle == "INCREASE_NXNYSAMPLE":
# Reduce bin size to the largest acceptable square bins
newBinSize = min(
maskedImage.getWidth(),
maskedImage.getHeight()) // (minNumberGridPoints - 1)
if newBinSize < 1:
raise ValueError("Binsize must be greater than 0")
newNx = maskedImage.getWidth() // newBinSize + 1
newNy = maskedImage.getHeight() // newBinSize + 1
bctrl.setNxSample(newNx)
bctrl.setNySample(newNy)
self.log.warn(
"Decreasing binSize from (%d, %d) to %d for a grid of (%d, %d)"
% (binSizeX, binSizeY, newBinSize, newNx, newNy))
actrl = afwMath.ApproximateControl(
afwMath.ApproximateControl.CHEBYSHEV, order, order,
self.config.weighting)
bctrl.setApproximateControl(actrl)
bg = afwMath.makeBackground(maskedImage, bctrl)
if bg is None:
raise RuntimeError(
"lsst.afw.math.makeBackground failed to fit a background model"
)
return bg
def matchObjectsToSources(self, refCat, sourceCat, wcs, refFluxField,
match_tolerance=None):
"""!Match sources to position reference stars
@param[in] refCat catalog of reference objects that overlap the exposure; reads fields for:
- coord
- the specified flux field
@param[in] sourceCat catalog of sources found on an exposure; reads fields for:
- centroid
- centroid flag
- edge flag
- saturated flag
- aperture flux, if found, else PSF flux
@param[in] wcs estimated WCS
@param[in] refFluxField field of refCat to use for flux
@param[in] match_tolerance a MatchTolerance object for specifying
tolerances. Must at minimum contain a lsst.afw.geom.Angle
called maxMatchDist that communicates state between AstrometryTask
and the matcher Task.
@return an lsst.pipe.base.Struct with fields:
- matches a list of matches, each instance of lsst.afw.table.ReferenceMatch
- usableSourcCat a catalog of sources potentially usable for matching.
For this fitter usable sources include unresolved sources not too near the edge.
It includes saturated sources, even those these are removed from the final match list,
because saturated sources may be used to determine the match list.
"""
import lsstDebug
debug = lsstDebug.Info(__name__)
preNumObj = len(refCat)
refCat = self.filterStars(refCat)
numRefObj = len(refCat)
if self.log:
self.log.info("filterStars purged %d reference stars, leaving %d stars" %
(preNumObj - numRefObj, numRefObj))
if match_tolerance is None:
match_tolerance = MatchTolerance()
# usableSourceCat: sources that are good but may be saturated
numSources = len(sourceCat)
selectedSources = self.sourceSelector.run(sourceCat)
usableSourceCat = selectedSources.sourceCat
numUsableSources = len(usableSourceCat)
self.log.info("Purged %d unusable sources, leaving %d usable sources" %
(numSources - numUsableSources, numUsableSources))
if len(usableSourceCat) == 0:
raise pipeBase.TaskError("No sources are usable")
del sourceCat # avoid accidentally using sourceCat; use usableSourceCat or goodSourceCat from now on
minMatchedPairs = min(self.config.minMatchedPairs,
int(self.config.minFracMatchedPairs * min([len(refCat), len(usableSourceCat)])))
# match usable (possibly saturated) sources and then purge saturated sources from the match list
usableMatches = self._doMatch(
refCat=refCat,
sourceCat=usableSourceCat,
wcs=wcs,
refFluxField=refFluxField,
numUsableSources=numUsableSources,
minMatchedPairs=minMatchedPairs,
maxMatchDist=match_tolerance.maxMatchDist,
sourceFluxField=self.sourceSelector.fluxField,
verbose=debug.verbose,
)
# cull non-good sources
matches = []
self._getIsGoodKeys(usableSourceCat.schema)
for match in usableMatches:
if self._isGoodTest(match.second):
# Append the isGood match.
matches.append(match)
self.log.debug("Found %d usable matches, of which %d had good sources",
len(usableMatches), len(matches))
if len(matches) == 0:
raise RuntimeError("Unable to match sources")
self.log.info("Matched %d sources" % len(matches))
if len(matches) < minMatchedPairs:
self.log.warn("Number of matches is smaller than request")
return pipeBase.Struct(
matches=matches,
usableSourceCat=usableSourceCat,
match_tolerance=match_tolerance,
)
def fitWcs(self,
matches,
initWcs,
bbox=None,
refCat=None,
sourceCat=None,
exposure=None):
"""!Fit a TAN-SIP WCS from a list of reference object/source matches
@param[in,out] matches a list of lsst::afw::table::ReferenceMatch
The following fields are read:
- match.first (reference object) coord
- match.second (source) centroid
The following fields are written:
- match.first (reference object) centroid,
- match.second (source) centroid
- match.distance (on sky separation, in radians)
@param[in] initWcs initial WCS
@param[in] bbox the region over which the WCS will be valid (an lsst:afw::geom::Box2I);
if None or an empty box then computed from matches
@param[in,out] refCat reference object catalog, or None.
If provided then all centroids are updated with the new WCS,
otherwise only the centroids for ref objects in matches are updated.
Required fields are "centroid_x", "centroid_y", "coord_ra", and "coord_dec".
@param[in,out] sourceCat source catalog, or None.
If provided then coords are updated with the new WCS;
otherwise only the coords for sources in matches are updated.
Required fields are "slot_Centroid_x", "slot_Centroid_y", and "coord_ra", and "coord_dec".
@param[in] exposure Ignored; present for consistency with FitSipDistortionTask.
@return an lsst.pipe.base.Struct with the following fields:
- wcs the fit WCS as an lsst.afw.geom.Wcs
- scatterOnSky median on-sky separation between reference objects and sources in "matches",
as an lsst.afw.geom.Angle
"""
if bbox is None:
bbox = afwGeom.Box2I()
import lsstDebug
debug = lsstDebug.Info(__name__)
wcs = self.initialWcs(matches, initWcs)
rejected = np.zeros(len(matches), dtype=bool)
for rej in range(self.config.numRejIter):
sipObject = self._fitWcs(
[mm for i, mm in enumerate(matches) if not rejected[i]], wcs)
wcs = sipObject.getNewWcs()
rejected = self.rejectMatches(matches, wcs, rejected)
if rejected.sum() == len(rejected):
raise RuntimeError("All matches rejected in iteration %d" %
(rej + 1, ))
self.log.debug(
"Iteration {0} of astrometry fitting: rejected {1} outliers, "
"out of {2} total matches.".format(rej, rejected.sum(),
len(rejected)))
if debug.plot:
print("Plotting fit after rejection iteration %d/%d" %
(rej + 1, self.config.numRejIter))
self.plotFit(matches, wcs, rejected)
# Final fit after rejection
sipObject = self._fitWcs(
[mm for i, mm in enumerate(matches) if not rejected[i]], wcs)
wcs = sipObject.getNewWcs()
if debug.plot:
print("Plotting final fit")
self.plotFit(matches, wcs, rejected)
if refCat is not None:
self.log.debug("Updating centroids in refCat")
afwTable.updateRefCentroids(wcs, refList=refCat)
else:
self.log.warn(
"Updating reference object centroids in match list; refCat is None"
)
afwTable.updateRefCentroids(
wcs, refList=[match.first for match in matches])
if sourceCat is not None:
self.log.debug("Updating coords in sourceCat")
afwTable.updateSourceCoords(wcs, sourceList=sourceCat)
else:
self.log.warn(
"Updating source coords in match list; sourceCat is None")
afwTable.updateSourceCoords(
wcs, sourceList=[match.second for match in matches])
self.log.debug("Updating distance in match list")
setMatchDistance(matches)
scatterOnSky = sipObject.getScatterOnSky()
if scatterOnSky.asArcseconds() > self.config.maxScatterArcsec:
raise pipeBase.TaskError(
"Fit failed: median scatter on sky = %0.3f arcsec > %0.3f config.maxScatterArcsec"
% (scatterOnSky.asArcseconds(), self.config.maxScatterArcsec))
return pipeBase.Struct(
wcs=wcs,
scatterOnSky=scatterOnSky,
)
def fetchInPatches(self, butler, exposure, tract, patchList, band):
"""!
Get the reference catalogs from a given tract,patchlist
This will remove objects where the child is inside the catalog boundary, but
the parent is outside the boundary.
@param[in] butler A Butler used to get the reference catalogs
@param[in] exposure A deepDiff_exposure on which to run the measurements
@param[in] tract The tract
@param[in] patchList A list of patches that need to be checked
.
@return Combined SourceCatalog from all the patches
"""
dataset = f"{self.config.coaddName}Coadd_meas"
catalog = None
for patch in patchList:
dataId = {
'tract': tract.getId(),
'patch': "%d,%d" % patch.getIndex()
}
dataId['filter'] = band
self.log.info("Getting references in %s" % (dataId, ))
if not butler.datasetExists(dataset, dataId):
if self.config.skipMissing:
self.log.info("Could not find %s for dataset %s" %
(dataId, dataset))
continue
raise pipeBase.TaskError("Reference %s doesn't exist" %
(dataId, ))
new_catalog = butler.get(dataset, dataId, immediate=True)
patchBox = geom.Box2D(patch.getOuterBBox())
tractWcs = tract.getWcs()
expBox = geom.Box2D(exposure.getBBox())
expWcs = exposure.getWcs()
# only use objects that overlap patch bounding box and overlap exposure
validPatch = np.array([
patchBox.contains(tractWcs.skyToPixel(s.getCoord()))
for s in new_catalog
])
validExposure = np.array([
expBox.contains(expWcs.skyToPixel(s.getCoord()))
for s in new_catalog
])
if validPatch.size == 0 or validExposure.size == 0:
self.log.debug("No valid sources %s for dataset %s" %
(dataId, dataset))
continue
if catalog is None:
catalog = new_catalog[validPatch & validExposure]
else:
catalog.extend(new_catalog[validPatch & validExposure])
if catalog is None:
return None
# if the parent is not in the catalog remove
refCatIdDict = {ref.getId(): ref.getParent() for ref in catalog}
refCatIdDict[0] = 0
parentGood = np.array(
[refCatIdDict[ref.getId()] in refCatIdDict for ref in catalog])
if np.sum(parentGood == False) > 1:
self.log.info("Removing %d/%d objects without parents" %
(np.sum(parentGood == False), len(parentGood)))
catalog = catalog.copy(deep=True)[parentGood]
return catalog
