def run(self, exposure, defects=None, keepCRs=None):
"""!Repair an Exposure's defects and cosmic rays
\param[in, out] exposure lsst.afw.image.Exposure to process. Exposure must have a valid Psf.
Modified in place.
\param[in] defects an lsst.meas.algorithms.DefectListT object. If None, do no
defect correction.
\param[in] keepCRs don't interpolate over the CR pixels (defer to RepairConfig if None)
\throws AssertionError with the following strings:
<DL>
<DT> No exposure provided
<DD> The object provided as exposure evaluates to False
<DT> No PSF provided
<DD> The Exposure has no associated Psf
</DL>
"""
assert exposure, "No exposure provided"
psf = exposure.getPsf()
assert psf, "No PSF provided"
frame = getDebugFrame(self._display, "repair.before")
if frame:
getDisplay(frame).mtv(exposure)
if defects is not None and self.config.doInterpolate:
self.interp.run(exposure, defects=defects)
if self.config.doCosmicRay:
self.cosmicRay(exposure, keepCRs=keepCRs)
frame = getDebugFrame(self._display, "repair.after")
if frame:
getDisplay(frame).mtv(exposure)
def postprocessExposure(self, outExposure, inExposure):
"""Set exposure non-image attributes, including wcs and metadata and display exposure (if requested)
Call after assembling the pixels
@param[in,out] outExposure assembled exposure:
- removes unwanted keywords
- sets calib, filter, and detector
@param[in] inExposure input exposure
"""
self.setWcs(outExposure = outExposure, inExposure = inExposure)
exposureMetadata = inExposure.getMetadata()
for key in self.allKeysToRemove:
if exposureMetadata.exists(key):
exposureMetadata.remove(key)
outExposure.setMetadata(exposureMetadata)
if self.config.setGain:
self.setGain(outExposure = outExposure)
inCalib = inExposure.getCalib()
outCalib = outExposure.getCalib()
outCalib.setExptime(inCalib.getExptime())
outCalib.setMidTime(inCalib.getMidTime())
outExposure.setFilter(inExposure.getFilter())
frame = getDebugFrame(self._display, "assembledExposure")
if frame:
getDisplay(frame).mtv(outExposure)
def postprocessExposure(self, outExposure, inExposure):
"""Set exposure non-image attributes, including wcs and metadata and display exposure (if requested)
Call after assembling the pixels
@param[in,out] outExposure assembled exposure:
- removes unwanted keywords
- sets calib, filter, and detector
@param[in] inExposure input exposure
"""
self.setWcs(outExposure=outExposure, inExposure=inExposure)
exposureMetadata = inExposure.getMetadata()
for key in self.allKeysToRemove:
if exposureMetadata.exists(key):
exposureMetadata.remove(key)
outExposure.setMetadata(exposureMetadata)
# note: Calib is not copied, presumably because it is assumed unknown in raw data
outExposure.setFilter(inExposure.getFilter())
outExposure.getInfo().setVisitInfo(inExposure.getInfo().getVisitInfo())
frame = getDebugFrame(self._display, "assembledExposure")
if frame:
getDisplay(frame).mtv(outExposure)
def subtractXTalk(mi, coeffs, minPixelToMask=45000, crosstalkStr="CROSSTALK"):
"""Subtract the crosstalk from MaskedImage mi given a set of coefficients
The pixels affected by signal over minPixelToMask have the crosstalkStr bit set
"""
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(mi.getMask().getPlaneBitMask("BAD"))
bkgd = afwMath.makeStatistics(mi, afwMath.MEDIAN, sctrl).getValue()
#
# These are the pixels that are bright enough to cause crosstalk (more precisely,
# the ones that we label as causing crosstalk; in reality all pixels cause crosstalk)
#
tempStr = "TEMP" # mask plane used to record the bright pixels that we need to mask
msk = mi.getMask()
msk.addMaskPlane(tempStr)
try:
fs = afwDetect.FootprintSet(mi, afwDetect.Threshold(minPixelToMask), tempStr)
mi.getMask().addMaskPlane(crosstalkStr)
afwDisplay.getDisplay().setMaskPlaneColor(crosstalkStr, afwDisplay.MAGENTA)
# the crosstalkStr bit will now be set whenever we subtract crosstalk
fs.setMask(mi.getMask(), crosstalkStr)
crosstalk = mi.getMask().getPlaneBitMask(crosstalkStr)
width, height = mi.getDimensions()
for i in range(nAmp):
bbox = afwGeom.BoxI(afwGeom.PointI(i*(width//nAmp), 0), afwGeom.ExtentI(width//nAmp, height))
ampI = mi.Factory(mi, bbox)
for j in range(nAmp):
if i == j:
continue
bbox = afwGeom.BoxI(afwGeom.PointI(j*(width//nAmp), 0), afwGeom.ExtentI(width//nAmp, height))
if (i + j)%2 == 1:
ampJ = afwMath.flipImage(mi.Factory(mi, bbox), True, False) # no need for a deep copy
else:
ampJ = mi.Factory(mi, bbox, afwImage.LOCAL, True)
msk = ampJ.getMask()
if np.all(msk.getArray() & msk.getPlaneBitMask("BAD")):
# Bad amplifier; ignore it completely --- its effect will come out in the bias
continue
msk &= crosstalk
ampJ -= bkgd
ampJ *= coeffs[j][i]
ampI -= ampJ
#
# Clear the crosstalkStr bit in the original bright pixels, where tempStr is set
#
msk = mi.getMask()
temp = msk.getPlaneBitMask(tempStr)
xtalk_temp = crosstalk | temp
np_msk = msk.getArray()
mask_indicies = np.where(np.bitwise_and(np_msk, xtalk_temp) == xtalk_temp)
np_msk[mask_indicies] &= getattr(np, np_msk.dtype.name)(~crosstalk)
finally:
msk.removeAndClearMaskPlane(tempStr, True) # added in afw #1853
def run(self, exposure, defects=None, keepCRs=None):
"""!Repair an Exposure's defects and cosmic rays
\param[in, out] exposure lsst.afw.image.Exposure to process. Exposure must have a valid Psf.
Modified in place.
\param[in] defects an lsst.meas.algorithms.DefectListT object. If None, do no
defect correction.
\param[in] keepCRs don't interpolate over the CR pixels (defer to RepairConfig if None)
\throws AssertionError with the following strings:
<DL>
<DT> No exposure provided
<DD> The object provided as exposure evaluates to False
<DT> No PSF provided
<DD> The Exposure has no associated Psf
</DL>
"""
assert exposure, "No exposure provided"
psf = exposure.getPsf()
assert psf, "No PSF provided"
frame = getDebugFrame(self._display, "repair.before")
if frame:
getDisplay(frame).mtv(exposure)
if defects is not None and self.config.doInterpolate:
self.interp.run(exposure, defects=defects)
if self.config.doCosmicRay:
self.cosmicRay(exposure, keepCRs=keepCRs)
frame = getDebugFrame(self._display, "repair.after")
if frame:
getDisplay(frame).mtv(exposure)
def postprocessExposure(self, outExposure, inExposure):
"""Set exposure non-image attributes, including wcs and metadata and display exposure (if requested)
Call after assembling the pixels
@param[in,out] outExposure assembled exposure:
- removes unwanted keywords
- sets calib, filter, and detector
@param[in] inExposure input exposure
"""
self.setWcs(outExposure = outExposure, inExposure = inExposure)
exposureMetadata = inExposure.getMetadata()
for key in self.allKeysToRemove:
if exposureMetadata.exists(key):
exposureMetadata.remove(key)
outExposure.setMetadata(exposureMetadata)
if self.config.setGain:
self.setGain(outExposure = outExposure)
inCalib = inExposure.getCalib()
outCalib = outExposure.getCalib()
outCalib.setExptime(inCalib.getExptime())
outCalib.setMidTime(inCalib.getMidTime())
outExposure.setFilter(inExposure.getFilter())
frame = getDebugFrame(self._display, "assembledExposure")
if frame:
getDisplay(frame).mtv(outExposure)
def subtractXTalk(mi, coeffs, minPixelToMask=45000, crosstalkStr="CROSSTALK"):
"""Subtract the crosstalk from MaskedImage mi given a set of coefficients
The pixels affected by signal over minPixelToMask have the crosstalkStr bit set
"""
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(mi.getMask().getPlaneBitMask("DETECTED"))
bkgd = afwMath.makeStatistics(mi, afwMath.MEDIAN, sctrl).getValue()
#
# These are the pixels that are bright enough to cause crosstalk (more precisely,
# the ones that we label as causing crosstalk; in reality all pixels cause crosstalk)
#
tempStr = "TEMP" # mask plane used to record the bright pixels that we need to mask
mi.getMask().addMaskPlane(tempStr)
try:
fs = afwDetect.FootprintSet(mi, afwDetect.Threshold(minPixelToMask), tempStr)
mi.getMask().addMaskPlane(crosstalkStr)
afwDisplay.getDisplay().setMaskPlaneColor(crosstalkStr, afwDisplay.MAGENTA)
fs.setMask(mi.getMask(), crosstalkStr) # the crosstalkStr bit will now be set whenever
# we subtract crosstalk
crosstalk = mi.getMask().getPlaneBitMask(crosstalkStr)
width, height = mi.getDimensions()
for i in range(nAmp):
bbox = afwGeom.BoxI(afwGeom.PointI(i*(width//nAmp), 0), afwGeom.ExtentI(width//nAmp, height))
ampI = mi.Factory(mi, bbox)
for j in range(nAmp):
if i == j:
continue
bbox = afwGeom.BoxI(afwGeom.PointI(j*(width//nAmp), 0), afwGeom.ExtentI(width//nAmp, height))
if (i + j)%2 == 1:
ampJ = afwMath.flipImage(mi.Factory(mi, bbox), True, False) # no need for a deep copy
else:
ampJ = mi.Factory(mi, bbox, afwImage.LOCAL, True)
msk = ampJ.getMask()
if np.all(msk.getArray() & msk.getPlaneBitMask("SAT")):
# Bad amplifier; ignore it completely --- its effect will come out in the bias
continue
msk &= crosstalk
ampJ -= bkgd
ampJ *= coeffs[j][i]
ampI -= ampJ
#
# Clear the crosstalkStr bit in the original bright pixels, where tempStr is set
#
msk = mi.getMask()
temp = msk.getPlaneBitMask(tempStr)
xtalk_temp = crosstalk | temp
np_msk = msk.getArray()
mask_indicies = np.where(np.bitwise_and(np_msk, xtalk_temp) == xtalk_temp)
np_msk[mask_indicies] &= getattr(np, np_msk.dtype.name)(~crosstalk)
finally:
msk.removeAndClearMaskPlane(tempStr, True) # added in afw #1853
def fixCcd(butler, visit, ccd, coeffs, display=True):
"""Apply cross-talk correction to a CCD, given the cross-talk coefficients"""
mi = readImage(butler, visit=visit, ccd=ccd)
if display:
afwDisplay.getDisplay(frame=1).mtv(mi.getImage(), title="CCD %d" % ccd)
subtractXTalk(mi, coeffs)
if display:
afwDisplay.getDisplay(frame=2).mtv(mi, title="corrected %d" % ccd)
def fixCcd(butler, visit, ccd, coeffs, display=True):
"""Apply cross-talk correction to a CCD, given the cross-talk coefficients"""
mi = readImage(butler, visit=visit, ccd=ccd)
if display:
afwDisplay.getDisplay(frame=1).mtv(mi.getImage(), title="CCD %d" % ccd)
subtractXTalk(mi, coeffs)
if display:
afwDisplay.getDisplay(frame=2).mtv(mi, title="corrected %d" % ccd)
def runDataRef(self, sensorRef):
"""Do instrument signature removal on an exposure
Correct for saturation, bias, overscan, dark, flat..., perform CCD assembly,
optionally combine snaps, and interpolate over defects and saturated pixels.
If config.doSnapCombine true then combine the two ISR-corrected snaps to produce the final exposure.
If config.doSnapCombine false then uses ISR-corrected snap 0 as the final exposure.
In either case, the final exposure is persisted as "postISRCCD" if config.doWriteSpans is True,
and the two snaps are persisted as "snapExp" if config.doWriteSnaps is True.
@param sensorRef daf.persistence.butlerSubset.ButlerDataRef of the data to be processed
@return a pipeBase.Struct with fields:
- exposure: the exposure after application of ISR
"""
self.log.info("Performing ISR on sensor %s", sensorRef.dataId)
camera = sensorRef.get("camera")
snapDict = dict()
for snapRef in sensorRef.subItems(level="snap"):
snapId = snapRef.dataId['snap']
if snapId not in (0, 1):
raise RuntimeError("Unrecognized snapId=%s" % (snapId, ))
self.log.info("Performing ISR on snap %s", snapRef.dataId)
ccdExposure = snapRef.get('raw')
isrData = self.readIsrData(snapRef, ccdExposure)
ccdExposure = self.run(ccdExposure,
camera=camera,
**isrData.getDict()).exposure
snapDict[snapId] = ccdExposure
if self.config.doWriteSnaps:
sensorRef.put(ccdExposure, "snapExp", snap=snapId)
frame = getDebugFrame(self._display, "snapExp%d" % (snapId, ))
if frame:
getDisplay(frame).mtv(ccdExposure)
if self.config.doSnapCombine:
loadSnapDict(snapDict, snapIdList=(0, 1), sensorRef=sensorRef)
postIsrExposure = self.snapCombine.run(snapDict[0],
snapDict[1]).exposure
else:
self.log.warn("doSnapCombine false; using snap 0 as the result")
loadSnapDict(snapDict, snapIdList=(0, ), sensorRef=sensorRef)
postIsrExposure = snapDict[0]
if self.config.doWrite:
sensorRef.put(postIsrExposure, "postISRCCD")
frame = getDebugFrame(self._display, "postISRCCD")
if frame:
getDisplay(frame).mtv(postIsrExposure)
return pipeBase.Struct(exposure=postIsrExposure, )
def runDataRef(self, sensorRef):
"""Do instrument signature removal on an exposure
Correct for saturation, bias, overscan, dark, flat..., perform CCD assembly,
optionally combine snaps, and interpolate over defects and saturated pixels.
If config.doSnapCombine true then combine the two ISR-corrected snaps to produce the final exposure.
If config.doSnapCombine false then uses ISR-corrected snap 0 as the final exposure.
In either case, the final exposure is persisted as "postISRCCD" if config.doWriteSpans is True,
and the two snaps are persisted as "snapExp" if config.doWriteSnaps is True.
@param sensorRef daf.persistence.butlerSubset.ButlerDataRef of the data to be processed
@return a pipeBase.Struct with fields:
- exposure: the exposure after application of ISR
"""
self.log.info("Performing ISR on sensor %s", sensorRef.dataId)
camera = sensorRef.get("camera")
snapDict = dict()
for snapRef in sensorRef.subItems(level="snap"):
snapId = snapRef.dataId['snap']
if snapId not in (0, 1):
raise RuntimeError("Unrecognized snapId=%s" % (snapId,))
self.log.info("Performing ISR on snap %s", snapRef.dataId)
ccdExposure = snapRef.get('raw')
isrData = self.readIsrData(snapRef, ccdExposure)
ccdExposure = self.run(ccdExposure, camera=camera, **isrData.getDict()).exposure
snapDict[snapId] = ccdExposure
if self.config.doWriteSnaps:
sensorRef.put(ccdExposure, "snapExp", snap=snapId)
frame = getDebugFrame(self._display, "snapExp%d" % (snapId,))
if frame:
getDisplay(frame).mtv(ccdExposure)
if self.config.doSnapCombine:
loadSnapDict(snapDict, snapIdList=(0, 1), sensorRef=sensorRef)
postIsrExposure = self.snapCombine.run(snapDict[0], snapDict[1]).exposure
else:
self.log.warn("doSnapCombine false; using snap 0 as the result")
loadSnapDict(snapDict, snapIdList=(0,), sensorRef=sensorRef)
postIsrExposure = snapDict[0]
if self.config.doWrite:
sensorRef.put(postIsrExposure, "postISRCCD")
frame = getDebugFrame(self._display, "postISRCCD")
if frame:
getDisplay(frame).mtv(postIsrExposure)
return pipeBase.Struct(
exposure=postIsrExposure,
)
def run(self, exposure, background=None, stats=True, statsKeys=None):
"""Fit and subtract the background of an exposure.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
Exposure whose background is to be subtracted.
background : `lsst.afw.math.BackgroundList`
Initial background model already subtracted. May be None if no background
has been subtracted.
stats : `bool`
If True then measure the mean and variance of the full background model and
record the results in the exposure's metadata.
statsKeys : `tuple`
Key names used to store the mean and variance of the background in the
exposure's metadata (another tuple); if None then use ("BGMEAN", "BGVAR");
ignored if stats is false.
Returns
-------
background : `lsst.afw.math.BackgroundLst`
Full background model (initial model with changes), contained in an
`lsst.pipe.base.Struct`.
"""
if background is None:
background = afwMath.BackgroundList()
maskedImage = exposure.getMaskedImage()
fitBg = self.fitBackground(maskedImage)
maskedImage -= fitBg.getImageF(self.config.algorithm, self.config.undersampleStyle)
actrl = fitBg.getBackgroundControl().getApproximateControl()
background.append((fitBg, getattr(afwMath.Interpolate, self.config.algorithm),
fitBg.getAsUsedUndersampleStyle(), actrl.getStyle(),
actrl.getOrderX(), actrl.getOrderY(), actrl.getWeighting()))
if stats:
self._addStats(exposure, background, statsKeys=statsKeys)
subFrame = getDebugFrame(self._display, "subtracted")
if subFrame:
subDisp = afwDisplay.getDisplay(frame=subFrame)
subDisp.mtv(exposure, title="subtracted")
bgFrame = getDebugFrame(self._display, "background")
if bgFrame:
bgDisp = afwDisplay.getDisplay(frame=bgFrame)
bgImage = background.getImage()
bgDisp.mtv(bgImage, title="background")
return pipeBase.Struct(
background=background,
)
def show_diffim(butler, dataId):
"""
Construct a display of various difference imaging related views
Parameters
----------
butler : daf_persistence.Butler, used in interacting with the data repository
dataId : Dictionary, data identifiers to lookup specific data
"""
def display_image_and_srcs(display, image, title, *srcs):
"""
Display an image with a title and up to 4 source catalogs
Parameters
----------
display : afw_display.Display, used to display image and plot catalogs
image afw_image.Image-like, pixel data to send to the display backend
title str, title for the display frame
*srcs afw_tab;e.SourceCatalogs, points to plot
"""
if len(srcs) > 4:
print(
"WARNING: more than four source catalogs sent. Only plotting the first four."
)
syms = ['o', '+', 'x', 't']
colors = ['green', 'red', 'blue', 'white']
with display.Buffering():
display.mtv(image, title=title)
for src, plot_sym, color in zip(srcs, syms, colors):
for s in src:
display.dot(plot_sym, s.getX(), s.getY(), size=5, ctype=color)
display0 = afw_display.getDisplay(frame=0, )
display0.setMaskTransparency(75)
display0.scale('linear', 'zscale')
display1 = afw_display.getDisplay(frame=1)
display1.setMaskTransparency(75)
display1.scale('linear', 'zscale')
display2 = afw_display.getDisplay(frame=2)
display2.setMaskTransparency(75)
display2.scale('linear', 'zscale')
exp = butler.get('calexp', dataId)
src = butler.get('src', dataId)
diasrc = butler.get('deepDiff_diaSrc', dataId)
diffexp = butler.get('deepDiff_differenceExp', dataId)
display_image_and_srcs(display0, exp, 'Direct', src, diasrc)
display_image_and_srcs(display1, diffexp, 'Diffim', src, diasrc)
im1 = exp.getMaskedImage().getImage()
im2 = diffexp.getMaskedImage().getImage()
im1 -= im2
display_image_and_srcs(display2, im1, 'Direct - Diffim', src, diasrc)
def _getDisplayFromDisplayOrFrame(display, frame=None):
"""!Return an afwDisplay.Display given either a display or a frame ID.
If the two arguments are consistent, return the desired display; if they are not,
raise a RuntimeError exception.
If the desired display is None, return None;
if (display, frame) == ("deferToFrame", None), return the default display"""
import lsst.afw.display as afwDisplay # import locally to allow this file to be imported by __init__
if display in ("deferToFrame", None):
if display is None and frame is None:
return None
# "deferToFrame" is the default value, and means "obey frame"
display = None
if display and not hasattr(display, "frame"):
raise RuntimeError("display == %s doesn't support .frame" % display)
if frame and display and display.frame != frame:
raise RuntimeError("Please specify display *or* frame")
if display:
frame = display.frame
display = afwDisplay.getDisplay(frame, create=True)
return display
def _getDisplayFromDisplayOrFrame(display, frame=None):
"""Return an `lsst.afw.display.Display` given either a display or a frame ID.
Notes
-----
If the two arguments are consistent, return the desired display; if they are not,
raise a `RuntimeError` exception.
If the desired display is `None`, return `None`;
if ``(display, frame) == ("deferToFrame", None)``, return the default display
"""
# import locally to allow this file to be imported by __init__
import lsst.afw.display as afwDisplay
if display in ("deferToFrame", None):
if display is None and frame is None:
return None
# "deferToFrame" is the default value, and means "obey frame"
display = None
if display and not hasattr(display, "frame"):
raise RuntimeError(f"display == {display} doesn't support .frame")
if frame and display and display.frame != frame:
raise RuntimeError("Please specify display *or* frame")
if display:
frame = display.frame
display = afwDisplay.getDisplay(frame, create=True)
return display
def show_cat(butler, lc, ref_table, target_idx, field, tract=None):
if tract is None:
tract = get_tract_for_field(field)
dId = {'field': field, 'filter': 'H', 'tract': tract, 'patch': '0,0'}
calexp = butler.get('deepCoadd', dataId=dId)
display = afwDisplay.getDisplay(backend='ds9')
display.mtv(calexp)
display.setMaskTransparency(80)
display.scale('asinh', -2, 25)
X = 'slot_Centroid_x'
Y = 'slot_Centroid_y'
display.erase()
with display.Buffering():
for s in ref_table:
display.dot('o', s[X], s[Y], size=10, ctype='orange')
target_ref = ref_table[target_idx]
display.dot('o', target_ref[X], target_ref[Y], size=20, ctype='green')
def displayFunc(exposure, sourceCat, frame):
display = afwDisplay.getDisplay(frame)
display.mtv(exposure)
with display.Buffering():
for s in sourceCat:
xy = s.getCentroid()
display.dot('+', *xy, ctype=afwDisplay.CYAN if s.get("flags_negative") else afwDisplay.GREEN)
def show_diffim(butler, dataId):
"""
Construct a display of various difference imaging related views
Parameters
----------
butler : daf_persistence.Butler, used in interacting with the data repository
dataId : Dictionary, data identifiers to lookup specific data
"""
def display_image_and_srcs(display, image, title, *srcs):
"""
Display an image with a title and up to 4 source catalogs
Parameters
----------
display : afw_display.Display, used to display image and plot catalogs
image afw_image.Image-like, pixel data to send to the display backend
title str, title for the display frame
*srcs afw_tab;e.SourceCatalogs, points to plot
"""
if len(srcs) > 4:
print("WARNING: more than four source catalogs sent. Only plotting the first four.")
syms = ['o', '+', 'x', 't']
colors = ['green', 'red', 'blue', 'white']
with display.Buffering():
display.mtv(image, title=title)
for src, plot_sym, color in zip(srcs, syms, colors):
for s in src:
display.dot(plot_sym, s.getX(), s.getY(), size=5, ctype=color)
display0 = afw_display.getDisplay(frame=0, )
display0.setMaskTransparency(75)
display0.scale('linear', 'zscale')
display1 = afw_display.getDisplay(frame=1)
display1.setMaskTransparency(75)
display1.scale('linear', 'zscale')
display2 = afw_display.getDisplay(frame=2)
display2.setMaskTransparency(75)
display2.scale('linear', 'zscale')
exp = butler.get('calexp', dataId)
src = butler.get('src', dataId)
diasrc = butler.get('deepDiff_diaSrc', dataId)
diffexp = butler.get('deepDiff_differenceExp', dataId)
display_image_and_srcs(display0, exp, 'Direct', src, diasrc)
display_image_and_srcs(display1, diffexp, 'Diffim', src, diasrc)
im1 = exp.getMaskedImage().getImage()
im2 = diffexp.getMaskedImage().getImage()
im1 -= im2
display_image_and_srcs(display2, im1, 'Direct - Diffim', src, diasrc)
def setUp(self):
global oldBackend
if backend != oldBackend:
afwDisplay.setDefaultBackend(backend)
afwDisplay.delAllDisplays() # as some may use the old backend
oldBackend = backend
dirName = os.path.split(__file__)[0]
self.fileName = os.path.join(dirName, "data", "HSC-0908120-056-small.fits")
self.display0 = afwDisplay.getDisplay(frame=0, verbose=True)
def displayFunc(exposure, sourceCat, frame):
display = afwDisplay.getDisplay(frame)
display.mtv(exposure)
with display.Buffering():
for s in sourceCat:
xy = s.getCentroid()
display.dot('+',
*xy,
ctype=afwDisplay.CYAN
if s.get("flags_negative") else afwDisplay.GREEN)
def setUp(self):
global oldBackend
if backend != oldBackend:
afwDisplay.setDefaultBackend(backend)
afwDisplay.delAllDisplays() # as some may use the old backend
oldBackend = backend
dirName = os.path.split(__file__)[0]
self.fileName = os.path.join(dirName, "data", "HSC-0908120-056-small.fits")
self.display0 = afwDisplay.getDisplay(frame=0, verbose=True)
def plotDeblendFamily(mi, parent, kids, mapperInfo=None, dkids=[],
background=-10, symbolSize=2,
plotb=False,
arcsinh=True, maskbit=False, display=afwDisplay.getDisplay(0)):
"""Display a deblend using afwDisplay
Each child is marked with a + at its centre (green if deblended-as-psf else red)
all the other peaks in its footprint are marked with x (cyan if deblended-as-psf else magenta)
"""
if mi:
try:
mi = mi.getMaskedImage() # maybe it's an Exposure?
except AttributeError:
pass
mos = makeDeblendFamilyMosaic(mi, parent, kids, mapperInfo, background, maskbit)
if mapperInfo:
# some displays, e.g. ds9, don't handle those chars well
title = re.sub(r"[{}']", "", str(mapperInfo.getId(parent, None)))
else:
title = "0x%x == %d" % (parent.getId(), (parent.getId() & 0xffff))
mosaicImage = mos.makeMosaic(display=display, title=title)
if display is not None:
display.dot("%s (%.1f, %1.f)" % (title, parent.getX(), parent.getY()),
0.5*mosaicImage.getWidth(), 1.03*mosaicImage.getHeight(),
ctype=afwDisplay.BLACK, fontFamily="times", size=3)
px0, py0 = footprintToImage(parent.getFootprint(), mi).getXY0()
with display.Buffering():
for i, src in enumerate([parent] + kids):
x0, y0 = mos.getBBox(i).getMin()
x0 -= px0; y0 -= py0
if src.get("deblend_deblendedAsPsf"):
centroid_ctype = afwDisplay.GREEN
peak_ctype = afwDisplay.CYAN
else:
centroid_ctype = afwDisplay.RED
peak_ctype = afwDisplay.MAGENTA
display.dot("+", src.getX() + x0, src.getY() + y0,
size=symbolSize, ctype=centroid_ctype)
for p in src.getFootprint().getPeaks():
display.dot("x", p.getFx() + x0, p.getFy() + y0,
size=0.5*symbolSize if i == 0 else symbolSize,
ctype=afwDisplay.YELLOW if i == 0 else peak_ctype)
return mosaicImage
def testShowPsfMosaic(self):
""" Test that the showPsfMosaic function works.
This function is usually called without display=None, which would activate ds9
"""
testDisplay = display if display else afwDisplay.getDisplay(
backend="virtualDevice")
mos = showPsfMosaic(self.exposure,
showEllipticity=True,
showFwhm=True,
display=testDisplay)
self.assertTrue(len(mos.images) > 0)
def run(self, exposure, background=None, stats=True, statsKeys=None):
"""!Fit and subtract the background of an exposure
@param[in,out] exposure exposure whose background is to be subtracted
@param[in,out] background initial background model already subtracted from exposure
(an lsst.afw.math.BackgroundList). May be None if no background has been subtracted.
@param[in] stats if True then measure the mean and variance of the full background model
and record the results in the exposure's metadata
@param[in] statsKeys key names used to store the mean and variance of the background
in the exposure's metadata (a pair of strings); if None then use ("BGMEAN", "BGVAR");
ignored if stats is false
@return an lsst.pipe.base.Struct containing:
- background full background model (initial model with changes), an lsst.afw.math.BackgroundList
"""
if background is None:
background = afwMath.BackgroundList()
maskedImage = exposure.getMaskedImage()
fitBg = self.fitBackground(maskedImage)
maskedImage -= fitBg.getImageF()
background.append(fitBg)
if stats:
self._addStats(exposure, background, statsKeys=statsKeys)
subFrame = getDebugFrame(self._display, "subtracted")
if subFrame:
subDisp = afwDisplay.getDisplay(frame=subFrame)
subDisp.mtv(exposure, title="subtracted")
bgFrame = getDebugFrame(self._display, "background")
if bgFrame:
bgDisp = afwDisplay.getDisplay(frame=bgFrame)
bgImage = background.getImage()
bgDisp.mtv(bgImage, title="background")
return pipeBase.Struct(
background = background,
)
def run(self, exposure, background=None, stats=True, statsKeys=None):
"""!Fit and subtract the background of an exposure
@param[in,out] exposure exposure whose background is to be subtracted
@param[in,out] background initial background model already subtracted from exposure
(an lsst.afw.math.BackgroundList). May be None if no background has been subtracted.
@param[in] stats if True then measure the mean and variance of the full background model
and record the results in the exposure's metadata
@param[in] statsKeys key names used to store the mean and variance of the background
in the exposure's metadata (a pair of strings); if None then use ("BGMEAN", "BGVAR");
ignored if stats is false
@return an lsst.pipe.base.Struct containing:
- background full background model (initial model with changes), an lsst.afw.math.BackgroundList
"""
if background is None:
background = afwMath.BackgroundList()
maskedImage = exposure.getMaskedImage()
fitBg = self.fitBackground(maskedImage)
maskedImage -= fitBg.getImageF()
background.append(fitBg)
if stats:
self._addStats(exposure, background, statsKeys=statsKeys)
subFrame = getDebugFrame(self._display, "subtracted")
if subFrame:
subDisp = afwDisplay.getDisplay(frame=subFrame)
subDisp.mtv(exposure, title="subtracted")
bgFrame = getDebugFrame(self._display, "background")
if bgFrame:
bgDisp = afwDisplay.getDisplay(frame=bgFrame)
bgImage = background.getImage()
bgDisp.mtv(bgImage, title="background")
return pipeBase.Struct(
background=background,
)
def showAmp(amp,
imageSource=FakeImageDataSource(isTrimmed=False),
display=None,
overlay=True,
imageFactory=afwImage.ImageU):
"""!Show an amp in an image display
@param[in] amp amp record to use in display
@param[in] imageSource Source for getting the amp image. Must have a getAmpImage method.
@param[in] display image display to use
@param[in] overlay Overlay bounding boxes?
@param[in] imageFactory Type of image to display (only used if ampImage is None)
"""
if not display:
display = afwDisplay.getDisplay()
ampImage = imageSource.getAmpImage(amp, imageFactory=imageFactory)
ampImSize = ampImage.getDimensions()
title = amp.getName()
display.mtv(ampImage, title=title)
if overlay:
with afwDisplay.Buffering():
if amp.getHasRawInfo() and ampImSize == amp.getRawBBox(
).getDimensions():
bboxes = [
(amp.getRawBBox(), 0.49, afwDisplay.GREEN),
]
xy0 = bboxes[0][0].getMin()
bboxes.append(
(amp.getRawHorizontalOverscanBBox(), 0.49, afwDisplay.RED))
bboxes.append((amp.getRawDataBBox(), 0.49, afwDisplay.BLUE))
bboxes.append(
(amp.getRawPrescanBBox(), 0.49, afwDisplay.YELLOW))
bboxes.append((amp.getRawVerticalOverscanBBox(), 0.49,
afwDisplay.MAGENTA))
else:
bboxes = [
(amp.getBBox(), 0.49, None),
]
xy0 = bboxes[0][0].getMin()
for bbox, borderWidth, ctype in bboxes:
if bbox.isEmpty():
continue
bbox = afwGeom.Box2I(bbox)
bbox.shift(-afwGeom.ExtentI(xy0))
displayUtils.drawBBox(bbox,
borderWidth=borderWidth,
ctype=ctype,
display=display)
def show_image_and_mask(exp):
"""
show the image and mask in ds9, with
mask colored
Parameters
----------
exp: afw_image.MaskedImageF
The image to show
"""
import lsst.afw.display as afw_display
display = afw_display.getDisplay(backend='ds9')
display.mtv(exp)
display.scale('log', 'minmax')
def plotDeblendFamily(mi, parent, kids, mapperInfo=None, dkids=[],
background=-10, symbolSize=2,
plotb=False,
arcsinh=True, maskbit=False, display=afwDisplay.getDisplay(0)):
"""Display a deblend using afwDisplay
Each child is marked with a + at its centre (green if deblended-as-psf else red)
all the other peaks in its footprint are marked with x (cyan if deblended-as-psf else magenta)
"""
if mi:
try:
mi = mi.getMaskedImage() # maybe it's an Exposure?
except AttributeError:
pass
mos = makeDeblendFamilyMosaic(mi, parent, kids, mapperInfo, background, maskbit)
if mapperInfo:
# some displays, e.g. ds9, don't handle those chars well
title = re.sub(r"[{}']", "", str(mapperInfo.getId(parent, None)))
else:
title = "0x%x == %d" % (parent.getId(), (parent.getId() & 0xffff))
mosaicImage = mos.makeMosaic(display=display, title=title)
display.dot("%s (%.1f, %1.f)" % (title, parent.getX(), parent.getY()),
0.5*mosaicImage.getWidth(), 1.03*mosaicImage.getHeight(),
ctype=afwDisplay.BLACK, fontFamily="times", size=3)
px0, py0 = footprintToImage(parent.getFootprint(), mi).getXY0()
with display.Buffering():
for i, src in enumerate([parent] + kids):
x0, y0 = mos.getBBox(i).getMin()
x0 -= px0; y0 -= py0
if src.get("deblend_deblendedAsPsf"):
centroid_ctype = afwDisplay.GREEN
peak_ctype = afwDisplay.CYAN
else:
centroid_ctype = afwDisplay.RED
peak_ctype = afwDisplay.MAGENTA
display.dot("+", src.getX() + x0, src.getY() + y0,
size=symbolSize, ctype=centroid_ctype)
for p in src.getFootprint().getPeaks():
display.dot("x", p.getFx() + x0, p.getFy() + y0,
size=0.5*symbolSize if i == 0 else symbolSize,
ctype=afwDisplay.YELLOW if i == 0 else peak_ctype)
def testImageTypes(self):
"""Check that we can display a range of types of image"""
with afwDisplay.getDisplay("dummy", "virtualDevice") as dummy:
for imageType in [afwImage.DecoratedImageF,
afwImage.ExposureF,
afwImage.ImageU,
afwImage.ImageI,
afwImage.ImageF,
afwImage.MaskedImageF,
]:
im = imageType(self.fileName)
dummy.mtv(im)
im = afwImage.MaskU(self.fileName, 3)
dummy.mtv(im)
def testImageTypes(self):
"""Check that we can display a range of types of image"""
with afwDisplay.getDisplay("dummy", "virtualDevice") as dummy:
for imageType in [afwImage.DecoratedImageF,
afwImage.ExposureF,
afwImage.ImageU,
afwImage.ImageI,
afwImage.ImageF,
afwImage.MaskedImageF,
]:
im = imageType(self.fileName)
dummy.mtv(im)
im = afwImage.MaskU(self.fileName, 3)
dummy.mtv(im)
def do_display(repo, filename, image_dataset='calex', cat_dataset='src'):
display = afwDisplay.getDisplay()
butler = dafPersist.Butler(repo)
dataId = {'filename': filename}
exp = butler.get(image_dataset, dataId=dataId)
src = butler.get(cat_dataset, dataId=dataId)
with display.Buffering():
display.mtv(exp)
for s in src:
if s['parent'] == 0:
display.dot('o', s.getX(), s.getY(), ctype='blue')
else:
display.dot('+', s.getX(), s.getY(), ctype='red')
def testShowPsf(self):
""" Test that the showPsfMosaic function works.
This function is usually called without display=None, which would activate ds9
"""
# Measure PSF so we have a real PSF to work with
self.setupDeterminer()
metadata = dafBase.PropertyList()
stars = self.starSelector.run(self.catalog, exposure=self.exposure)
psfCandidateList = self.makePsfCandidates.run(
stars.sourceCat, self.exposure).psfCandidates
psf, cellSet = self.psfDeterminer.determinePsf(self.exposure,
psfCandidateList,
metadata)
testDisplay = display if display else afwDisplay.getDisplay(
backend="virtualDevice")
mos = showPsf(psf, display=testDisplay)
self.assertTrue(len(mos.images) > 0)
def debugView(self, stepname, exposure):
"""Utility function to examine the image being processed.
Parameters
----------
stepname : `str`
State of processing to view.
exposure : `lsst.afw.image.Exposure`
Exposure to view.
"""
frame = getDebugFrame(self._display, stepname)
if frame:
display = getDisplay(frame)
display.scale('asinh', 'zscale')
display.mtv(exposure)
prompt = "Press Enter to continue: "
while True:
ans = input(prompt).lower()
if ans in ("", "c",):
break
def showAmp(amp, imageSource=FakeImageDataSource(isTrimmed=False), display=None, overlay=True,
imageFactory=afwImage.ImageU):
"""!Show an amp in an image display
@param[in] amp amp record to use in display
@param[in] imageSource Source for getting the amp image. Must have a getAmpImage method.
@param[in] display image display to use
@param[in] overlay Overlay bounding boxes?
@param[in] imageFactory Type of image to display (only used if ampImage is None)
"""
if not display:
display = afwDisplay.getDisplay()
ampImage = imageSource.getAmpImage(amp, imageFactory=imageFactory)
ampImSize = ampImage.getDimensions()
title = amp.getName()
display.mtv(ampImage, title=title)
if overlay:
with afwDisplay.Buffering():
if amp.getHasRawInfo() and ampImSize == amp.getRawBBox().getDimensions():
bboxes = [(amp.getRawBBox(), 0.49, afwDisplay.GREEN),]
xy0 = bboxes[0][0].getMin()
bboxes.append((amp.getRawHorizontalOverscanBBox(), 0.49, afwDisplay.RED))
bboxes.append((amp.getRawDataBBox(), 0.49, afwDisplay.BLUE))
bboxes.append((amp.getRawPrescanBBox(), 0.49, afwDisplay.YELLOW))
bboxes.append((amp.getRawVerticalOverscanBBox(), 0.49, afwDisplay.MAGENTA))
else:
bboxes = [(amp.getBBox(), 0.49, None),]
xy0 = bboxes[0][0].getMin()
for bbox, borderWidth, ctype in bboxes:
if bbox.isEmpty():
continue
bbox = afwGeom.Box2I(bbox)
bbox.shift(-afwGeom.ExtentI(xy0))
displayUtils.drawBBox(bbox, borderWidth=borderWidth, ctype=ctype, display=display)
def displaySources(self, exposure, matches, reserved, frame=1):
"""Display sources we'll use for photocal
Sources that will be actually used will be green.
Sources reserved from the fit will be red.
Parameters
----------
exposure : `lsst.afw.image.ExposureF`
Exposure to display.
matches : `list` of `lsst.afw.table.RefMatch`
Matches used for photocal.
reserved : `numpy.ndarray` of type `bool`
Boolean array indicating sources that are reserved.
frame : `int`
Frame number for display.
"""
disp = afwDisplay.getDisplay(frame=frame)
disp.mtv(exposure, title="photocal")
with disp.Buffering():
for mm, rr in zip(matches, reserved):
x, y = mm.second.getCentroid()
ctype = afwDisplay.RED if rr else afwDisplay.GREEN
disp.dot("o", x, y, size=4, ctype=ctype)
def displaySources(self, exposure, matches, reserved, frame=1):
"""Display sources we'll use for photocal
Sources that will be actually used will be green.
Sources reserved from the fit will be red.
Parameters
----------
exposure : `lsst.afw.image.ExposureF`
Exposure to display.
matches : `list` of `lsst.afw.table.RefMatch`
Matches used for photocal.
reserved : `numpy.ndarray` of type `bool`
Boolean array indicating sources that are reserved.
frame : `int`
Frame number for display.
"""
disp = afwDisplay.getDisplay(frame=frame)
disp.mtv(exposure, title="photocal")
with disp.Buffering():
for mm, rr in zip(matches, reserved):
x, y = mm.second.getCentroid()
ctype = afwDisplay.RED if rr else afwDisplay.GREEN
disp.dot("o", x, y, size=4, ctype=ctype)
def fitBackground(self, maskedImage, nx=0, ny=0, algorithm=None):
"""!Estimate the background of a masked image
@param[in] maskedImage masked image whose background is to be computed
@param[in] nx number of x bands; if 0 compute from width and config.binSizeX
@param[in] ny number of y bands; if 0 compute from height and config.binSizeY
@param[in] algorithm name of interpolation algorithm; if None use self.config.algorithm
@return fit background as an lsst.afw.math.Background
@throw RuntimeError if lsst.afw.math.makeBackground returns None,
which is apparently one way it indicates failure
"""
binSizeX = self.config.binSize if self.config.binSizeX == 0 else self.config.binSizeX
binSizeY = self.config.binSize if self.config.binSizeY == 0 else self.config.binSizeY
if not nx:
nx = maskedImage.getWidth() // binSizeX + 1
if not ny:
ny = maskedImage.getHeight() // binSizeY + 1
unsubFrame = getDebugFrame(self._display, "unsubtracted")
if unsubFrame:
unsubDisp = afwDisplay.getDisplay(frame=unsubFrame)
unsubDisp.mtv(maskedImage, title="unsubtracted")
xPosts = numpy.rint(
numpy.linspace(0,
maskedImage.getWidth() + 1,
num=nx,
endpoint=True))
yPosts = numpy.rint(
numpy.linspace(0,
maskedImage.getHeight() + 1,
num=ny,
endpoint=True))
with unsubDisp.Buffering():
for (xMin, xMax), (yMin, yMax) in itertools.product(
zip(xPosts[:-1], xPosts[1:]),
zip(yPosts[:-1], yPosts[1:])):
unsubDisp.line([(xMin, yMin), (xMin, yMax), (xMax, yMax),
(xMax, yMin), (xMin, yMin)])
sctrl = afwMath.StatisticsControl()
badMask = maskedImage.mask.getPlaneBitMask(
self.config.ignoredPixelMask)
sctrl.setAndMask(badMask)
sctrl.setNanSafe(self.config.isNanSafe)
self.log.debug("Ignoring mask planes: %s" %
", ".join(self.config.ignoredPixelMask))
if (maskedImage.mask.getArray() & badMask).all():
raise pipeBase.TaskError(
"All pixels masked. Cannot estimate background")
if algorithm is None:
algorithm = self.config.algorithm
# TODO: DM-22814. This call to a deprecated BackgroundControl constructor
# is necessary to support the algorithm parameter; it # should be replaced with
#
# afwMath.BackgroundControl(nx, ny, sctrl, self.config.statisticsProperty)
#
# when algorithm has been deprecated and removed.
with suppress_deprecations():
bctrl = afwMath.BackgroundControl(algorithm, nx, ny,
self.config.undersampleStyle,
sctrl,
self.config.statisticsProperty)
# TODO: The following check should really be done within lsst.afw.math.
# With the current code structure, it would need to be accounted for in the doGetImage()
# function in BackgroundMI.cc (which currently only checks against the interpolation settings,
# which is not appropriate when useApprox=True)
# and/or the makeApproximate() function in afw/Approximate.cc.
# See ticket DM-2920: "Clean up code in afw for Approximate background
# estimation" (which includes a note to remove the following and the
# similar checks in pipe_tasks/matchBackgrounds.py once implemented)
#
# Check that config setting of approxOrder/binSize make sense
# (i.e. ngrid (= shortDimension/binSize) > approxOrderX) and perform
# appropriate undersampleStlye behavior.
if self.config.useApprox:
if self.config.approxOrderY not in (self.config.approxOrderX, -1):
raise ValueError(
"Error: approxOrderY not in (approxOrderX, -1)")
order = self.config.approxOrderX
minNumberGridPoints = order + 1
if min(nx, ny) <= order:
self.log.warn(
"Too few points in grid to constrain fit: min(nx, ny) < approxOrder) "
"[min(%d, %d) < %d]" % (nx, ny, order))
if self.config.undersampleStyle == "THROW_EXCEPTION":
raise ValueError(
"Too few points in grid (%d, %d) for order (%d) and binSize (%d, %d)"
% (nx, ny, order, binSizeX, binSizeY))
elif self.config.undersampleStyle == "REDUCE_INTERP_ORDER":
if order < 1:
raise ValueError(
"Cannot reduce approxOrder below 0. "
"Try using undersampleStyle = \"INCREASE_NXNYSAMPLE\" instead?"
)
order = min(nx, ny) - 1
self.log.warn("Reducing approxOrder to %d" % order)
elif self.config.undersampleStyle == "INCREASE_NXNYSAMPLE":
# Reduce bin size to the largest acceptable square bins
newBinSize = min(
maskedImage.getWidth(),
maskedImage.getHeight()) // (minNumberGridPoints - 1)
if newBinSize < 1:
raise ValueError("Binsize must be greater than 0")
newNx = maskedImage.getWidth() // newBinSize + 1
newNy = maskedImage.getHeight() // newBinSize + 1
bctrl.setNxSample(newNx)
bctrl.setNySample(newNy)
self.log.warn(
"Decreasing binSize from (%d, %d) to %d for a grid of (%d, %d)"
% (binSizeX, binSizeY, newBinSize, newNx, newNy))
actrl = afwMath.ApproximateControl(
afwMath.ApproximateControl.CHEBYSHEV, order, order,
self.config.weighting)
bctrl.setApproximateControl(actrl)
bg = afwMath.makeBackground(maskedImage, bctrl)
if bg is None:
raise RuntimeError(
"lsst.afw.math.makeBackground failed to fit a background model"
)
return bg
def showBlend(calexp,
families,
key='d',
background=0.0,
display=afwDisplay.getDisplay(0),
imageDisplay=None,
mtv=False):
"""Show blends interactively on an afwDisplay
\param calexp Exposure containing objects of interest
\param families A Families object
\param key Key to display the family under the cursor in Display display
\param display The afwDisplay.Display to display the families
\param imageDisplay The afwDisplay.Display displaying calexp (see mtv)
\param mtv If true, display calexp on display
E.g.
import lsst.daf.persistence as dafPersist
import lsst.analysis.deblender as deblender
butler = dafPersist.Butler("/home/astro/hsc/hsc/HSC/rerun/rhl/tmp")
did = dict(visit=905518, ccd=31)
calexp = butler.get("calexp", **did)
ss = butler.get("src", **did)
families = deblender.Families(ss, butler, nChildMin=0)
deblender.showBlend(calexp, families, display=afwDisplay.Display(1))
Then hit 'key' (default: d) on objects of interest; 'r' for an rgb image
"""
if display is not None:
if mtv:
imageDisplay.mtv(calexp)
old = {}
try:
for k in set(list("ha") + [key]):
old[k] = display.setCallback(k)
display.setCallback(
key, makeDisplayFamily(calexp, families, display=display))
def new_h(*args):
old['h'](*args)
print " 1,2,4,8: Zoom to specified scale"
print " a: show All the pixels"
print " %s: show family under the cursor and return to python prompt" % key
print " l: cycle through stretch types"
display.setCallback('h', new_h)
display.setCallback('a', lambda k, x, y: display.zoom("to fit"))
for z in [1, 2, 4, 8]:
def _zoom(k, x, y, z=z):
"""Zoom by %d""" % z
display.zoom(z)
display.setCallback('%d' % z, _zoom)
def callbackLog(k, x, y, i=[0]):
"""Cycle through stretches"""
i[0] = (i[0] + 1) % 3
if i[0] == 0:
display.scale("log", "minmax")
elif i[0] == 1:
display.scale("linear", "minmax")
elif i[0] == 2:
display.scale("linear", "zscale")
display.setCallback('l', callbackLog)
display.setCallback('r', makeDisplayFamily(calexp, families, rgb=True))
display.interact()
except Exception as e:
print("Error in callback: %s" % e)
finally:
# Cleaning up; disable if you want to be able to call and debug the callbacks
for k, func in old.items():
display.setCallback(k, func)
def setUp(self):
size = 128 # size of image (pixels)
center = afwGeom.Point2D(size//2, size//2) # object center
width = 2.0 # PSF width
flux = 10.0 # Flux of object
variance = 1.0 # Mean variance value
varianceStd = 0.1 # Standard deviation of the variance value
# Set a seed for predictable randomness
np.random.seed(300)
# Create a random image to be used as variance plane
variancePlane = np.random.normal(variance, varianceStd, size*size).reshape(size,size)
# Initial setup of an image
exp = afwImage.ExposureF(size, size)
image = exp.getMaskedImage().getImage()
mask = exp.getMaskedImage().getMask()
var = exp.getMaskedImage().getVariance()
image.set(0.0)
mask.set(0)
var.getArray()[:,:] = variancePlane
# Put down a PSF
psfSize = int(6*width + 1) # Size of PSF image; must be odd
psf = afwDetection.GaussianPsf(psfSize, psfSize, width)
exp.setPsf(psf)
psfImage = psf.computeImage(center).convertF()
psfImage *= flux
image.Factory(image, psfImage.getBBox(afwImage.PARENT)).__iadd__(psfImage)
var.Factory(var, psfImage.getBBox(afwImage.PARENT)).__iadd__(psfImage)
# Put in some bad pixels to ensure they're ignored
for i in range(-5, 6):
bad = size//2 + i*width
var.getArray()[bad, :] = float("nan")
mask.getArray()[bad, :] = mask.getPlaneBitMask("BAD")
var.getArray()[:, bad] = float("nan")
mask.getArray()[:, bad] = mask.getPlaneBitMask("BAD")
# Put in some unmasked bad pixels outside the expected aperture, to ensure the aperture is working
var.getArray()[0, 0] = float("nan")
var.getArray()[0, -1] = float("nan")
var.getArray()[-1, 0] = float("nan")
var.getArray()[-1, -1] = float("nan")
if display:
import lsst.afw.display as afwDisplay
afwDisplay.getDisplay(1).mtv(image)
afwDisplay.getDisplay(2).mtv(mask)
afwDisplay.getDisplay(3).mtv(var)
config = measBase.SingleFrameMeasurementConfig()
config.plugins.names = ["base_NaiveCentroid", "base_SdssShape", "base_Variance"]
config.slots.centroid = "base_NaiveCentroid"
config.slots.psfFlux = None
config.slots.apFlux = None
config.slots.modelFlux = None
config.slots.instFlux = None
config.slots.calibFlux = None
config.slots.shape = "base_SdssShape"
config.plugins["base_Variance"].mask = ["BAD", "SAT"]
config.validate()
schema = afwTable.SourceTable.makeMinimalSchema()
task = measBase.SingleFrameMeasurementTask(schema, config=config)
catalog = afwTable.SourceCatalog(schema)
foot = afwDetection.Footprint(afwGeom.Point2I(center), width)
peak = foot.getPeaks().addNew()
peak.setIx(int(center.getX()))
peak.setIy(int(center.getY()))
peak.setFx(center.getX())
peak.setFy(center.getY())
peak.setPeakValue(flux)
source = catalog.addNew()
source.setFootprint(foot)
self.variance = variance
self.varianceStd = varianceStd
self.mask = mask
self.catalog = catalog
self.exp = exp
self.task = task
self.source = source
def fitBackground(self, maskedImage, nx=0, ny=0, algorithm=None):
"""!Estimate the background of a masked image
@param[in] maskedImage masked image whose background is to be computed
@param[in] nx number of x bands; if 0 compute from width and config.binSize
@param[in] ny number of y bands; if 0 compute from height and config.binSize
@param[in] algorithm name of interpolation algorithm; if None use self.config.algorithm
@return fit background as an lsst.afw.math.Background
@throw RuntimeError if lsst.afw.math.makeBackground returns None,
which is apparently one way it indicates failure
"""
if not nx:
nx = maskedImage.getWidth()//self.config.binSize + 1
if not ny:
ny = maskedImage.getHeight()//self.config.binSize + 1
unsubFrame = getDebugFrame(self._display, "unsubtracted")
if unsubFrame:
unsubDisp = afwDisplay.getDisplay(frame=unsubFrame)
unsubDisp.mtv(maskedImage, title="unsubtracted")
xPosts = numpy.rint(numpy.linspace(0, maskedImage.getWidth() + 1, num=nx, endpoint=True))
yPosts = numpy.rint(numpy.linspace(0, maskedImage.getHeight() + 1, num=ny, endpoint=True))
with unsubDisp.Buffering():
for (xMin, xMax), (yMin, yMax) in itertools.product(zip(xPosts[:-1], xPosts[1:]),
zip(yPosts[:-1], yPosts[1:])):
unsubDisp.line([(xMin, yMin), (xMin, yMax), (xMax, yMax), (xMax, yMin), (xMin, yMin)])
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(reduce(lambda x, y: x | maskedImage.getMask().getPlaneBitMask(y),
self.config.ignoredPixelMask, 0x0))
sctrl.setNanSafe(self.config.isNanSafe)
self.log.logdebug("Ignoring mask planes: %s" % ", ".join(self.config.ignoredPixelMask))
if algorithm is None:
algorithm = self.config.algorithm
bctrl = afwMath.BackgroundControl(algorithm, nx, ny,
self.config.undersampleStyle, sctrl,
self.config.statisticsProperty)
# TODO: The following check should really be done within lsst.afw.math.
# With the current code structure, it would need to be accounted for in the doGetImage()
# function in BackgroundMI.cc (which currently only checks against the interpolation settings,
# which is not appropriate when useApprox=True)
# and/or the makeApproximate() function in afw/Approximate.cc.
# See ticket DM-2920: "Clean up code in afw for Approximate background
# estimation" (which includes a note to remove the following and the
# similar checks in pipe_tasks/matchBackgrounds.py once implemented)
#
# Check that config setting of approxOrder/binSize make sense
# (i.e. ngrid (= shortDimension/binSize) > approxOrderX) and perform
# appropriate undersampleStlye behavior.
if self.config.useApprox:
if self.config.approxOrderY not in (self.config.approxOrderX, -1):
raise ValueError("Error: approxOrderY not in (approxOrderX, -1)")
order = self.config.approxOrderX
minNumberGridPoints = self.config.approxOrderX + 1
if min(nx,ny) <= self.config.approxOrderX:
self.log.warn("Too few points in grid to constrain fit: min(nx, ny) < approxOrder) "
"[min(%d, %d) < %d]" % (nx, ny, self.config.approxOrderX))
if self.config.undersampleStyle == "THROW_EXCEPTION":
raise ValueError("Too few points in grid (%d, %d) for order (%d) and binsize (%d)" % (
nx, ny, self.config.approxOrderX, self.config.binSize))
elif self.config.undersampleStyle == "REDUCE_INTERP_ORDER":
if order < 1:
raise ValueError("Cannot reduce approxOrder below 0. " +
"Try using undersampleStyle = \"INCREASE_NXNYSAMPLE\" instead?")
order = min(nx, ny) - 1
self.log.warn("Reducing approxOrder to %d" % order)
elif self.config.undersampleStyle == "INCREASE_NXNYSAMPLE":
newBinSize = min(maskedImage.getWidth(),maskedImage.getHeight())//(minNumberGridPoints-1)
if newBinSize < 1:
raise ValueError("Binsize must be greater than 0")
newNx = maskedImage.getWidth()//newBinSize + 1
newNy = maskedImage.getHeight()//newBinSize + 1
bctrl.setNxSample(newNx)
bctrl.setNySample(newNy)
self.log.warn("Decreasing binSize from %d to %d for a grid of (%d, %d)" %
(self.config.binSize, newBinSize, newNx, newNy))
actrl = afwMath.ApproximateControl(afwMath.ApproximateControl.CHEBYSHEV, order, order,
self.config.weighting)
bctrl.setApproximateControl(actrl)
bg = afwMath.makeBackground(maskedImage, bctrl)
if bg is None:
raise RuntimeError("lsst.afw.math.makeBackground failed to fit a background model")
return bg
def testZoomPan(self):
self.display0.pan(205, 180)
self.display0.zoom(4)
afwDisplay.getDisplay(1).zoom(4, 205, 180)
def showBlend(calexp, families, key='d', background=-0.1, display0=afwDisplay.getDisplay(0),
display=None, mtv=False):
"""Show blends interactively on an afwDisplay
\param calexp Exposure containing objects of interest
\param families A Families object
\param key Key to display the family under the cursor in Display display
\param display0 The afwDisplay.Display to display the families
\param display The afwDisplay.Display displaying calexp (see mtv)
\param mtv If true, display calexp on display
E.g.
import lsst.daf.persistence as dafPersist
import lsst.analysis.deblender as deblender
butler = dafPersist.Butler("/home/astro/hsc/hsc/HSC/rerun/rhl/tmp")
did = dict(visit=905518, ccd=31)
calexp = butler.get("calexp", **did)
ss = butler.get("src", **did)
families = deblender.Families(ss, butler, nChildMin=0)
deblender.showBlend(calexp, families, display=afwDisplay.Display(1))
Then hit 'key' (default: d) on objects of interest; 'r' for an rgb image
"""
if display is not None:
if mtv:
display.mtv(calexp)
old = {}
try:
for k in set(list("ha") + [key]):
old[k] = display.setCallback(k)
display.setCallback(key, makeDisplayFamily(calexp, families, display=display0))
def new_h(*args):
old['h'](*args)
print " 1,2,4,8: Zoom to specified scale"
print " a: show All the pixels"
print " %s: show family under the cursor and return to python prompt" % key
print " l: cycle through stretch types"
display.setCallback('h', new_h)
display.setCallback('a', lambda k, x, y: display.zoom("to fit"))
for z in [1, 2, 4, 8]:
def _zoom(k, x, y, z=z):
display0.zoom(z)
display.setCallback('%d' % z, _zoom)
def callbackLog(k, x, y, i=[0]):
"""Cycle through stretches"""
i[0] = (i[0] + 1)%3
if i[0] == 0:
display.scale("log", "minmax")
elif i[0] == 1:
display.scale("linear", "minmax")
elif i[0] == 2:
display.scale("linear", "zscale")
display.setCallback('r', makeDisplayFamily(calexp, families, rgb=True))
display.interact()
except Exception, e:
print "RHL", e
def displayAstrometry(refCat=None,
sourceCat=None,
distortedCentroidKey=None,
bbox=None,
exposure=None,
matches=None,
frame=1,
title="",
pause=True):
"""Show an astrometry debug image.
Parameters
----------
refCat : `lsst.afw.table.SimpleCatalog`
reference object catalog; must have fields "centroid_x" an
"centroid_y"
sourceCat : `lsst.afw.table.SourceCatalg`
source catalog; must have field "slot_Centroid_x" and "slot_Centroid_y"
distortedCentroidKey : `lsst.afw.table.Key`
key for sourceCat with field to use for distorted positions
exposure : `lsst.afw.image.Exposure`
exposure to display
bbox : `lsst.geom.Box2I`
bounding box of exposure; Used if the exposure is `None`
matches : `list` of `lsst.afw.table.ReferenceMatch`
List of matched objects
frame : `int`
frame number for display
title : `str`
title for display
pause : `bool`
pause for inspection of display? This is done by dropping into pdb.
Notes
-----
- reference objects in refCat are shown as red X
- sources in sourceCat are shown as green +
- distorted sources in sourceCat (position given by distortedCentroidKey)
are shown as green o
- matches are shown as a yellow circle around the source and a yellow line
connecting the reference object and source
- if both exposure and bbox are `None`, no image is displayed
"""
disp = afwDisplay.getDisplay(frame=frame)
if exposure is not None:
disp.mtv(exposure, title=title)
elif bbox is not None:
disp.mtv(exposure=ExposureF(bbox), title=title)
with disp.Buffering():
if refCat is not None:
refCentroidKey = Point2DKey(refCat.schema["centroid"])
for refObj in refCat:
rx, ry = refObj.get(refCentroidKey)
disp.dot("x", rx, ry, size=10, ctype=afwDisplay.RED)
if sourceCat is not None:
sourceCentroidKey = Point2DKey(sourceCat.schema["slot_Centroid"])
for source in sourceCat:
sx, sy = source.get(sourceCentroidKey)
disp.dot("+", sx, sy, size=10, ctype=afwDisplay.GREEN)
if distortedCentroidKey is not None:
dx, dy = source.get(distortedCentroidKey)
disp.dot("o", dx, dy, size=10, ctype=afwDisplay.GREEN)
disp.line([(sx, sy), (dx, dy)], ctype=afwDisplay.GREEN)
if matches is not None:
refCentroidKey = Point2DKey(matches[0].first.schema["centroid"])
sourceCentroidKey = Point2DKey(
matches[0].second.schema["slot_Centroid"])
radArr = np.ndarray(len(matches))
for i, m in enumerate(matches):
refCentroid = m.first.get(refCentroidKey)
sourceCentroid = m.second.get(sourceCentroidKey)
radArr[i] = math.hypot(*(refCentroid - sourceCentroid))
sx, sy = sourceCentroid
disp.dot("o", sx, sy, size=10, ctype=afwDisplay.YELLOW)
disp.line([refCentroid, sourceCentroid],
ctype=afwDisplay.YELLOW)
print("<match radius> = %.4g +- %.4g [%d matches]" %
(radArr.mean(), radArr.std(), len(matches)))
if pause:
print(
"Dropping into debugger to allow inspection of display. Type 'continue' when done."
)
import pdb
pdb.set_trace()
def testWith(self):
"""Test using displays with with statement"""
with afwDisplay.getDisplay(0) as disp:
self.assertIsNotNone(disp)
def run(self, ccdExposure, bias=None, dark=None, flat=None, defects=None, fringes=None, bfKernel=None):
"""!Perform instrument signature removal on an exposure
Steps include:
- Detect saturation, apply overscan correction, bias, dark and flat
- Perform CCD assembly
- Interpolate over defects, saturated pixels and all NaNs
\param[in] ccdExposure -- lsst.afw.image.exposure of detector data
\param[in] bias -- exposure of bias frame
\param[in] dark -- exposure of dark frame
\param[in] flat -- exposure of flatfield
\param[in] defects -- list of detects
\param[in] fringes -- a pipeBase.Struct with field fringes containing
exposure of fringe frame or list of fringe exposure
\param[in] bfKernel -- kernel for brighter-fatter correction
\return a pipeBase.Struct with field:
- exposure
"""
#Validate Input
if self.config.doBias and bias is None:
raise RuntimeError("Must supply a bias exposure if config.doBias True")
if self.config.doDark and dark is None:
raise RuntimeError("Must supply a dark exposure if config.doDark True")
if self.config.doFlat and flat is None:
raise RuntimeError("Must supply a flat exposure if config.doFlat True")
if self.config.doBrighterFatter and bfKernel is None:
raise RuntimeError("Must supply a kernel if config.doBrighterFatter True")
if fringes is None:
fringes = pipeBase.Struct(fringes=None)
if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
raise RuntimeError("Must supply fringe exposure as a pipeBase.Struct")
defects = [] if defects is None else defects
ccd = ccdExposure.getDetector()
ccdExposure = self.convertIntToFloat(ccdExposure)
if not ccd:
assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
ccd = [FakeAmp(ccdExposure, self.config)]
for amp in ccd:
#if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
if ccdExposure.getBBox().contains(amp.getBBox()):
self.saturationDetection(ccdExposure, amp)
self.overscanCorrection(ccdExposure, amp)
if self.config.doAssembleCcd:
ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
if self.config.doBias:
self.biasCorrection(ccdExposure, bias)
if self.config.doBrighterFatter:
self.brighterFatterCorrection(ccdExposure, bfKernel,
self.config.brighterFatterMaxIter,
self.config.brighterFatterThreshold,
self.config.brighterFatterApplyGain,
)
if self.config.doDark:
self.darkCorrection(ccdExposure, dark)
for amp in ccd:
#if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
if ccdExposure.getBBox().contains(amp.getBBox()):
ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
self.updateVariance(ampExposure, amp)
if self.config.doFringe and not self.config.fringeAfterFlat:
self.fringe.run(ccdExposure, **fringes.getDict())
if self.config.doFlat:
self.flatCorrection(ccdExposure, flat)
self.maskAndInterpDefect(ccdExposure, defects)
self.saturationInterpolation(ccdExposure)
self.maskAndInterpNan(ccdExposure)
if self.config.doFringe and self.config.fringeAfterFlat:
self.fringe.run(ccdExposure, **fringes.getDict())
ccdExposure.getCalib().setFluxMag0(self.config.fluxMag0T1 * ccdExposure.getCalib().getExptime())
frame = getDebugFrame(self._display, "postISRCCD")
if frame:
getDisplay(frame).mtv(ccdExposure)
return pipeBase.Struct(
exposure = ccdExposure,
)
def runDebug(self, exposure, subtractRes, selectSources, kernelSources, diaSources):
"""@todo Test and update for current debug display and slot names
"""
import lsstDebug
display = lsstDebug.Info(__name__).display
showSubtracted = lsstDebug.Info(__name__).showSubtracted
showPixelResiduals = lsstDebug.Info(__name__).showPixelResiduals
showDiaSources = lsstDebug.Info(__name__).showDiaSources
showDipoles = lsstDebug.Info(__name__).showDipoles
maskTransparency = lsstDebug.Info(__name__).maskTransparency
if display:
disp = afwDisplay.getDisplay(frame=lsstDebug.frame)
if not maskTransparency:
maskTransparency = 0
disp.setMaskTransparency(maskTransparency)
if display and showSubtracted:
disp.mtv(subtractRes.subtractedExposure, title="Subtracted image")
mi = subtractRes.subtractedExposure.getMaskedImage()
x0, y0 = mi.getX0(), mi.getY0()
with disp.Buffering():
for s in diaSources:
x, y = s.getX() - x0, s.getY() - y0
ctype = "red" if s.get("flags_negative") else "yellow"
if (s.get("base_PixelFlags_flag_interpolatedCenter") or
s.get("base_PixelFlags_flag_saturatedCenter") or
s.get("base_PixelFlags_flag_crCenter")):
ptype = "x"
elif (s.get("base_PixelFlags_flag_interpolated") or
s.get("base_PixelFlags_flag_saturated") or
s.get("base_PixelFlags_flag_cr")):
ptype = "+"
else:
ptype = "o"
disp.dot(ptype, x, y, size=4, ctype=ctype)
lsstDebug.frame += 1
if display and showPixelResiduals and selectSources:
nonKernelSources = []
for source in selectSources:
if source not in kernelSources:
nonKernelSources.append(source)
diUtils.plotPixelResiduals(exposure,
subtractRes.warpedExposure,
subtractRes.subtractedExposure,
subtractRes.kernelCellSet,
subtractRes.psfMatchingKernel,
subtractRes.backgroundModel,
nonKernelSources,
self.subtract.config.kernel.active.detectionConfig,
origVariance=False)
diUtils.plotPixelResiduals(exposure,
subtractRes.warpedExposure,
subtractRes.subtractedExposure,
subtractRes.kernelCellSet,
subtractRes.psfMatchingKernel,
subtractRes.backgroundModel,
nonKernelSources,
self.subtract.config.kernel.active.detectionConfig,
origVariance=True)
if display and showDiaSources:
flagChecker = SourceFlagChecker(diaSources)
isFlagged = [flagChecker(x) for x in diaSources]
isDipole = [x.get("ip_diffim_ClassificationDipole_value") for x in diaSources]
diUtils.showDiaSources(diaSources, subtractRes.subtractedExposure, isFlagged, isDipole,
frame=lsstDebug.frame)
lsstDebug.frame += 1
if display and showDipoles:
DipoleAnalysis().displayDipoles(subtractRes.subtractedExposure, diaSources,
frame=lsstDebug.frame)
lsstDebug.frame += 1
def testWith(self):
"""Test using displays with with statement"""
with afwDisplay.getDisplay(0) as disp:
self.assertTrue(disp is not None)
def combine(self, cache, struct, outputId):
"""Combine multiple exposures of a particular CCD and write the output
Only the slave nodes execute this method.
Parameters
----------
cache : `lsst.pipe.base.Struct`
Process pool cache.
struct : `lsst.pipe.base.Struct`
Parameters for the combination, which has the following components:
- ``ccdName`` (`tuple`): Name tuple for CCD.
- ``ccdIdList`` (`list`): List of data identifiers for combination.
- ``scales``: Unused by this implementation.
Returns
-------
outputId : `dict`
Data identifier for combined image (exposure part only).
"""
combineResults = super().combine(cache, struct, outputId)
dataRefList = combineResults.dataRefList
outputId = combineResults.outputId
sumFlat = None # Sum of flat-fields
sumExpect = None # Sum of what we expect
xOffsets = []
for ii, expRef in enumerate(dataRefList):
exposure = expRef.get('postISRCCD')
slitOffset = expRef.getButler().queryMetadata("raw", "slitOffset", expRef.dataId)
assert len(slitOffset) == 1, "Expect a single answer for this single dataset"
xOffsets.append(slitOffset.pop())
detMap = expRef.get('detectormap')
traces = self.trace.run(exposure.maskedImage, detMap)
self.log.info('%d FiberTraces found for %s' % (traces.size(), expRef.dataId))
spectra = traces.extractSpectra(exposure.maskedImage, detMap, True)
expect = spectra.makeImage(exposure.getBBox(), traces)
maskVal = exposure.mask.getPlaneBitMask(["BAD", "SAT", "CR"])
bad = (expect.array <= 0.0) | (exposure.mask.array & maskVal > 0)
exposure.image.array[bad] = 0.0
exposure.variance.array[bad] = 0.0
expect.array[bad] = 0.0
if sumFlat is None:
sumFlat = exposure.maskedImage
sumExpect = expect
else:
sumFlat += exposure.maskedImage
sumExpect += expect
self.log.info('xOffsets = %s' % (xOffsets,))
if sumFlat is None:
raise RuntimeError("Unable to find any valid flats")
if np.all(sumExpect.array == 0.0):
raise RuntimeError("No good pixels")
# Avoid NANs when dividing
empty = sumExpect.array == 0
sumFlat.image.array[empty] = 1.0
sumFlat.variance.array[empty] = 1.0
sumExpect.array[empty] = 1.0
sumFlat.mask.addMaskPlane("BAD_FLAT")
badFlat = sumFlat.mask.getPlaneBitMask("BAD_FLAT")
sumFlat /= sumExpect
sumFlat.mask.array[empty] |= badFlat
# Mask bad pixels
snr = sumFlat.image.array/np.sqrt(sumFlat.variance.array)
bad = (snr < self.config.minSNR) | ~np.isfinite(snr)
sumFlat.image.array[bad] = 1.0
sumFlat.mask.array[bad] |= badFlat
import lsstDebug
di = lsstDebug.Info(__name__)
if di.display:
import lsst.afw.display as afwDisplay
if di.framesFlat >= 0:
display = afwDisplay.getDisplay(frame=di.framesFlat)
display.mtv(sumFlat, title='normalized Flat')
if di.zoomPan:
display.zoom(*di.zoomPan)
# Write fiber flat
flatExposure = afwImage.makeExposure(sumFlat)
self.recordCalibInputs(cache.butler, flatExposure, struct.ccdIdList, outputId)
self.interpolateNans(flatExposure)
self.write(cache.butler, flatExposure, outputId)
def run(self, snap0, snap1, defects=None):
"""Combine two snaps
@param[in] snap0: snapshot exposure 0
@param[in] snap1: snapshot exposure 1
@defects[in] defect list (for repair task)
@return a pipe_base Struct with fields:
- exposure: snap-combined exposure
- sources: detected sources, or None if detection not performed
"""
# initialize optional outputs
sources = None
if self.config.doRepair:
self.log.info("snapCombine repair")
psf = self.makeInitialPsf(snap0, fwhmPix=self.config.repairPsfFwhm)
snap0.setPsf(psf)
snap1.setPsf(psf)
self.repair.run(snap0, defects=defects, keepCRs=False)
self.repair.run(snap1, defects=defects, keepCRs=False)
repair0frame = getDebugFrame(self._display, "repair0")
if repair0frame:
getDisplay(repair0frame).mtv(snap0)
repair1frame = getDebugFrame(self._display, "repair1")
if repair1frame:
getDisplay(repair1frame).mtv(snap1)
if self.config.doDiffIm:
if self.config.doPsfMatch:
self.log.info("snapCombine psfMatch")
diffRet = self.diffim.run(snap0, snap1, "subtractExposures")
diffExp = diffRet.subtractedImage
# Measure centroid and width of kernel; dependent on ticket #1980
# Useful diagnostic for the degree of astrometric shift between snaps.
diffKern = diffRet.psfMatchingKernel
width, height = diffKern.getDimensions()
else:
diffExp = afwImage.ExposureF(snap0, True)
diffMi = diffExp.getMaskedImage()
diffMi -= snap1.getMaskedImage()
psf = self.makeInitialPsf(snap0)
diffExp.setPsf(psf)
table = afwTable.SourceTable.make(self.schema)
table.setMetadata(self.algMetadata)
detRet = self.detection.makeSourceCatalog(table, diffExp)
sources = detRet.sources
fpSets = detRet.fpSets
if self.config.doMeasurement:
self.measurement.measure(diffExp, sources)
mask0 = snap0.getMaskedImage().getMask()
mask1 = snap1.getMaskedImage().getMask()
fpSets.positive.setMask(mask0, "DETECTED")
fpSets.negative.setMask(mask1, "DETECTED")
maskD = diffExp.getMaskedImage().getMask()
fpSets.positive.setMask(maskD, "DETECTED")
fpSets.negative.setMask(maskD, "DETECTED_NEGATIVE")
combinedExp = self.addSnaps(snap0, snap1)
return pipeBase.Struct(
exposure=combinedExp,
sources=sources,
)
def testZoomPan(self):
self.display0.pan(205, 180)
self.display0.zoom(4)
afwDisplay.getDisplay(1).zoom(4, 205, 180)
plt.errorbar(lc_filt['mjd'], lc_filt['base_PsfFlux_mag'], lc_filt['base_PsfFlux_magSigma'], label=filt,
marker='o', color=colors[filt], linestyle='none')
ylim = plt.ylim()
plt.ylim(ylim[::-1])
plt.xlabel('MJD')
plt.ylabel('mag')
plt.legend()
plot_file = '{:s}_lc.pdf'.format(field)
plt.savefig(plot_file)
plt.show()
display = afwDisplay.getDisplay(backend='ds9')
display.mtv(calexp)
display.setMaskTransparency(80)
display.scale("asinh", -2, 25)
X = 'slot_Centroid_x'
Y = 'slot_Centroid_y'
display.erase()
with display.Buffering():
for s in ref_table:
display.dot("o", s[X], s[Y], size=10, ctype='orange')
def run(config, inputFiles, weightFiles=None, varianceFiles=None,
returnCalibSources=False, displayResults=[], verbose=False):
#
# Create the tasks
#
schema = afwTable.SourceTable.makeMinimalSchema()
algMetadata = dafBase.PropertyList()
isrTask = IsrTask(config=config.isr)
calibrateTask = CalibrateTask(config=config.calibrate)
sourceDetectionTask = SourceDetectionTask(config=config.detection, schema=schema)
if config.doDeblend:
if SourceDeblendTask:
sourceDeblendTask = SourceDeblendTask(config=config.deblend, schema=schema)
else:
print >> sys.stderr, "Failed to import lsst.meas.deblender; setting doDeblend = False"
config.doDeblend = False
sourceMeasurementTask = SingleFrameMeasurementTask(config=config.measurement,
schema=schema, algMetadata=algMetadata)
sourceMeasurementTask.config.doApplyApCorr = 'yes'
#
# Add fields needed to identify stars while calibrating
#
keysToCopy = [(schema.addField(afwTable.Field["Flag"]("calib_detected",
"Source was detected by calibrate")), None)]
for key in calibrateTask.getCalibKeys():
keysToCopy.append((schema.addField(calibrateTask.schema.find(key).field), key))
exposureDict = {}; calibSourcesDict = {}; sourcesDict = {}
for inputFile, weightFile, varianceFile in zip(inputFiles, weightFiles, varianceFiles):
#
# Create the output table
#
tab = afwTable.SourceTable.make(schema)
#
# read the data
#
if verbose:
print "Reading %s" % inputFile
exposure = makeExposure(inputFile, weightFile, varianceFile,
config.badPixelValue, config.variance)
#
if config.interpPlanes:
import lsst.ip.isr as ipIsr
defects = ipIsr.getDefectListFromMask(exposure.getMaskedImage(), config.interpPlanes,
growFootprints=0)
isrTask.run(exposure, defects=defects)
#
# process the data
#
if config.doCalibrate:
result = calibrateTask.run(exposure)
exposure, calibSources = result.exposure, result.sources
else:
calibSources = None
if not exposure.getPsf():
calibrateTask.installInitialPsf(exposure)
exposureDict[inputFile] = exposure
calibSourcesDict[inputFile] = calibSources if returnCalibSources else None
result = sourceDetectionTask.run(tab, exposure)
sources = result.sources
sourcesDict[inputFile] = sources
if config.doDeblend:
sourceDeblendTask.run(exposure, sources, exposure.getPsf())
sourceMeasurementTask.measure(exposure, sources)
if verbose:
print "Detected %d objects" % len(sources)
propagateCalibFlags(keysToCopy, calibSources, sources)
if displayResults: # display results of processing (see also --debug argparse option)
showApertures = "showApertures".upper() in displayResults
showShapes = "showShapes".upper() in displayResults
display = afwDisplay.getDisplay(frame=1)
if algMetadata.exists("base_CircularApertureFlux_radii"):
radii = algMetadata.get("base_CircularApertureFlux_radii")
else:
radii = []
display.mtv(exposure, title=os.path.split(inputFile)[1])
with display.Buffering():
for s in sources:
xy = s.getCentroid()
display.dot('+', *xy,
ctype=afwDisplay.CYAN if s.get("flags_negative") else afwDisplay.GREEN)
if showShapes:
display.dot(s.getShape(), *xy, ctype=afwDisplay.RED)
if showApertures:
for radius in radii:
display.dot('o', *xy, size=radius, ctype=afwDisplay.YELLOW)
return exposureDict, calibSourcesDict, sourcesDict
def displayAstrometry(refCat=None, sourceCat=None, distortedCentroidKey=None, bbox=None, exposure=None,
matches=None, frame=1, title="", pause=True):
"""Show an astrometry debug image
- reference objects in refCat are shown as red X
- sources in sourceCat are shown as green +
- distorted sources in sourceCat (position given by distortedCentroidKey) are shown as green o
- matches are shown as a yellow circle around the source and a yellow line
connecting the reference object and source
- if both exposure and bbox are None, no image is displayed
@param[in] refCat reference object catalog; must have fields "centroid_x" and "centroid_y"
@param[in] sourceCat source catalog; must have field "slot_Centroid_x" and "slot_Centroid_y"
@param[in] distortedCentroidKey key for sourceCat with field to use for distorted positions, or None
@param[in] exposure exposure to display, or None for a blank exposure
@param[in] bbox bounding box of exposure; used if exposure is None for a blank image
@param[in] matches list of matches (an lsst.afw.table.ReferenceMatchVector), or None
@param[in] frame frame number for ds9 display
@param[in] title title for ds9 display
@param[in] pause pause for inspection of display? This is done by dropping into pdb.
"""
disp = afwDisplay.getDisplay(frame)
if exposure is not None:
disp.mtv(exposure, title=title)
elif bbox is not None:
disp.mtv(exposure=ExposureF(bbox), title=title)
with disp.Buffering():
if refCat is not None:
refCentroidKey = Point2DKey(refCat.schema["centroid"])
for refObj in refCat:
rx, ry = refObj.get(refCentroidKey)
disp.dot("x", rx, ry, size=10, ctype=afwDisplay.RED)
if sourceCat is not None:
sourceCentroidKey = Point2DKey(sourceCat.schema["slot_Centroid"])
for source in sourceCat:
sx, sy = source.get(sourceCentroidKey)
disp.dot("+", sx, sy, size=10, ctype=afwDisplay.GREEN)
if distortedCentroidKey is not None:
dx, dy = source.get(distortedCentroidKey)
disp.dot("o", dx, dy, size=10, ctype=afwDisplay.GREEN)
disp.line([(sx, sy), (dx, dy)], ctype=afwDisplay.GREEN)
if matches is not None:
refCentroidKey = Point2DKey(matches[0].first.schema["centroid"])
sourceCentroidKey = Point2DKey(matches[0].second.schema["slot_Centroid"])
radArr = np.ndarray(len(matches))
for i, m in enumerate(matches):
refCentroid = m.first.get(refCentroidKey)
sourceCentroid = m.second.get(sourceCentroidKey)
radArr[i] = math.hypot(*(refCentroid - sourceCentroid))
sx, sy = sourceCentroid
disp.dot("o", sx, sy, size=10, ctype=afwDisplay.YELLOW)
disp.line([refCentroid, sourceCentroid], ctype=afwDisplay.YELLOW)
print("<match radius> = %.4g +- %.4g [%d matches]" %
(radArr.mean(), radArr.std(), len(matches)))
if pause:
print("Dropping into debugger to allow inspection of display. Type 'continue' when done.")
import pdb
pdb.set_trace()
def run(self, snap0, snap1, defects=None):
"""Combine two snaps
@param[in] snap0: snapshot exposure 0
@param[in] snap1: snapshot exposure 1
@defects[in] defect list (for repair task)
@return a pipe_base Struct with fields:
- exposure: snap-combined exposure
- sources: detected sources, or None if detection not performed
"""
# initialize optional outputs
sources = None
if self.config.doRepair:
self.log.info("snapCombine repair")
psf = self.makeInitialPsf(snap0, fwhmPix=self.config.repairPsfFwhm)
snap0.setPsf(psf)
snap1.setPsf(psf)
self.repair.run(snap0, defects=defects, keepCRs=False)
self.repair.run(snap1, defects=defects, keepCRs=False)
repair0frame = getDebugFrame(self._display, "repair0")
if repair0frame:
getDisplay(repair0frame).mtv(snap0)
repair1frame = getDebugFrame(self._display, "repair1")
if repair1frame:
getDisplay(repair1frame).mtv(snap1)
if self.config.doDiffIm:
if self.config.doPsfMatch:
self.log.info("snapCombine psfMatch")
diffRet = self.diffim.run(snap0, snap1, "subtractExposures")
diffExp = diffRet.subtractedImage
# Measure centroid and width of kernel; dependent on ticket #1980
# Useful diagnostic for the degree of astrometric shift between snaps.
diffKern = diffRet.psfMatchingKernel
width, height = diffKern.getDimensions()
else:
diffExp = afwImage.ExposureF(snap0, True)
diffMi = diffExp.getMaskedImage()
diffMi -= snap1.getMaskedImage()
psf = self.makeInitialPsf(snap0)
diffExp.setPsf(psf)
table = afwTable.SourceTable.make(self.schema)
table.setMetadata(self.algMetadata)
detRet = self.detection.run(table, diffExp)
sources = detRet.sources
fpSets = detRet.fpSets
if self.config.doMeasurement:
self.measurement.measure(diffExp, sources)
mask0 = snap0.getMaskedImage().getMask()
mask1 = snap1.getMaskedImage().getMask()
fpSets.positive.setMask(mask0, "DETECTED")
fpSets.negative.setMask(mask1, "DETECTED")
maskD = diffExp.getMaskedImage().getMask()
fpSets.positive.setMask(maskD, "DETECTED")
fpSets.negative.setMask(maskD, "DETECTED_NEGATIVE")
combinedExp = self.addSnaps(snap0, snap1)
return pipeBase.Struct(
exposure=combinedExp,
sources=sources,
)
def run(self,
ccdExposure,
bias=None,
dark=None,
flat=None,
defects=None,
fringes=None):
"""!Perform instrument signature removal on an exposure
Steps include:
- Detect saturation, apply overscan correction, bias, dark and flat
- Perform CCD assembly
- Interpolate over defects, saturated pixels and all NaNs
\param[in] ccdExposure -- lsst.afw.image.exposure of detector data
\param[in] bias -- exposure of bias frame
\param[in] dark -- exposure of dark frame
\param[in] flat -- exposure of flatfield
\param[in] defects -- list of detects
\param[in] fringes -- a pipeBase.Struct with field fringes containing
exposure of fringe frame or list of fringe exposure
\return a pipeBase.Struct with field:
- exposure
"""
#Validate Input
if self.config.doBias and bias is None:
raise RuntimeError(
"Must supply a bias exposure if config.doBias True")
if self.config.doDark and dark is None:
raise RuntimeError(
"Must supply a dark exposure if config.doDark True")
if self.config.doFlat and flat is None:
raise RuntimeError(
"Must supply a flat exposure if config.doFlat True")
if fringes is None:
fringes = pipeBase.Struct(fringes=None)
if self.config.doFringe and not isinstance(fringes, pipeBase.Struct):
raise RuntimeError(
"Must supply fringe exposure as a pipeBase.Struct")
defects = [] if defects is None else defects
ccd = ccdExposure.getDetector()
ccdExposure = self.convertIntToFloat(ccdExposure)
if not ccd:
assert not self.config.doAssembleCcd, "You need a Detector to run assembleCcd"
ccd = [FakeAmp(ccdExposure, self.config)]
for amp in ccd:
#if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
if ccdExposure.getBBox().contains(amp.getBBox()):
self.saturationDetection(ccdExposure, amp)
self.overscanCorrection(ccdExposure, amp)
if self.config.doAssembleCcd:
ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
if self.config.doBias:
self.biasCorrection(ccdExposure, bias)
if self.config.doDark:
self.darkCorrection(ccdExposure, dark)
for amp in ccd:
#if ccdExposure is one amp, check for coverage to prevent performing ops multiple times
if ccdExposure.getBBox().contains(amp.getBBox()):
ampExposure = ccdExposure.Factory(ccdExposure, amp.getBBox())
self.updateVariance(ampExposure, amp)
if self.config.doFringe and not self.config.fringeAfterFlat:
self.fringe.run(ccdExposure, **fringes.getDict())
if self.config.doFlat:
self.flatCorrection(ccdExposure, flat)
self.maskAndInterpDefect(ccdExposure, defects)
self.saturationInterpolation(ccdExposure)
self.maskAndInterpNan(ccdExposure)
if self.config.doFringe and self.config.fringeAfterFlat:
self.fringe.run(ccdExposure, **fringes.getDict())
if False: # RHL XXX This scalar/dict disagreement needs to get resolved between HSC and LSST (i.e. Paul and Lauren). Do not check me in
ccdExposure.getCalib().setFluxMag0(
self.config.fluxMag0T1 * ccdExposure.getCalib().getExptime())
frame = getDebugFrame(self._display, "postISRCCD")
if frame:
getDisplay(frame).mtv(ccdExposure)
return pipeBase.Struct(exposure=ccdExposure, )
