def testBin(self):
"""Test that we can bin images anisotropically"""
inImage = afwImage.ImageF(203, 131)
val = 1
inImage.set(val)
binX, binY = 2, 4
outImage = afwMath.binImage(inImage, binX, binY)
self.assertEqual(outImage.getWidth(), inImage.getWidth() // binX)
self.assertEqual(outImage.getHeight(), inImage.getHeight() // binY)
stats = afwMath.makeStatistics(outImage, afwMath.MAX | afwMath.MIN)
self.assertEqual(stats.getValue(afwMath.MIN), val)
self.assertEqual(stats.getValue(afwMath.MAX), val)
inImage.set(0)
subImg = inImage.Factory(
inImage, afwGeom.BoxI(afwGeom.PointI(4, 4), afwGeom.ExtentI(4, 8)),
afwImage.LOCAL)
subImg.set(100)
del subImg
outImage = afwMath.binImage(inImage, binX, binY)
if display:
ds9.mtv(inImage, frame=2, title="unbinned")
ds9.mtv(outImage, frame=3, title="binned %dx%d" % (binX, binY))
def testBin(self):
"""Test that we can bin images anisotropically"""
inImage = afwImage.ImageF(203, 131)
val = 1
inImage.set(val)
binX, binY = 2, 4
outImage = afwMath.binImage(inImage, binX, binY)
self.assertEqual(outImage.getWidth(), inImage.getWidth()//binX)
self.assertEqual(outImage.getHeight(), inImage.getHeight()//binY)
stats = afwMath.makeStatistics(outImage, afwMath.MAX | afwMath.MIN)
self.assertEqual(stats.getValue(afwMath.MIN), val)
self.assertEqual(stats.getValue(afwMath.MAX), val)
inImage.set(0)
subImg = inImage.Factory(inImage, afwGeom.BoxI(afwGeom.PointI(4, 4), afwGeom.ExtentI(4, 8)),
afwImage.LOCAL)
subImg.set(100)
del subImg
outImage = afwMath.binImage(inImage, binX, binY)
if display:
ds9.mtv(inImage, frame=2, title="unbinned")
ds9.mtv(outImage, frame=3, title="binned %dx%d" % (binX, binY))
def getCcdImage(self, ccd, imageFactory=afwImage.ImageF, binSize=1):
bbox = ccd.getBBox()
try:
photoCalib = self.photoCalib[ccd.getId()]
except KeyError:
result = imageFactory(bbox)
return afwMath.binImage(result, binSize), ccd
tempImage = afwImage.ExposureF(bbox)
tempImage.image.array[:, :] = 1.0
result = afwImage.ImageF(tempImage.image, deep=True)
photoCalib.computeScaledZeroPoint().divideImage(result, xStep=100, yStep=16)
assert type(result) == imageFactory
return afwMath.binImage(result, binSize), ccd
def run(self, inputExp, camera):
"""Bin input image, attach associated detector.
Parameters
----------
inputExp : `lsst.afw.image.Exposure`
Input exposure data to bin.
camera : `lsst.afw.cameraGeom.Camera`
Input camera to use for mosaic geometry.
Returns
-------
output : `lsst.pipe.base.Struct`
Results struct with attribute:
``outputExp``
Binned version of input image (`lsst.afw.image.Exposure`).
"""
if inputExp.getDetector() is None:
detectorId = inputExp.getMetadata().get(
self.config.detectorKeyword)
if detectorId is not None:
inputExp.setDetector(camera[detectorId])
binned = inputExp.getMaskedImage()
binned = afwMath.binImage(binned, self.config.binning)
outputExp = afwImage.makeExposure(binned)
outputExp.setInfo(inputExp.getInfo())
return pipeBase.Struct(outputExp=outputExp, )
def writeThumbnail(self, dataRef, dataset, exposure):
"""Write out exposure to a snapshot file named outfile in the given size.
"""
filename = dataRef.get(dataset + "_filename")[0]
directory = os.path.dirname(filename)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
# Don't fail if directory exists due to race
if e.errno != errno.EEXIST:
raise e
binning = self.config.thumbnailBinning
binnedImage = afwMath.binImage(exposure.getMaskedImage(), binning, binning, afwMath.MEAN)
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setAndMask(binnedImage.getMask().getPlaneBitMask(["SAT", "BAD", "INTRP"]))
stats = afwMath.makeStatistics(binnedImage,
afwMath.MEDIAN | afwMath.STDEVCLIP | afwMath.MAX, statsCtrl)
low = stats.getValue(afwMath.MEDIAN) - self.config.thumbnailStdev*stats.getValue(afwMath.STDEVCLIP)
try:
makeRGB(binnedImage, binnedImage, binnedImage, minimum=low, dataRange=self.config.thumbnailRange,
Q=self.config.thumbnailQ, fileName=filename,
saturatedBorderWidth=self.config.thumbnailSatBorder,
saturatedPixelValue=stats.getValue(afwMath.MAX))
except Exception as exc:
# makeRGB can fail with:
# SystemError: <built-in method flush of _io.BufferedWriter object at 0x2b2a0081c938>
# returned a result with an error set
# This unhelpful error comes from the bowels of matplotlib, so not much we can do about it
# except keep it from being fatal.
self.log.warn("Unable to write thumbnail for %s: %s", dataRef.dataId, exc)
def writeThumbnail(self, dataRef, dataset, exposure):
"""Write out exposure to a snapshot file named outfile in the given size.
"""
filename = dataRef.get(dataset + "_filename")[0]
directory = os.path.dirname(filename)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
# Don't fail if directory exists due to race
if e.errno != errno.EEXIST:
raise e
binning = self.config.thumbnailBinning
binnedImage = afwMath.binImage(exposure.getMaskedImage(), binning,
binning, afwMath.MEAN)
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setAndMask(binnedImage.getMask().getPlaneBitMask(
["SAT", "BAD", "INTRP"]))
stats = afwMath.makeStatistics(
binnedImage, afwMath.MEDIAN | afwMath.STDEVCLIP | afwMath.MAX,
statsCtrl)
low = stats.getValue(
afwMath.MEDIAN) - self.config.thumbnailStdev * stats.getValue(
afwMath.STDEVCLIP)
makeRGB(binnedImage,
binnedImage,
binnedImage,
minimum=low,
dataRange=self.config.thumbnailRange,
Q=self.config.thumbnailQ,
fileName=filename,
saturatedBorderWidth=self.config.thumbnailSatBorder,
saturatedPixelValue=stats.getValue(afwMath.MAX))
def getImage(self, ccd, amp=None, imageFactory=None):
"""Return an image of dlnI/dr"""
ccdNum = ccd.getId().getSerial()
try:
if self.kwargs.get("ccd") is not None and not ccdNum in self.kwargs.get("ccd"):
raise RuntimeError
dataId = self.kwargs
if dataId.has_key("ccd"):
dataId = self.kwargs.copy()
del dataId["ccd"]
raw = cgUtils.trimExposure(self.butler.get("raw", ccd=ccdNum, immediate=True, **dataId).convertF(),
subtractBias=True, rotate=True)
if False:
msk = raw.getMaskedImage().getMask()
BAD=msk.getPlaneBitMask("BAD")
msk |= BAD
msk[15:-15, 0:-20] &= ~BAD
except Exception, e:
if self.verbose and str(e):
print e
ccdImage = afwImage.ImageF(*ccd.getAllPixels(self.isTrimmed).getDimensions())
if self.bin:
ccdImage = afwMath.binImage(ccdImage, self.bin)
return ccdImage
def getImage(self, ccd, amp=None, imageFactory=None):
"""Return an image of dlnI/dr"""
ccdNum = ccd.getId().getSerial()
try:
if self.kwargs.get("ccd") is not None and ccdNum not in self.kwargs.get("ccd"):
raise RuntimeError
dataId = self.kwargs
if "ccd" in dataId:
dataId = self.kwargs.copy()
del dataId["ccd"]
raw = cgUtils.trimExposure(self.butler.get("raw", ccd=ccdNum, immediate=True,
**dataId).convertF(),
subtractBias=True, rotate=True)
if False:
msk = raw.getMaskedImage().getMask()
BAD = msk.getPlaneBitMask("BAD")
msk |= BAD
msk[15:-15, 0:-20] &= ~BAD
except Exception as e:
if self.verbose and str(e):
print(e)
ccdImage = afwImage.ImageF(*ccd.getAllPixels(self.isTrimmed).getDimensions())
if self.bin:
ccdImage = afwMath.binImage(ccdImage, self.bin)
return ccdImage
return fitPatches(raw, bin=self.bin, returnResidualImage=False)[2]
def _prepareImage(self, ccd, im, binSize, allowRotate=True):
if binSize > 1:
im = afwMath.binImage(im, binSize)
if allowRotate:
im = afwMath.rotateImageBy90(im, ccd.getOrientation().getNQuarter())
return im
def _prepareImage(self, ccd, im, binSize, allowRotate=True):
if binSize > 1:
im = afwMath.binImage(im, binSize)
if allowRotate:
im = afwMath.rotateImageBy90(im,
ccd.getOrientation().getNQuarter())
return im
def makeImageFromCcd(ccd, imageSource=SynthesizeCcdImage(), amp=None,
isTrimmed=None, imageFactory=afwImage.ImageU, bin=1, natural=False):
"""Make an Image of a Ccd (or just a single amp)
If natural is True, return the CCD image without worrying about whether it's rotated when
placed into the camera
"""
if isTrimmed is None:
isTrimmed = ccd.isTrimmed()
imageSource.setTrimmed(isTrimmed)
if amp:
ampImage = imageFactory(amp.getAllPixels(isTrimmed).getDimensions())
ampImage <<= imageSource.getImage(ccd, amp, imageFactory=imageFactory)
if bin > 1:
ampImage = afwMath.binImage(ampImage, bin)
return ampImage
#
# Start by building the image in "Natural" (non-rotated) orientation
# (unless it's 'raw', in which case it's just easier to prepareAmpData)
#
if imageSource.isRaw:
ccdImage = imageFactory(ccd.getAllPixelsNoRotation(isTrimmed))
for a in ccd:
im = ccdImage.Factory(ccdImage, a.getAllPixels(isTrimmed), afwImage.LOCAL)
im <<= a.prepareAmpData(imageSource.getImage(ccd, a, imageFactory=imageFactory))
else:
ccdImage = imageSource.getImage(ccd, imageFactory=imageFactory)
if bin > 1:
ccdImage = afwMath.binImage(ccdImage, bin)
#
# Now rotate to the as-installed orientation
#
if not natural and not imageSource.isRaw:
ccdImage = afwMath.rotateImageBy90(ccdImage, ccd.getOrientation().getNQuarter())
return ccdImage
def makeImageFromCcd(ccd, isTrimmed=True, showAmpGain=True, imageFactory=afwImage.ImageU, rcMarkSize=10,
binSize=1):
"""Make an Image of a CCD.
Parameters
----------
ccd : `lsst.afw.cameraGeom.Detector`
Detector to use in making the image.
isTrimmed : `bool`
Assemble a trimmed Detector image.
showAmpGain : `bool`
Use the per-amp gain to color the pixels in the image?
imageFactory : callable like `lsst.afw.image.Image`
Image type to generate.
rcMarkSize : `float`
Size of the mark to make in the amp images at the read corner.
binSize : `int`
Bin the image by this factor in both dimensions.
Returns
-------
image : `lsst.afw.image.Image`
Image of the Detector (type returned by ``imageFactory``).
"""
ampImages = []
index = 0
if isTrimmed:
bbox = ccd.getBBox()
else:
bbox = calcRawCcdBBox(ccd)
for amp in ccd:
if amp.getHasRawInfo():
if showAmpGain:
ampImages.append(makeImageFromAmp(
amp, imageFactory=imageFactory, markSize=rcMarkSize))
else:
ampImages.append(makeImageFromAmp(amp, imValue=(index + 1)*1000,
imageFactory=imageFactory, markSize=rcMarkSize))
index += 1
if len(ampImages) > 0:
ccdImage = imageFactory(bbox)
for ampImage, amp in zip(ampImages, ccd):
if isTrimmed:
assembleAmplifierImage(ccdImage, ampImage, amp)
else:
assembleAmplifierRawImage(ccdImage, ampImage, amp)
else:
if not isTrimmed:
raise RuntimeError(
"Cannot create untrimmed CCD without amps with raw information")
ccdImage = imageFactory(ccd.getBBox())
ccdImage = afwMath.binImage(ccdImage, binSize)
return ccdImage
def makeImageFromCcd(ccd, isTrimmed=True, showAmpGain=True, imageFactory=afwImage.ImageU, rcMarkSize=10,
binSize=1):
"""Make an Image of a CCD.
Parameters
----------
ccd : `lsst.afw.cameraGeom.Detector`
Detector to use in making the image.
isTrimmed : `bool`
Assemble a trimmed Detector image.
showAmpGain : `bool`
Use the per-amp gain to color the pixels in the image?
imageFactory : callable like `lsst.afw.image.Image`
Image type to generate.
rcMarkSize : `float`
Size of the mark to make in the amp images at the read corner.
binSize : `int`
Bin the image by this factor in both dimensions.
Returns
-------
image : `lsst.afw.image.Image`
Image of the Detector (type returned by ``imageFactory``).
"""
ampImages = []
index = 0
if isTrimmed:
bbox = ccd.getBBox()
else:
bbox = calcRawCcdBBox(ccd)
for amp in ccd:
if amp.getHasRawInfo():
if showAmpGain:
ampImages.append(makeImageFromAmp(
amp, imageFactory=imageFactory, markSize=rcMarkSize))
else:
ampImages.append(makeImageFromAmp(amp, imValue=(index + 1)*1000,
imageFactory=imageFactory, markSize=rcMarkSize))
index += 1
if len(ampImages) > 0:
ccdImage = imageFactory(bbox)
for ampImage, amp in zip(ampImages, ccd):
if isTrimmed:
assembleAmplifierImage(ccdImage, ampImage, amp)
else:
assembleAmplifierRawImage(ccdImage, ampImage, amp)
else:
if not isTrimmed:
raise RuntimeError(
"Cannot create untrimmed CCD without amps with raw information")
ccdImage = imageFactory(ccd.getBBox())
ccdImage = afwMath.binImage(ccdImage, binSize)
return ccdImage
def focalPlaneBackgroundRun(self, camera, cacheExposures, idList, config):
"""Perform full focal-plane background subtraction
This method runs on the master node.
Parameters
----------
camera : `lsst.afw.cameraGeom.Camera`
Camera description.
cacheExposures : `list` of `lsst.afw.image.Exposures`
List of loaded and processed input calExp.
idList : `list` of `int`
List of detector ids to iterate over.
config : `lsst.pipe.drivers.background.FocalPlaneBackgroundConfig`
Configuration to use for background subtraction.
Returns
-------
exposures : `list` of `lsst.afw.image.Image`
List of binned images, for creating focal plane image.
newCacheBgList : `list` of `lsst.afwMath.backgroundList`
Background lists generated.
cacheBgModel : `FocalPlaneBackground`
Full focal plane background model.
"""
bgModel = FocalPlaneBackground.fromCamera(config, camera)
data = [
pipeBase.Struct(id=id, bgModel=bgModel.clone()) for id in idList
]
bgModelList = []
for nodeData, cacheExp in zip(data, cacheExposures):
nodeData.bgModel.addCcd(cacheExp)
bgModelList.append(nodeData.bgModel)
for ii, bg in enumerate(bgModelList):
self.log.info("Background %d: %d pixels", ii,
bg._numbers.getArray().sum())
bgModel.merge(bg)
exposures = []
newCacheBgList = []
cacheBgModel = []
for cacheExp in cacheExposures:
nodeExp, nodeBgModel, nodeBgList = self.subtractModelRun(
cacheExp, bgModel)
exposures.append(
afwMath.binImage(nodeExp.getMaskedImage(),
self.config.binning))
cacheBgModel.append(nodeBgModel)
newCacheBgList.append(nodeBgList)
return exposures, newCacheBgList, cacheBgModel
def makeImageFromCcd(ccd,
isTrimmed=True,
showAmpGain=True,
imageFactory=afwImage.ImageU,
rcMarkSize=10,
binSize=1):
"""!Make an Image of a Ccd
@param[in] ccd Detector to use in making the image
@param[in] isTrimmed Assemble a trimmed Detector image if True
@param[in] showAmpGain Use the per amp gain to color the pixels in the image
@param[in] imageFactory Image type to generate
@param[in] rcMarkSize Size of the mark to make in the amp images at the read corner
@param[in] binSize Bin the image by this factor in both dimensions
@return Image of the Detector
"""
ampImages = []
index = 0
if isTrimmed:
bbox = ccd.getBBox()
else:
bbox = calcRawCcdBBox(ccd)
for amp in ccd:
if amp.getHasRawInfo():
if showAmpGain:
ampImages.append(
makeImageFromAmp(amp,
imageFactory=imageFactory,
markSize=rcMarkSize))
else:
ampImages.append(
makeImageFromAmp(amp,
imValue=(index + 1) * 1000,
imageFactory=imageFactory,
markSize=rcMarkSize))
index += 1
if len(ampImages) > 0:
ccdImage = imageFactory(bbox)
for ampImage, amp in zip(ampImages, ccd):
if isTrimmed:
assembleAmplifierImage(ccdImage, ampImage, amp)
else:
assembleAmplifierRawImage(ccdImage, ampImage, amp)
else:
if not isTrimmed:
raise RuntimeError(
"Cannot create untrimmed CCD without amps with raw information"
)
ccdImage = imageFactory(ccd.getBBox())
ccdImage = afwMath.binImage(ccdImage, binSize)
return ccdImage
def makeImageFromCcd(ccd, imageSource=SynthesizeCcdImage(), amp=None,
isTrimmed=None, imageFactory=afwImage.ImageU, bin=1,
natural=False, display=False):
"""Make an Image of a Ccd (or just a single amp)
If natural is True, return the CCD image without worrying about whether it's rotated when
placed into the camera
"""
if isTrimmed is None:
isTrimmed = ccd.isTrimmed()
imageSource.setTrimmed(isTrimmed)
if amp:
ampImage = imageFactory(amp.getAllPixels(isTrimmed).getDimensions())
ampImage <<= imageSource.getImage(ccd, amp, imageFactory=imageFactory)
if bin > 1:
ampImage = afwMath.binImage(ampImage, bin)
return ampImage
#
# If the image is raw it may need to be assembled into a full sensor. The Amp object knows
# the coordinates system in which the data is being stored on disk. Since all bounding box
# information is held in camera coordinates, there is no need to rotate the image after assembly.
#
if imageSource.isRaw:
ccdImage = imageFactory(ccd.getAllPixels(isTrimmed))
for a in ccd:
im = ccdImage.Factory(ccdImage, a.getAllPixels(isTrimmed), afwImage.LOCAL)
im <<= imageSource.getImage(ccd, a, imageFactory=imageFactory)
else:
ccdImage = imageSource.getImage(ccd, imageFactory=imageFactory)
if bin > 1:
ccdImage = afwMath.binImage(ccdImage, bin)
if display:
showCcd(ccd, ccdImage=ccdImage, isTrimmed=isTrimmed)
return ccdImage
def getCcdImage(self, ccd, imageFactory=afwImage.ImageF, binSize=1):
bbox = ccd.getBBox()
try:
ffp = self.ffp[ccd.getId()]
wcs = self.wcs[ccd.getId()]
except KeyError:
result = imageFactory(bbox)
return afwMath.binImage(result, binSize), ccd
nQuarter = ccd.getOrientation().getNQuarter()
# Rotate WCS from persisted LSST coords to meas_mosaic coords
if nQuarter % 4 != 0:
# Have to put this import here due to circular dependencies
import lsst.meas.astrom as measAstrom
wcs = measAstrom.rotateWcsPixelsBy90(wcs, nQuarter,
bbox.getDimensions())
if nQuarter % 2:
width, height = bbox.getHeight(), bbox.getWidth()
else:
width, height = bbox.getWidth(), bbox.getHeight()
if self.fcor:
result = getFCorImg(ffp, width, height)
if self.jacobian:
result *= getJImg(wcs, width, height)
elif self.jacobian:
result = getJImg(wcs, width, height)
else:
result = imageFactory(bbox)
return afwMath.binImage(result, binSize), ccd
# Rotate images to LSST coords
if nQuarter % 4 != 0:
result = afwMath.rotateImageBy90(result, 4 - nQuarter)
result.setXY0(bbox.getMin())
assert bbox == result.getBBox(), "%s != %s" % (bbox, result.getBBox())
assert type(result) == imageFactory
return afwMath.binImage(result, binSize), ccd
def getCcdImage(self, ccd, imageFactory=afwImage.ImageF, binSize=1):
bbox = ccd.getBBox()
try:
ffp = self.ffp[ccd.getId()]
wcs = self.wcs[ccd.getId()]
except KeyError:
result = imageFactory(bbox)
return afwMath.binImage(result, binSize), ccd
nQuarter = ccd.getOrientation().getNQuarter()
# Rotate WCS from persisted LSST coords to meas_mosaic coords
if nQuarter%4 != 0:
# Have to put this import here due to circular dependencies
import lsst.meas.astrom as measAstrom
wcs = measAstrom.rotateWcsPixelsBy90(wcs, nQuarter, bbox.getDimensions())
if nQuarter%2:
width, height = bbox.getHeight(), bbox.getWidth()
else:
width, height = bbox.getWidth(), bbox.getHeight()
if self.fcor:
result = getFCorImg(ffp, width, height)
if self.jacobian:
result *= getJImg(wcs, width, height)
elif self.jacobian:
result = getJImg(wcs, width, height)
else:
result = imageFactory(bbox)
return afwMath.binImage(result, binSize), ccd
# Rotate images to LSST coords
if nQuarter%4 != 0:
result = afwMath.rotateImageBy90(result, 4 - nQuarter)
result.setXY0(bbox.getMin())
assert bbox == result.getBBox(), "%s != %s" % (bbox, result.getBBox())
assert type(result) == imageFactory
return afwMath.binImage(result, binSize), ccd
def makeThumbnail(exposure, isrQaConfig=None):
"""Create a snapshot thumbnail from input exposure.
The output thumbnail image is constructed based on the parameters
in the configuration file. Currently, the asinh mapping is the
only mapping method used.
Parameters
----------
exposure : `lsst.afw.image.Exposure`
The exposure to be converted into a thumbnail.
isrQaConfig : `Config`
Configuration object containing all parameters to control the
thumbnail generation.
Returns
-------
rgbImage : `numpy.ndarray`
Binned and scaled version of the exposure, converted to an
integer array to allow it to be written as PNG.
"""
if isrQaConfig is not None:
binning = isrQaConfig.thumbnailBinning
binnedImage = afwMath.binImage(exposure.getMaskedImage(), binning,
binning, afwMath.MEAN)
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setAndMask(binnedImage.getMask().getPlaneBitMask(
["SAT", "BAD", "INTRP"]))
stats = afwMath.makeStatistics(
binnedImage, afwMath.MEDIAN | afwMath.STDEVCLIP | afwMath.MAX,
statsCtrl)
low = stats.getValue(
afwMath.MEDIAN) - isrQaConfig.thumbnailStdev * stats.getValue(
afwMath.STDEVCLIP)
if isrQaConfig.thumbnailSatBorder:
afwRGB.replaceSaturatedPixels(binnedImage, binnedImage,
binnedImage,
isrQaConfig.thumbnailSatBorder,
stats.getValue(afwMath.MAX))
asinhMap = afwRGB.AsinhMapping(low,
isrQaConfig.thumbnailRange,
Q=isrQaConfig.thumbnailQ)
rgbImage = asinhMap.makeRgbImage(binnedImage)
return rgbImage
def testBin(self):
"""Test that we can bin images"""
inImage = afwImage.ImageF(203, 131)
inImage.set(1)
bin = 4
outImage = afwMath.binImage(inImage, bin)
self.assertEqual(outImage.getWidth(), inImage.getWidth() // bin)
self.assertEqual(outImage.getHeight(), inImage.getHeight() // bin)
stats = afwMath.makeStatistics(outImage, afwMath.MAX | afwMath.MIN)
self.assertEqual(stats.getValue(afwMath.MIN), 1)
self.assertEqual(stats.getValue(afwMath.MAX), 1)
def testBin(self):
"""Test that we can bin images"""
inImage = afwImage.ImageF(afwGeom.Extent2I(203, 131))
inImage.set(1)
bin = 4
outImage = afwMath.binImage(inImage, bin)
self.assertEqual(outImage.getWidth(), inImage.getWidth()//bin)
self.assertEqual(outImage.getHeight(), inImage.getHeight()//bin)
stats = afwMath.makeStatistics(outImage, afwMath.MAX | afwMath.MIN)
self.assertEqual(stats.getValue(afwMath.MIN), 1)
self.assertEqual(stats.getValue(afwMath.MAX), 1)
def _makeCalibrationImage(self,
psfSigma,
width,
kernelWidth=None,
kernelSigma=None):
"""Make a fake satellite trail with the PSF to calibrate moments we measure.
@param psfSigma Gaussian sigma for a double-Gaussian PSF model (in pixels)
@param width Width of the trail in pixels (0.0 for PSF alone, but wider for aircraft trails)
@param kernelWidth
@param kernelSigma
@return calImg An afwImage containing a fake satellite/aircraft trail
"""
kernelWidth = kernelWidth or self.kernelWidth
kernelSigma = kernelSigma or self.kernelSigma
# tricky. We have to make a fake trail so it's just like one in the real image
# To get the binning right, we start with an image 'bins'-times too big
cx, cy = (self.bins * kernelWidth) // 2 - 0.5, 0
calImg = afwImage.ImageF(self.bins * kernelWidth,
self.bins * kernelWidth)
calArr = calImg.getArray()
# Make a trail with the requested (unbinned) width
calTrail = satTrail.SatelliteTrail(cx, cy)
# for wide trails, just add a constant with the stated width
if width > 8.0 * psfSigma:
profile = satTrail.ConstantProfile(1.0, width)
insertWidth = width
# otherwise, use a double gaussian
else:
profile = satTrail.DoubleGaussianProfile(1.0,
width / 2.0 + psfSigma)
insertWidth = 4.0 * (width / 2.0 + psfSigma)
calTrail.insert(calArr, profile, insertWidth)
# Now bin and smooth, just as we did the real image
calArr = afwMath.binImage(calImg, self.bins).getArray()
calArr = satUtil.smooth(calArr, self.sigmaSmooth)
return calArr
def collectBinnedImage(self, exposure, image):
"""Return the binned image required for visualization
This method just helps to cut down on boilerplate.
Parameters
----------
image : `lsst.afw.image.MaskedImage`
Image to go into visualisation.
Returns
-------
detId : `int`
Detector identifier.
image : `lsst.afw.image.MaskedImage`
Binned image.
"""
return (exposure.getDetector().getId(), afwMath.binImage(image, self.config.binning))
def makeImageFromCcd(ccd, isTrimmed=True, showAmpGain=True, imageFactory=afwImage.ImageU, rcMarkSize=10,
binSize=1):
"""!Make an Image of a Ccd
@param[in] ccd Detector to use in making the image
@param[in] isTrimmed Assemble a trimmed Detector image if True
@param[in] showAmpGain Use the per amp gain to color the pixels in the image
@param[in] imageFactory Image type to generate
@param[in] rcMarkSize Size of the mark to make in the amp images at the read corner
@param[in] binSize Bin the image by this factor in both dimensions
@return Image of the Detector
"""
ampImages = []
index = 0
if isTrimmed:
bbox = ccd.getBBox()
else:
bbox = calcRawCcdBBox(ccd)
for amp in ccd:
if amp.getHasRawInfo():
if showAmpGain:
ampImages.append(makeImageFromAmp(amp, imageFactory=imageFactory, markSize=rcMarkSize))
else:
ampImages.append(makeImageFromAmp(amp, imValue=(index+1)*1000,
imageFactory=imageFactory, markSize=rcMarkSize))
index += 1
if len(ampImages) > 0:
ccdImage = imageFactory(bbox)
for ampImage, amp in itertools.izip(ampImages, ccd):
if isTrimmed:
assembleAmplifierImage(ccdImage, ampImage, amp)
else:
assembleAmplifierRawImage(ccdImage, ampImage, amp)
else:
if not isTrimmed:
raise RuntimeError("Cannot create untrimmed CCD without amps with raw information")
ccdImage = imageFactory(ccd.getBBox())
ccdImage = afwMath.binImage(ccdImage, binSize)
return ccdImage
def _makeCalibrationImage(self, psfSigma, width, kernelWidth=None, kernelSigma=None):
"""Make a fake satellite trail with the PSF to calibrate moments we measure.
@param psfSigma Gaussian sigma for a double-Gaussian PSF model (in pixels)
@param width Width of the trail in pixels (0.0 for PSF alone, but wider for aircraft trails)
@param kernelWidth
@param kernelSigma
@return calImg An afwImage containing a fake satellite/aircraft trail
"""
kernelWidth = kernelWidth or self.kernelWidth
kernelSigma = kernelSigma or self.kernelSigma
# tricky. We have to make a fake trail so it's just like one in the real image
# To get the binning right, we start with an image 'bins'-times too big
cx, cy = (self.bins*kernelWidth)//2 - 0.5, 0
calImg = afwImage.ImageF(self.bins*kernelWidth, self.bins*kernelWidth)
calArr = calImg.getArray()
# Make a trail with the requested (unbinned) width
calTrail = satTrail.SatelliteTrail(cx, cy)
# for wide trails, just add a constant with the stated width
if width > 8.0*psfSigma:
profile = satTrail.ConstantProfile(1.0, width)
insertWidth = width
# otherwise, use a double gaussian
else:
profile = satTrail.DoubleGaussianProfile(1.0, width/2.0 + psfSigma)
insertWidth = 4.0*(width/2.0 + psfSigma)
calTrail.insert(calArr, profile, insertWidth)
# Now bin and smooth, just as we did the real image
calArr = afwMath.binImage(calImg, self.bins).getArray()
calArr = satUtil.smooth(calArr, self.sigmaSmooth)
return calArr
def readImage(self, cache, dataId):
"""Collect original image for visualisation
This method runs on the slave nodes.
Parameters
----------
cache : `lsst.pipe.base.Struct`
Process pool cache.
dataId : `dict`
Data identifier.
Returns
-------
detId : `int`
Detector identifier.
image : `lsst.afw.image.MaskedImage`
Binned image.
"""
exposure = cache.butler.get("calexp", dataId)
return (exposure.getDetector().getId(),
afwMath.binImage(exposure.getMaskedImage(),
self.config.binning))
def measureMeanVarCov(self, exposure1, exposure2, region=None, covAstierRealSpace=False):
"""Calculate the mean of each of two exposures and the variance
and covariance of their difference. The variance is calculated
via afwMath, and the covariance via the methods in Astier+19
(appendix A). In theory, var = covariance[0,0]. This should
be validated, and in the future, we may decide to just keep
one (covariance).
Parameters
----------
exposure1 : `lsst.afw.image.exposure.exposure.ExposureF`
First exposure of flat field pair.
exposure2 : `lsst.afw.image.exposure.exposure.ExposureF`
Second exposure of flat field pair.
region : `lsst.geom.Box2I`, optional
Region of each exposure where to perform the calculations (e.g, an amplifier).
covAstierRealSpace : `bool`, optional
Should the covariannces in Astier+19 be calculated in real space or via FFT?
See Appendix A of Astier+19.
Returns
-------
mu : `float` or `NaN`
0.5*(mu1 + mu2), where mu1, and mu2 are the clipped means of the regions in
both exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
varDiff : `float` or `NaN`
Half of the clipped variance of the difference of the regions inthe two input
exposures. If either mu1 or m2 are NaN's, the returned value is NaN.
covDiffAstier : `list` or `NaN`
List with tuples of the form (dx, dy, var, cov, npix), where:
dx : `int`
Lag in x
dy : `int`
Lag in y
var : `float`
Variance at (dx, dy).
cov : `float`
Covariance at (dx, dy).
nPix : `int`
Number of pixel pairs used to evaluate var and cov.
If either mu1 or m2 are NaN's, the returned value is NaN.
"""
if region is not None:
im1Area = exposure1.maskedImage[region]
im2Area = exposure2.maskedImage[region]
else:
im1Area = exposure1.maskedImage
im2Area = exposure2.maskedImage
if self.config.binSize > 1:
im1Area = afwMath.binImage(im1Area, self.config.binSize)
im2Area = afwMath.binImage(im2Area, self.config.binSize)
im1MaskVal = exposure1.getMask().getPlaneBitMask(self.config.maskNameList)
im1StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
self.config.nIterSigmaClipPtc,
im1MaskVal)
im1StatsCtrl.setNanSafe(True)
im1StatsCtrl.setAndMask(im1MaskVal)
im2MaskVal = exposure2.getMask().getPlaneBitMask(self.config.maskNameList)
im2StatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
self.config.nIterSigmaClipPtc,
im2MaskVal)
im2StatsCtrl.setNanSafe(True)
im2StatsCtrl.setAndMask(im2MaskVal)
# Clipped mean of images; then average of mean.
mu1 = afwMath.makeStatistics(im1Area, afwMath.MEANCLIP, im1StatsCtrl).getValue()
mu2 = afwMath.makeStatistics(im2Area, afwMath.MEANCLIP, im2StatsCtrl).getValue()
if np.isnan(mu1) or np.isnan(mu2):
self.log.warn(f"Mean of amp in image 1 or 2 is NaN: {mu1}, {mu2}.")
return np.nan, np.nan, None
mu = 0.5*(mu1 + mu2)
# Take difference of pairs
# symmetric formula: diff = (mu2*im1-mu1*im2)/(0.5*(mu1+mu2))
temp = im2Area.clone()
temp *= mu1
diffIm = im1Area.clone()
diffIm *= mu2
diffIm -= temp
diffIm /= mu
diffImMaskVal = diffIm.getMask().getPlaneBitMask(self.config.maskNameList)
diffImStatsCtrl = afwMath.StatisticsControl(self.config.nSigmaClipPtc,
self.config.nIterSigmaClipPtc,
diffImMaskVal)
diffImStatsCtrl.setNanSafe(True)
diffImStatsCtrl.setAndMask(diffImMaskVal)
varDiff = 0.5*(afwMath.makeStatistics(diffIm, afwMath.VARIANCECLIP, diffImStatsCtrl).getValue())
# Get the mask and identify good pixels as '1', and the rest as '0'.
w1 = np.where(im1Area.getMask().getArray() == 0, 1, 0)
w2 = np.where(im2Area.getMask().getArray() == 0, 1, 0)
w12 = w1*w2
wDiff = np.where(diffIm.getMask().getArray() == 0, 1, 0)
w = w12*wDiff
if np.sum(w) < self.config.minNumberGoodPixelsForFft:
self.log.warn(f"Number of good points for FFT ({np.sum(w)}) is less than threshold "
f"({self.config.minNumberGoodPixelsForFft})")
return np.nan, np.nan, None
maxRangeCov = self.config.maximumRangeCovariancesAstier
if covAstierRealSpace:
covDiffAstier = computeCovDirect(diffIm.image.array, w, maxRangeCov)
else:
shapeDiff = diffIm.image.array.shape
fftShape = (fftSize(shapeDiff[0] + maxRangeCov), fftSize(shapeDiff[1]+maxRangeCov))
c = CovFft(diffIm.image.array, w, fftShape, maxRangeCov)
covDiffAstier = c.reportCovFft(maxRangeCov)
return mu, varDiff, covDiffAstier
def run(self, calExpArray, calBkgArray, skyCalibs, camera):
"""Duplicate runDataRef method without ctrl_pool for Gen3.
Parameters
----------
calExpArray : `list` of `lsst.afw.image.Exposure`
Array of detector input calExp images for the exposure to
process.
calBkgArray : `list` of `lsst.afw.math.BackgroundList`
Array of detector input background lists matching the
calExps to process.
skyCalibs : `list` of `lsst.afw.image.Exposure`
Array of SKY calibrations for the input detectors to be
processed.
camera : `lsst.afw.cameraGeom.Camera`
Camera matching the input data to process.
Returns
-------
results : `pipeBase.Struct` containing
calExpCamera : `lsst.afw.image.Exposure`
Full camera image of the sky-corrected data.
skyCorr : `list` of `lsst.afw.math.BackgroundList`
Detector-level sky-corrected background lists.
See Also
--------
~lsst.pipe.drivers.SkyCorrectionTask.runDataRef()
"""
# To allow SkyCorrectionTask to run in the Gen3 butler
# environment, a new run() method was added that performs the
# same operations in a serial environment (pipetask processing
# does not support MPI processing as of 2019-05-03). Methods
# used in runDataRef() are used as appropriate in run(), but
# some have been rewritten in serial form. Please ensure that
# any updates to runDataRef() or the methods it calls with
# pool.mapToPrevious() are duplicated in run() and its
# methods.
#
# Variable names here should match those in runDataRef() as
# closely as possible. Variables matching data stored in the
# pool cache have a prefix indicating this. Variables that
# would be local to an MPI processing client have a prefix
# "node".
idList = [exp.getDetector().getId() for exp in calExpArray]
# Construct arrays that match the cache in self.runDataRef() after
# self.loadImage() is map/reduced.
cacheExposures = []
cacheBgList = []
exposures = []
for calExp, calBgModel in zip(calExpArray, calBkgArray):
nodeExp, nodeBgList = self.loadImageRun(calExp, calBgModel)
cacheExposures.append(nodeExp)
cacheBgList.append(nodeBgList)
exposures.append(afwMath.binImage(nodeExp.getMaskedImage(), self.config.binning))
if self.config.doBgModel:
# Generate focal plane background, updating backgrounds in the "cache".
exposures, newCacheBgList, cacheBgModel = self.focalPlaneBackgroundRun(
camera, cacheExposures, idList, self.config.bgModel
)
for cacheBg, newBg in zip(cacheBgList, newCacheBgList):
cacheBg.append(newBg)
if self.config.doSky:
# Measure the sky frame scale on all inputs. Results in
# values equal to self.measureSkyFrame() and
# self.sky.solveScales() in runDataRef().
cacheSky = []
measScales = []
for cacheExp, skyCalib in zip(cacheExposures, skyCalibs):
skyExp = self.sky.exposureToBackground(skyCalib)
cacheSky.append(skyExp)
scale = self.sky.measureScale(cacheExp.getMaskedImage(), skyExp)
measScales.append(scale)
scale = self.sky.solveScales(measScales)
self.log.info("Sky frame scale: %s" % (scale, ))
# Subtract sky frame, as in self.subtractSkyFrame(), with
# appropriate scale from the "cache".
exposures = []
newBgList = []
for cacheExp, nodeSky, nodeBgList in zip(cacheExposures, cacheSky, cacheBgList):
self.sky.subtractSkyFrame(cacheExp.getMaskedImage(), nodeSky, scale, nodeBgList)
exposures.append(afwMath.binImage(cacheExp.getMaskedImage(), self.config.binning))
if self.config.doBgModel2:
# As above, generate a focal plane background model and
# update the cache models.
exposures, newBgList, cacheBgModel = self.focalPlaneBackgroundRun(
camera, cacheExposures, idList, self.config.bgModel2
)
for cacheBg, newBg in zip(cacheBgList, newBgList):
cacheBg.append(newBg)
# Generate camera-level image of calexp and return it along
# with the list of sky corrected background models.
image = makeCameraImage(camera, zip(idList, exposures))
return pipeBase.Struct(
calExpCamera=image,
skyCorr=cacheBgList,
)
def getTrails(self, exposure, widths):
"""Detect satellite trails in exposure using provided widths
@param exposure The exposure to detect in
@param widths A list of widths [pixels] to use for calibration trails.
@return trails A SatelliteTrailList object containing detected trails.
"""
emsk = exposure.getMaskedImage().getMask()
DET = emsk.getPlaneBitMask("DETECTED")
MASK = 0
for plane in "BAD", "CR", "SAT", "INTRP", "EDGE", "SUSPECT":
MASK |= emsk.getPlaneBitMask(plane)
t1 = time.time()
#################################################
# Main detection image
#################################################
if self.bins == 1:
exp = exposure.clone()
else:
exp = type(exposure)(afwMath.binImage(exposure.getMaskedImage(),
self.bins))
exp.setMetadata(exposure.getMetadata())
exp.setPsf(exposure.getPsf())
img = exp.getMaskedImage().getImage().getArray()
msk = exp.getMaskedImage().getMask().getArray()
isBad = msk & MASK > 0
isGood = ~isBad
img[isBad] = 0.0 # convolution will smear bad pixels. Zero them out.
psfSigma = satUtil.getExposurePsfSigma(exposure, minor=True)
#################################################
# Faint-trail detection image
#################################################
# construct a specially clipped image to use to model a fine scale background
# - zero-out the detected pixels (in addition to otherwise bad pixels)
expClip = exp.clone()
mskClip = expClip.getMaskedImage().getMask().getArray()
imgClip = expClip.getMaskedImage().getImage().getArray()
imgClip[(mskClip & (MASK | DET) > 0)] = 0.0
# scale the detected pixels
if np.abs(self.scaleDetected - 1.0) > 1.0e-6:
self.log.logdebug("Scaling detected")
wDet = msk & DET > 0
sig = imgClip.std()
wSig = img > 2.0 * sig
# amplify detected pixels (make this configurable?)
img[wDet | wSig] *= self.scaleDetected
# subtract a small scale background when we search for PSFs
if self.doBackground:
self.log.logdebug("Median ring background")
back = satUtil.medianRing(imgClip, self.kernelWidth,
2.0 * self.sigmaSmooth)
img -= back
imgClip -= back
# - smooth
img = satUtil.smooth(img, self.sigmaSmooth)
imgClip = satUtil.smooth(imgClip, self.sigmaSmooth)
rms = imgClip[(mskClip & (MASK | DET) == 0)].std()
isCandidate = np.ones(img.shape, dtype=bool)
###########################################
# Try different kernel sizes
###########################################
# Different sized kernels should give the same results for a real trail
# but would be less likely to for noise.
# Unfortunately, this is costly, and the effect is small.
for kernelFactor in (1.0, self.growKernel):
self.log.logdebug("Getting moments growKernel=%.1f" %
(self.growKernel))
kernelWidth = 2 * int((kernelFactor * self.kernelWidth) // 2) + 1
kernelSigma = kernelFactor * self.kernelSigma
isKernelCandidate = np.zeros(img.shape, dtype=bool)
#################################################
# Calibration images
#################################################
calImages = []
for width in widths:
calImages.append(
self._makeCalibrationImage(psfSigma,
width,
kernelWidth=kernelWidth,
kernelSigma=kernelSigma))
################################################
# Moments
#################################################
mm = momCalc.MomentManager(img,
kernelWidth=kernelWidth,
kernelSigma=kernelSigma)
mmCals = []
nHits = []
#Selector = momCalc.PixelSelector
Selector = momCalc.PValuePixelSelector
maxPixels = 4000
for i, calImg in enumerate(calImages):
mmCal = momCalc.MomentManager(calImg,
kernelWidth=kernelWidth,
kernelSigma=kernelSigma,
isCalibration=True)
mmCals.append(mmCal)
sumI = momCalc.MomentLimit('sumI', self.luminosityLimit * rms,
'lower')
cent = momCalc.MomentLimit('center', 2.0 * self.centerLimit,
'center')
centP = momCalc.MomentLimit('center_perp', self.centerLimit,
'center')
skew = momCalc.MomentLimit('skew', 2.0 * self.skewLimit,
'center')
skewP = momCalc.MomentLimit('skew_perp', self.skewLimit,
'center')
ellip = momCalc.MomentLimit('ellip', self.eRange, 'center')
b = momCalc.MomentLimit('b', self.bLimit, 'center')
selector = Selector(mm, mmCal)
for limit in sumI, ellip, cent, centP, skew, skewP, b:
selector.append(limit)
pixels = selector.getPixels(maxPixels=maxPixels)
isKernelCandidate |= pixels
msg = "cand: nPix: %d tot: %d" % (pixels.sum(),
isKernelCandidate.sum())
self.log.logdebug(msg)
isCandidate &= isKernelCandidate
self.log.logdebug("total: %d" % (isCandidate.sum()))
nHits.append((widths[i], isKernelCandidate.sum()))
bestCal = sorted(nHits, key=lambda x: x[1], reverse=True)[0]
bestWidth = bestCal[0]
###############################################
# Theta Alignment
###############################################
self.log.logdebug("Theta alignment.")
xx, yy = np.meshgrid(np.arange(img.shape[1], dtype=int),
np.arange(img.shape[0], dtype=int))
nBeforeAlignment = isCandidate.sum()
maxSeparation = min([x / 2 for x in img.shape])
thetaMatch, newTheta = hough.thetaAlignment(
mm.theta[isCandidate],
xx[isCandidate],
yy[isCandidate],
tolerance=self.thetaTolerance,
limit=3,
maxSeparation=maxSeparation)
mm.theta[isCandidate] = newTheta
isCandidate[isCandidate] = thetaMatch
nAfterAlignment = isCandidate.sum()
self.log.logdebug("theta-alignment Bef/aft: %d / %d" %
(nBeforeAlignment, nAfterAlignment))
#################################################
# Hough transform
#################################################
self.log.logdebug("Hough Transform.")
rMax = np.linalg.norm(img.shape)
houghTransform = hough.HoughTransform(self.houghBins,
self.houghThresh,
rMax=rMax,
maxPoints=1000,
nIter=1,
maxResid=5.5,
log=self.log)
solutions = houghTransform(mm.theta[isCandidate], xx[isCandidate],
yy[isCandidate])
#################################################
# Construct Trail objects from Hough solutions
#################################################
self.log.logdebug("Constructing SatelliteTrail objects.")
trails = satTrail.SatelliteTrailList(nAfterAlignment, solutions.binMax,
psfSigma)
for s in solutions:
trail = satTrail.SatelliteTrail.fromHoughSolution(s, self.bins)
trail.detectWidth = bestWidth
trail.maskAndBits = self.maskAndBits
trail.measure(exp, bins=self.bins)
# last chance to drop it
if trail.width < self.maxTrailWidth:
nx, ny = exposure.getWidth(), exposure.getHeight()
self.log.info(str(trail) + "[%dpix]" % (trail.length(nx, ny)))
trails.append(trail)
else:
self.log.info("Dropping (maxWidth>%.1f): %s" %
(self.maxTrailWidth, trail))
# A bit hackish, but stash some useful info for diagnostic plots
self._mm = mm
self._mmCals = mmCals
self._isCandidate = isCandidate
self._brightFactor = 10
self._trails = trails
self._solutions = solutions
return trails
def getTrails(self, exposure, widths):
"""Detect satellite trails in exposure using provided widths
@param exposure The exposure to detect in
@param widths A list of widths [pixels] to use for calibration trails.
@return trails A SatelliteTrailList object containing detected trails.
"""
emsk = exposure.getMaskedImage().getMask()
DET = emsk.getPlaneBitMask("DETECTED")
MASK = 0
for plane in "BAD", "CR", "SAT", "INTRP", "EDGE", "SUSPECT":
MASK |= emsk.getPlaneBitMask(plane)
t1 = time.time()
#################################################
# Main detection image
#################################################
if self.bins == 1:
exp = exposure.clone()
else:
exp = type(exposure)(afwMath.binImage(exposure.getMaskedImage(), self.bins))
exp.setMetadata(exposure.getMetadata())
exp.setPsf(exposure.getPsf())
img = exp.getMaskedImage().getImage().getArray()
msk = exp.getMaskedImage().getMask().getArray()
isBad = msk & MASK > 0
isGood = ~isBad
img[isBad] = 0.0 # convolution will smear bad pixels. Zero them out.
psfSigma = satUtil.getExposurePsfSigma(exposure, minor=True)
#################################################
# Faint-trail detection image
#################################################
# construct a specially clipped image to use to model a fine scale background
# - zero-out the detected pixels (in addition to otherwise bad pixels)
expClip = exp.clone()
mskClip = expClip.getMaskedImage().getMask().getArray()
imgClip = expClip.getMaskedImage().getImage().getArray()
imgClip[(mskClip & (MASK | DET) > 0)] = 0.0
# scale the detected pixels
if np.abs(self.scaleDetected - 1.0) > 1.0e-6:
self.log.logdebug("Scaling detected")
wDet = msk & DET > 0
sig = imgClip.std()
wSig = img > 2.0*sig
# amplify detected pixels (make this configurable?)
img[wDet|wSig] *= self.scaleDetected
# subtract a small scale background when we search for PSFs
if self.doBackground:
self.log.logdebug("Median ring background")
back = satUtil.medianRing(imgClip, self.kernelWidth, 2.0*self.sigmaSmooth)
img -= back
imgClip -= back
# - smooth
img = satUtil.smooth(img, self.sigmaSmooth)
imgClip = satUtil.smooth(imgClip, self.sigmaSmooth)
rms = imgClip[(mskClip & (MASK | DET) == 0)].std()
isCandidate = np.ones(img.shape, dtype=bool)
###########################################
# Try different kernel sizes
###########################################
# Different sized kernels should give the same results for a real trail
# but would be less likely to for noise.
# Unfortunately, this is costly, and the effect is small.
for kernelFactor in (1.0, self.growKernel):
self.log.logdebug("Getting moments growKernel=%.1f" % (self.growKernel))
kernelWidth = 2*int((kernelFactor*self.kernelWidth)//2) + 1
kernelSigma = kernelFactor*self.kernelSigma
isKernelCandidate = np.zeros(img.shape, dtype=bool)
#################################################
# Calibration images
#################################################
calImages = []
for width in widths:
calImages.append(self._makeCalibrationImage(psfSigma, width,
kernelWidth=kernelWidth, kernelSigma=kernelSigma))
################################################
# Moments
#################################################
mm = momCalc.MomentManager(img, kernelWidth=kernelWidth, kernelSigma=kernelSigma)
mmCals = []
nHits = []
#Selector = momCalc.PixelSelector
Selector = momCalc.PValuePixelSelector
maxPixels = 4000
for i, calImg in enumerate(calImages):
mmCal = momCalc.MomentManager(calImg, kernelWidth=kernelWidth, kernelSigma=kernelSigma,
isCalibration=True)
mmCals.append(mmCal)
sumI = momCalc.MomentLimit('sumI', self.luminosityLimit*rms, 'lower')
cent = momCalc.MomentLimit('center', 2.0*self.centerLimit, 'center')
centP = momCalc.MomentLimit('center_perp', self.centerLimit, 'center')
skew = momCalc.MomentLimit('skew', 2.0*self.skewLimit, 'center')
skewP = momCalc.MomentLimit('skew_perp', self.skewLimit, 'center')
ellip = momCalc.MomentLimit('ellip', self.eRange, 'center')
b = momCalc.MomentLimit('b', self.bLimit, 'center')
selector = Selector(mm, mmCal)
for limit in sumI, ellip, cent, centP, skew, skewP, b:
selector.append(limit)
pixels = selector.getPixels(maxPixels=maxPixels)
isKernelCandidate |= pixels
msg = "cand: nPix: %d tot: %d" % (pixels.sum(), isKernelCandidate.sum())
self.log.logdebug(msg)
isCandidate &= isKernelCandidate
self.log.logdebug("total: %d" % (isCandidate.sum()))
nHits.append((widths[i], isKernelCandidate.sum()))
bestCal = sorted(nHits, key=lambda x: x[1], reverse=True)[0]
bestWidth = bestCal[0]
###############################################
# Theta Alignment
###############################################
self.log.logdebug("Theta alignment.")
xx, yy = np.meshgrid(np.arange(img.shape[1], dtype=int), np.arange(img.shape[0], dtype=int))
nBeforeAlignment = isCandidate.sum()
maxSeparation = min([x/2 for x in img.shape])
thetaMatch, newTheta = hough.thetaAlignment(mm.theta[isCandidate], xx[isCandidate], yy[isCandidate],
tolerance=self.thetaTolerance,
limit=3, maxSeparation=maxSeparation)
mm.theta[isCandidate] = newTheta
isCandidate[isCandidate] = thetaMatch
nAfterAlignment = isCandidate.sum()
self.log.logdebug("theta-alignment Bef/aft: %d / %d" % (nBeforeAlignment, nAfterAlignment))
#################################################
# Hough transform
#################################################
self.log.logdebug("Hough Transform.")
rMax = np.linalg.norm(img.shape)
houghTransform = hough.HoughTransform(self.houghBins, self.houghThresh, rMax=rMax,
maxPoints=1000, nIter=1, maxResid=5.5, log=self.log)
solutions = houghTransform(mm.theta[isCandidate], xx[isCandidate], yy[isCandidate])
#################################################
# Construct Trail objects from Hough solutions
#################################################
self.log.logdebug("Constructing SatelliteTrail objects.")
trails = satTrail.SatelliteTrailList(nAfterAlignment, solutions.binMax, psfSigma)
for s in solutions:
trail = satTrail.SatelliteTrail.fromHoughSolution(s, self.bins)
trail.detectWidth = bestWidth
trail.maskAndBits = self.maskAndBits
trail.measure(exp, bins=self.bins)
# last chance to drop it
if trail.width < self.maxTrailWidth:
nx, ny = exposure.getWidth(), exposure.getHeight()
self.log.info(str(trail) + "[%dpix]" % (trail.length(nx,ny)))
trails.append(trail)
else:
self.log.info("Dropping (maxWidth>%.1f): %s" %(self.maxTrailWidth, trail))
# A bit hackish, but stash some useful info for diagnostic plots
self._mm = mm
self._mmCals = mmCals
self._isCandidate = isCandidate
self._brightFactor = 10
self._trails = trails
self._solutions = solutions
return trails
def getCcdImage(self, det, imageFactory, binSize):
return afwMath.binImage(self.image, binSize)
def getCcdImage(self, det, imageFactory, binSize):
return afwMath.binImage(self.image, binSize)
class SubaruIsrTask(IsrTask):
ConfigClass = SubaruIsrConfig
def __init__(self, *args, **kwargs):
super(SubaruIsrTask, self).__init__(*args, **kwargs)
self.makeSubtask("crosstalk")
if self.config.doWriteVignettePolygon:
theta = numpy.linspace(0,
2 * numpy.pi,
num=self.config.numPolygonPoints,
endpoint=False)
x = self.config.vignette.radius * numpy.cos(
theta) + self.config.vignette.xCenter
y = self.config.vignette.radius * numpy.sin(
theta) + self.config.vignette.yCenter
points = numpy.array([x, y]).transpose()
self.vignettePolygon = Polygon(
[afwGeom.Point2D(x, y) for x, y in reversed(points)])
def runDataRef(self, sensorRef):
self.log.log(self.log.INFO,
"Performing ISR on sensor %s" % (sensorRef.dataId))
ccdExposure = sensorRef.get('raw')
if self.config.removePcCards: # Remove any PC00N00M cards in the header
raw_md = sensorRef.get("raw_md")
nPc = 0
for i in (
1,
2,
):
for j in (
1,
2,
):
k = "PC%03d%03d" % (i, j)
for md in (raw_md, ccdExposure.getMetadata()):
if md.exists(k):
md.remove(k)
nPc += 1
if nPc:
self.log.log(
self.log.INFO, "Recreating Wcs after stripping PC00n00m" %
(sensorRef.dataId))
ccdExposure.setWcs(afwImage.makeWcs(raw_md))
ccdExposure = self.convertIntToFloat(ccdExposure)
ccd = ccdExposure.getDetector()
for amp in ccd:
self.measureOverscan(ccdExposure, amp)
if self.config.doSaturation:
self.saturationDetection(ccdExposure, amp)
if self.config.doOverscan:
ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(),
amp.getRawDataBBox(),
afwImage.PARENT)
overscan = afwImage.MaskedImageF(
ccdExposure.getMaskedImage(),
amp.getRawHorizontalOverscanBBox(), afwImage.PARENT)
overscanArray = overscan.getImage().getArray()
median = numpy.ma.median(
numpy.ma.masked_where(overscan.getMask().getArray(),
overscanArray))
bad = numpy.where(
numpy.abs(overscanArray -
median) > self.config.overscanMaxDev)
overscan.getMask().getArray()[bad] = overscan.getMask(
).getPlaneBitMask("SAT")
statControl = afwMath.StatisticsControl()
statControl.setAndMask(
ccdExposure.getMaskedImage().getMask().getPlaneBitMask(
"SAT"))
lsstIsr.overscanCorrection(
ampMaskedImage=ampImage,
overscanImage=overscan,
fitType=self.config.overscanFitType,
order=self.config.overscanOrder,
collapseRej=self.config.overscanRej,
statControl=statControl,
)
if self.config.doVariance:
# Ideally, this should be done after bias subtraction,
# but CCD assembly demands a variance plane
ampExposure = ccdExposure.Factory(ccdExposure,
amp.getRawDataBBox(),
afwImage.PARENT)
self.updateVariance(ampExposure, amp)
ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
ccd = ccdExposure.getDetector()
doRotateCalib = False # Rotate calib images for bias/dark/flat correction?
nQuarter = ccd.getOrientation().getNQuarter()
if nQuarter != 0:
doRotateCalib = True
if self.config.doDefect:
defects = sensorRef.get('defects', immediate=True)
self.maskAndInterpDefect(ccdExposure, defects)
if self.config.qa.doWriteOss:
sensorRef.put(ccdExposure, "ossImage")
if self.config.qa.doThumbnailOss:
self.writeThumbnail(sensorRef, "ossThumb", ccdExposure)
if self.config.doBias:
biasExposure = self.getIsrExposure(sensorRef, "bias")
if not doRotateCalib:
self.biasCorrection(ccdExposure, biasExposure)
else:
with self.rotated(ccdExposure) as exp:
self.biasCorrection(exp, biasExposure)
if self.config.doLinearize:
self.linearize(ccdExposure)
if self.config.doCrosstalk:
self.crosstalk.run(ccdExposure)
if self.config.doDark:
darkExposure = self.getIsrExposure(sensorRef, "dark")
if not doRotateCalib:
self.darkCorrection(ccdExposure, darkExposure)
else:
with self.rotated(ccdExposure) as exp:
self.darkCorrection(exp, darkExposure)
if self.config.doFlat:
flatExposure = self.getIsrExposure(sensorRef, "flat")
if not doRotateCalib:
self.flatCorrection(ccdExposure, flatExposure)
else:
with self.rotated(ccdExposure) as exp:
self.flatCorrection(exp, flatExposure)
if self.config.doApplyGains:
self.applyGains(ccdExposure, self.config.normalizeGains)
if self.config.doWidenSaturationTrails:
self.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
if self.config.doSaturation:
self.saturationInterpolation(ccdExposure)
if self.config.doFringe:
self.fringe.runDataRef(ccdExposure, sensorRef)
if self.config.doSetBadRegions:
self.setBadRegions(ccdExposure)
self.maskAndInterpNan(ccdExposure)
if self.config.qa.doWriteFlattened:
sensorRef.put(ccdExposure, "flattenedImage")
if self.config.qa.doThumbnailFlattened:
self.writeThumbnail(sensorRef, "flattenedThumb", ccdExposure)
self.measureBackground(ccdExposure)
if self.config.doGuider:
self.guider(ccdExposure)
self.roughZeroPoint(ccdExposure)
if self.config.doWriteVignettePolygon:
self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
if self.config.doWrite:
sensorRef.put(ccdExposure, "postISRCCD")
if self._display:
im = ccdExposure.getMaskedImage().getImage()
im_median = float(numpy.median(im.getArray()))
ds9.mtv(im)
ds9.scale(min=im_median * 0.95, max=im_median * 1.15)
return Struct(exposure=ccdExposure)
@contextmanager
def rotated(self, exp):
nQuarter = exp.getDetector().getOrientation().getNQuarter()
exp.setMaskedImage(
afwMath.rotateImageBy90(exp.getMaskedImage(), 4 - nQuarter))
try:
yield exp
finally:
exp.setMaskedImage(
afwMath.rotateImageBy90(exp.getMaskedImage(), nQuarter))
def applyGains(self, ccdExposure, normalizeGains):
ccd = ccdExposure.getDetector()
ccdImage = ccdExposure.getMaskedImage()
medians = []
for a in ccd:
sim = ccdImage.Factory(ccdImage, a.getDataSec())
sim *= a.getElectronicParams().getGain()
if normalizeGains:
medians.append(numpy.median(sim.getImage().getArray()))
if normalizeGains:
median = numpy.median(numpy.array(medians))
for i, a in enumerate(ccd):
sim = ccdImage.Factory(ccdImage, a.getDataSec())
sim *= median / medians[i]
def widenSaturationTrails(self, mask):
"""Grow the saturation trails by an amount dependent on the width of the trail"""
extraGrowDict = {}
for i in range(1, 6):
extraGrowDict[i] = 0
for i in range(6, 8):
extraGrowDict[i] = 1
for i in range(8, 10):
extraGrowDict[i] = 3
extraGrowMax = 4
if extraGrowMax <= 0:
return
saturatedBit = mask.getPlaneBitMask('SAT')
xmin, ymin = mask.getBBox().getMin()
width = mask.getWidth()
thresh = afwDetection.Threshold(saturatedBit,
afwDetection.Threshold.BITMASK)
fpList = afwDetection.FootprintSet(mask, thresh).getFootprints()
for fp in fpList:
for s in fp.getSpans():
x0, x1 = s.getX0(), s.getX1()
extraGrow = extraGrowDict.get(x1 - x0 + 1, extraGrowMax)
if extraGrow > 0:
y = s.getY() - ymin
x0 -= xmin + extraGrow
x1 -= xmin - extraGrow
if x0 < 0: x0 = 0
if x1 >= width - 1: x1 = width - 1
for x in range(x0, x1 + 1):
mask.set(x, y, mask.get(x, y) | saturatedBit)
def setBadRegions(self, exposure):
"""Set all BAD areas of the chip to the average of the rest of the exposure
@param[in,out] exposure exposure to process; must include both DataSec and BiasSec pixels
"""
if self.config.badStatistic == "MEDIAN":
statistic = afwMath.MEDIAN
elif self.config.badStatistic == "MEANCLIP":
statistic = afwMath.MEANCLIP
else:
raise RuntimeError(
"Impossible method %s of bad region correction" %
self.config.badStatistic)
mi = exposure.getMaskedImage()
mask = mi.getMask()
BAD = mask.getPlaneBitMask("BAD")
INTRP = mask.getPlaneBitMask("INTRP")
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(BAD)
value = afwMath.makeStatistics(mi, statistic, sctrl).getValue()
maskArray = mask.getArray()
imageArray = mi.getImage().getArray()
badPixels = numpy.logical_and((maskArray & BAD) > 0,
(maskArray & INTRP) == 0)
imageArray[:] = numpy.where(badPixels, value, imageArray)
self.log.info("Set %d BAD pixels to %.2f" % (badPixels.sum(), value))
def writeThumbnail(self, dataRef, dataset, exposure):
"""Write out exposure to a snapshot file named outfile in the given size.
"""
filename = dataRef.get(dataset + "_filename")[0]
directory = os.path.dirname(filename)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError, e:
# Don't fail if directory exists due to race
if e.errno != errno.EEXIST:
raise e
binning = self.config.thumbnailBinning
binnedImage = afwMath.binImage(exposure.getMaskedImage(), binning,
binning, afwMath.MEAN)
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setAndMask(binnedImage.getMask().getPlaneBitMask(
["SAT", "BAD", "INTRP"]))
stats = afwMath.makeStatistics(
binnedImage, afwMath.MEDIAN | afwMath.STDEVCLIP | afwMath.MAX,
statsCtrl)
low = stats.getValue(
afwMath.MEDIAN) - self.config.thumbnailStdev * stats.getValue(
afwMath.STDEVCLIP)
makeRGB(binnedImage,
binnedImage,
binnedImage,
min=low,
range=self.config.thumbnailRange,
Q=self.config.thumbnailQ,
fileName=filename,
saturatedBorderWidth=self.config.thumbnailSatBorder,
saturatedPixelValue=stats.getValue(afwMath.MAX))
