def _testZogyDiffimMapReduced(self, inImageSpace=False, doScorr=False, **kwargs):
"""Test running Zogy using ImageMapReduceTask framework.
Compare map-reduced version with non-map-reduced version.
Do it for pure Fourier-based calc. and also for real-space.
Also for computing pure diffim D and corrected likelihood image Scorr.
"""
config = ZogyMapReduceConfig()
config.gridStepX = config.gridStepY = 9
config.borderSizeX = config.borderSizeY = 3
if inImageSpace:
config.gridStepX = config.gridStepY = 8
config.borderSizeX = config.borderSizeY = 6 # need larger border size for image-space run
config.reducer.reduceOperation = 'average'
task = ImageMapReduceTask(config=config)
D_mapReduced = task.run(self.im1ex, template=self.im2ex, inImageSpace=inImageSpace,
doScorr=doScorr, forceEvenSized=False, **kwargs).exposure
config = ZogyConfig()
task = ZogyTask(templateExposure=self.im2ex, scienceExposure=self.im1ex, config=config)
if not doScorr:
D = task.computeDiffim(inImageSpace=inImageSpace, **kwargs).D
else:
D = task.computeScorr(inImageSpace=inImageSpace, **kwargs).S
self._compareExposures(D_mapReduced, D, tol=0.04, Scorr=doScorr)
def _testAverageVersusCopy(self, withNaNs=False):
"""Re-run `testExampleTaskNoOverlaps` and `testExampleTaskWithOverlaps`
on a more complex image (with random noise). Ensure that the results are
identical (within between 'copy' and 'average' reduceOperation.
"""
exposure1 = self.exposure.clone()
img = exposure1.getMaskedImage().getImage()
afwMath.randomGaussianImage(img, afwMath.Random())
exposure2 = exposure1.clone()
config = AddAmountImageMapReduceConfig()
task = ImageMapReduceTask(config)
config.mapper.addAmount = 5.
newExp = task.run(exposure1, addNans=withNaNs).exposure
newMI1 = newExp.getMaskedImage()
config.gridStepX = config.gridStepY = 8.
config.reducer.reduceOperation = 'average'
task = ImageMapReduceTask(config)
newExp = task.run(exposure2, addNans=withNaNs).exposure
newMI2 = newExp.getMaskedImage()
newMA1 = newMI1.getImage().getArray()
isnan = np.isnan(newMA1)
if not withNaNs:
self.assertEqual(np.sum(isnan), 0)
newMA2 = newMI2.getImage().getArray()
# Because the average uses a float accumulator, we can have differences, set a tolerance.
# Turns out (in practice for this test), only 7 pixels seem to have a small difference.
self.assertFloatsAlmostEqual(newMA1[~isnan], newMA2[~isnan], rtol=1e-7)
def _testCopySumNoOverlaps(self, reduceOp='copy', withNaNs=False):
"""Test sample grid task that adds 5.0 to input image and uses
`reduceOperation = 'copy'`. Optionally add NaNs to subimages.
"""
config = AddAmountImageMapReduceConfig()
task = ImageMapReduceTask(config)
config.mapper.addAmount = 5.
config.reducer.reduceOperation = reduceOp
newExp = task.run(self.exposure, addNans=withNaNs).exposure
newMI = newExp.getMaskedImage()
newArr = newMI.getImage().getArray()
isnan = np.isnan(newArr)
if not withNaNs:
self.assertEqual(np.sum(isnan),
0,
msg='Failed on withNaNs: %s' % str(withNaNs))
mi = self.exposure.getMaskedImage().getImage().getArray()
if reduceOp != 'sum':
self.assertFloatsAlmostEqual(mi[~isnan],
newArr[~isnan] - 5.,
msg='Failed on withNaNs: %s' %
str(withNaNs))
else: # We don't construct a new PSF if reduceOperation == 'copy'.
self._testCoaddPsf(newExp)
def _testAverageWithOverlaps(self, withNaNs=False):
"""Test sample grid task that adds 5.0 to input image and uses
'average' `reduceOperation`. Optionally add NaNs to subimages.
"""
config = AddAmountImageMapReduceConfig()
config.gridStepX = config.gridStepY = 8.
config.reducer.reduceOperation = 'average'
task = ImageMapReduceTask(config)
config.mapper.addAmount = 5.
newExp = task.run(self.exposure, addNans=withNaNs).exposure
newMI = newExp.getMaskedImage()
newArr = newMI.getImage().getArray()
mi = self.exposure.getMaskedImage()
isnan = np.isnan(newArr)
if not withNaNs:
self.assertEqual(np.sum(isnan),
0,
msg='Failed on withNaNs: %s' % str(withNaNs))
mi = self.exposure.getMaskedImage().getImage().getArray()
self.assertFloatsAlmostEqual(mi[~isnan],
newArr[~isnan] - 5.,
msg='Failed on withNaNs: %s' %
str(withNaNs))
self._testCoaddPsf(newExp)
def testMasks(self):
"""Test the mask for an exposure produced by a sample grid task
where we provide a set of `cellCentroids` and thus should have
many invalid pixels.
"""
config = AddAmountImageMapReduceConfig()
config.gridStepX = config.gridStepY = 8.
config.cellCentroidsX = [i for i in np.linspace(0, 128, 50)]
config.cellCentroidsY = config.cellCentroidsX
config.reducer.reduceOperation = 'average'
task = ImageMapReduceTask(config)
config.mapper.addAmount = 5.
newExp = task.run(self.exposure).exposure
newMI = newExp.getMaskedImage()
newArr = newMI.getImage().getArray()
mi = self.exposure.getMaskedImage()
isnan = np.isnan(newArr)
self.assertGreater(np.sum(isnan), 1000)
mi = self.exposure.getMaskedImage().getImage().getArray()
self.assertFloatsAlmostEqual(mi[~isnan], newArr[~isnan] - 5.)
mask = newMI.getMask() # Now check the mask
self.assertGreater(mask.getMaskPlane('INVALID_MAPREDUCE'), 0)
maskBit = mask.getPlaneBitMask('INVALID_MAPREDUCE')
nMasked = np.sum(np.bitwise_and(mask.getArray(), maskBit) != 0)
self.assertGreater(nMasked, 1000)
self.assertEqual(np.sum(np.isnan(newArr)), nMasked)
def testMasks(self):
"""Test the mask for an exposure produced by a sample grid task
where we provide a set of `cellCentroids` and thus should have
many invalid pixels.
"""
config = AddAmountImageMapReduceConfig()
config.gridStepX = config.gridStepY = 8.
config.cellCentroidsX = [i for i in np.linspace(0, 128, 50)]
config.cellCentroidsY = config.cellCentroidsX
config.reducer.reduceOperation = 'average'
task = ImageMapReduceTask(config)
config.mapper.addAmount = 5.
newExp = task.run(self.exposure).exposure
newMI = newExp.getMaskedImage()
newArr = newMI.getImage().getArray()
mi = self.exposure.getMaskedImage()
isnan = np.isnan(newArr)
self.assertGreater(np.sum(isnan), 1000)
mi = self.exposure.getMaskedImage().getImage().getArray()
self.assertFloatsAlmostEqual(mi[~isnan], newArr[~isnan] - 5.)
mask = newMI.getMask() # Now check the mask
self.assertGreater(mask.getMaskPlane('INVALID_MAPREDUCE'), 0)
maskBit = mask.getPlaneBitMask('INVALID_MAPREDUCE')
nMasked = np.sum(np.bitwise_and(mask.getArray(), maskBit) != 0)
self.assertGreater(nMasked, 1000)
self.assertEqual(np.sum(np.isnan(newArr)), nMasked)
def _runGridValidity(self, config, gstepx, gsizex, gstepy, gsizey,
adjustGridOption, expectedVal=1.):
"""Method to test the grid validity given an input config.
Here we also iterate over scaleByFwhm in (True, False) and
ensure that we get more `boxes` when `scaleByFwhm=False` than
vice versa.
Parameters
----------
config : `ipDiffim.AddAmountImageMapReduceConfig`
input AddAmountImageMapReduceConfig
gstepx : `float`
grid x-direction step size
gsizex : `float`
grid x-direction box size
gstepy : `float`
grid y-direction step size
gsizey : `float`
grid y-direction box size
expectedVal : `float`
float to add to exposure (to compare for testing)
"""
config.mapper.addAmount = expectedVal
lenBoxes = [0, 0]
for scaleByFwhm in (True, False):
config.scaleByFwhm = scaleByFwhm
if scaleByFwhm:
config.gridStepX = float(gstepx)
config.cellSizeX = float(gsizex)
config.gridStepY = float(gstepy)
config.cellSizeY = float(gsizey)
else: # otherwise the grid is too fine and elements too small.
config.gridStepX = gstepx * 3.
config.cellSizeX = gsizex * 3.
config.gridStepY = gstepy * 3.
config.cellSizeY = gsizey * 3.
config.adjustGridOption = adjustGridOption
task = ImageMapReduceTask(config)
task._generateGrid(self.exposure)
ind = 0 if scaleByFwhm else 1
lenBoxes[ind] = len(task.boxes0)
newExp = task.run(self.exposure).exposure
newMI = newExp.getMaskedImage()
newArr = newMI.getImage().getArray()
isnan = np.isnan(newArr)
self.assertEqual(np.sum(isnan), 0, msg='Failed NaN (%d), on config: %s' %
(np.sum(isnan), str(config)))
mi = self.exposure.getMaskedImage().getImage().getArray()
self.assertFloatsAlmostEqual(mi[~isnan], newArr[~isnan] - expectedVal,
msg='Failed on config: %s' % str(config))
self.assertLess(lenBoxes[0], lenBoxes[1], msg='Failed lengths on config: %s' %
str(config))
def _runGridValidity(self, config, gstepx, gsizex, gstepy, gsizey,
adjustGridOption, expectedVal=1.):
"""Method to test the grid validity given an input config.
Here we also iterate over scaleByFwhm in (True, False) and
ensure that we get more `boxes` when `scaleByFwhm=False` than
vice versa.
Parameters
----------
config : `ipDiffim.AddAmountImageMapReduceConfig`
input AddAmountImageMapReduceConfig
gstepx : `float`
grid x-direction step size
gsizex : `float`
grid x-direction box size
gstepy : `float`
grid y-direction step size
gsizey : `float`
grid y-direction box size
expectedVal : `float`
float to add to exposure (to compare for testing)
"""
config.mapper.addAmount = expectedVal
lenBoxes = [0, 0]
for scaleByFwhm in (True, False):
config.scaleByFwhm = scaleByFwhm
if scaleByFwhm:
config.gridStepX = float(gstepx)
config.cellSizeX = float(gsizex)
config.gridStepY = float(gstepy)
config.cellSizeY = float(gsizey)
else: # otherwise the grid is too fine and elements too small.
config.gridStepX = gstepx * 3.
config.cellSizeX = gsizex * 3.
config.gridStepY = gstepy * 3.
config.cellSizeY = gsizey * 3.
config.adjustGridOption = adjustGridOption
task = ImageMapReduceTask(config)
task._generateGrid(self.exposure)
ind = 0 if scaleByFwhm else 1
lenBoxes[ind] = len(task.boxes0)
newExp = task.run(self.exposure).exposure
newMI = newExp.getMaskedImage()
newArr = newMI.getImage().getArray()
isnan = np.isnan(newArr)
self.assertEqual(np.sum(isnan), 0, msg='Failed NaN (%d), on config: %s' %
(np.sum(isnan), str(config)))
mi = self.exposure.getMaskedImage().getImage().getArray()
self.assertFloatsAlmostEqual(mi[~isnan], newArr[~isnan] - expectedVal,
msg='Failed on config: %s' % str(config))
self.assertLess(lenBoxes[0], lenBoxes[1], msg='Failed lengths on config: %s' %
str(config))
def testNotNoneReduceWithNonExposureMapper(self):
"""Test that a combination of a mapper that returns a non-exposure
cannot work correctly with a reducer with reduceOperation='none'.
Should raise a TypeError.
"""
config = GetMeanImageMapReduceConfig() # mapper returns a float (mean)
config.gridStepX = config.gridStepY = 8.
config.reducer.reduceOperation = 'average' # not 'none'!
task = ImageMapReduceTask(config)
with self.assertRaises(TypeError):
task.run(self.exposure)
def _runDecorrelationTaskMapReduced(self, diffExp, mKernel):
""" Run decorrelation using the imageMapReducer.
"""
config = DecorrelateALKernelMapReduceConfig()
config.borderSizeX = config.borderSizeY = 3
config.reducer.reduceOperation = 'average'
task = ImageMapReduceTask(config=config)
decorrResult = task.run(diffExp, template=self.im2ex, science=self.im1ex,
psfMatchingKernel=mKernel, forceEvenSized=True)
corrected_diffExp = decorrResult.exposure
return corrected_diffExp
def _runDecorrelationTaskMapReduced(self, diffExp, mKernel):
""" Run decorrelation using the imageMapReducer.
"""
config = DecorrelateALKernelMapReduceConfig()
config.borderSizeX = config.borderSizeY = 3
config.reducer.reduceOperation = 'average'
task = ImageMapReduceTask(config=config)
decorrResult = task.run(diffExp, template=self.im2ex, science=self.im1ex,
psfMatchingKernel=mKernel, forceEvenSized=True)
corrected_diffExp = decorrResult.exposure
return corrected_diffExp
def testNotNoneReduceWithNonExposureMapper(self):
"""Test that a combination of a mapper that returns a non-exposure
cannot work correctly with a reducer with reduceOperation='none'.
Should raise a TypeError.
"""
config = GetMeanImageMapReduceConfig() # mapper returns a float (mean)
config.gridStepX = config.gridStepY = 8.
config.reducer.reduceOperation = 'average' # not 'none'!
task = ImageMapReduceTask(config)
with self.assertRaises(TypeError):
task.run(self.exposure)
def testMean(self):
"""Test sample grid task that returns the mean of the subimages and uses
'none' `reduceOperation`.
"""
config = GetMeanImageMapReduceConfig()
config.reducer.reduceOperation = 'none'
task = ImageMapReduceTask(config)
testExposure = self.exposure.clone()
testExposure.getMaskedImage().set(1.234)
subMeans = task.run(testExposure).result
subMeans = [x.subExposure for x in subMeans]
self.assertEqual(len(subMeans), len(task.boxes0))
firstPixel = testExposure.getMaskedImage().getImage().getArray()[0, 0]
self.assertFloatsAlmostEqual(np.array(subMeans), firstPixel)
def testMean(self):
"""Test sample grid task that returns the mean of the subimages and uses
'none' `reduceOperation`.
"""
config = GetMeanImageMapReduceConfig()
config.reducer.reduceOperation = 'none'
task = ImageMapReduceTask(config)
testExposure = self.exposure.clone()
testExposure.getMaskedImage().set(1.234)
subMeans = task.run(testExposure).result
subMeans = [x.subExposure for x in subMeans]
self.assertEqual(len(subMeans), len(task.boxes0))
firstPixel = testExposure.getMaskedImage().getImage().getArray()[0, 0]
self.assertFloatsAlmostEqual(np.array(subMeans), firstPixel)
def testCellCentroidsWrongLength(self):
"""Test sample grid task which is provided a set of `cellCentroids` and
returns the mean of the subimages surrounding those centroids using 'none'
for `reduceOperation`. In this case, we ensure that len(task.boxes0) !=
len(task.boxes1) and check for ValueError.
"""
config = GetMeanImageMapReduceConfig()
config.reducer.reduceOperation = 'none'
config.cellCentroidsX = [i for i in np.linspace(0, 128, 50)]
config.cellCentroidsY = [i for i in np.linspace(0, 128, 50)]
task = ImageMapReduceTask(config)
task._generateGrid(self.exposure)
del task.boxes0[-1] # remove the last box
with self.assertRaises(ValueError):
task.run(self.exposure)
def testCellCentroids(self):
"""Test sample grid task which is provided a set of `cellCentroids` and
returns the mean of the subimages surrounding those centroids using 'none'
for `reduceOperation`.
"""
config = GetMeanImageMapReduceConfig()
config.gridStepX = config.gridStepY = 8.
config.reducer.reduceOperation = 'none'
config.cellCentroidsX = [i for i in np.linspace(0, 128, 50)]
config.cellCentroidsY = config.cellCentroidsX
task = ImageMapReduceTask(config)
testExposure = self.exposure.clone()
testExposure.getMaskedImage().set(1.234)
subMeans = task.run(testExposure).result
subMeans = [x.subExposure for x in subMeans]
self.assertEqual(len(subMeans), len(config.cellCentroidsX))
firstPixel = testExposure.getMaskedImage().getImage().getArray()[0, 0]
self.assertFloatsAlmostEqual(np.array(subMeans), firstPixel)
def testCellCentroids(self):
"""Test sample grid task which is provided a set of `cellCentroids` and
returns the mean of the subimages surrounding those centroids using 'none'
for `reduceOperation`.
"""
config = GetMeanImageMapReduceConfig()
config.gridStepX = config.gridStepY = 8.
config.reducer.reduceOperation = 'none'
config.cellCentroidsX = [i for i in np.linspace(0, 128, 50)]
config.cellCentroidsY = config.cellCentroidsX
task = ImageMapReduceTask(config)
testExposure = self.exposure.clone()
testExposure.getMaskedImage().set(1.234)
subMeans = task.run(testExposure).result
subMeans = [x.subExposure for x in subMeans]
self.assertEqual(len(subMeans), len(config.cellCentroidsX))
firstPixel = testExposure.getMaskedImage().getImage().getArray()[0, 0]
self.assertFloatsAlmostEqual(np.array(subMeans), firstPixel)
def _testAverageVersusCopy(self, withNaNs=False):
"""Re-run `testExampleTaskNoOverlaps` and `testExampleTaskWithOverlaps`
on a more complex image (with random noise). Ensure that the results are
identical (within between 'copy' and 'average' reduceOperation.
"""
exposure1 = self.exposure.clone()
img = exposure1.getMaskedImage().getImage()
afwMath.randomGaussianImage(img, afwMath.Random())
exposure2 = exposure1.clone()
config = AddAmountImageMapReduceConfig()
task = ImageMapReduceTask(config)
config.mapper.addAmount = 5.
newExp = task.run(exposure1, addNans=withNaNs).exposure
newMI1 = newExp.getMaskedImage()
config.gridStepX = config.gridStepY = 8.
config.reducer.reduceOperation = 'average'
task = ImageMapReduceTask(config)
newExp = task.run(exposure2, addNans=withNaNs).exposure
newMI2 = newExp.getMaskedImage()
newMA1 = newMI1.getImage().getArray()
isnan = np.isnan(newMA1)
if not withNaNs:
self.assertEqual(np.sum(isnan), 0)
newMA2 = newMI2.getImage().getArray()
# Because the average uses a float accumulator, we can have differences, set a tolerance.
# Turns out (in practice for this test), only 7 pixels seem to have a small difference.
self.assertFloatsAlmostEqual(newMA1[~isnan], newMA2[~isnan], rtol=1e-7)
def _testZogyDiffimMapReduced(self, inImageSpace=False, doScorr=False, **kwargs):
"""Test running Zogy using ImageMapReduceTask framework.
Compare map-reduced version with non-map-reduced version.
Do it for pure Fourier-based calc. and also for real-space.
Also for computing pure diffim D and corrected likelihood image Scorr.
"""
config = ZogyMapReduceConfig()
config.gridStepX = config.gridStepY = 9
config.borderSizeX = config.borderSizeY = 3
if inImageSpace:
config.gridStepX = config.gridStepY = 8
config.borderSizeX = config.borderSizeY = 6 # need larger border size for image-space run
config.reducer.reduceOperation = 'average'
task = ImageMapReduceTask(config=config)
D_mapReduced = task.run(self.im1ex, template=self.im2ex, inImageSpace=inImageSpace,
doScorr=doScorr, forceEvenSized=False, **kwargs).exposure
config = ZogyConfig()
task = ZogyTask(templateExposure=self.im2ex, scienceExposure=self.im1ex, config=config)
if not doScorr:
D = task.computeDiffim(inImageSpace=inImageSpace, **kwargs).D
else:
D = task.computeScorr(inImageSpace=inImageSpace, **kwargs).S
self._compareExposures(D_mapReduced, D, tol=0.04, Scorr=doScorr)
def _testCopySumNoOverlaps(self, reduceOp='copy', withNaNs=False):
"""Test sample grid task that adds 5.0 to input image and uses
`reduceOperation = 'copy'`. Optionally add NaNs to subimages.
"""
config = AddAmountImageMapReduceConfig()
task = ImageMapReduceTask(config)
config.mapper.addAmount = 5.
config.reducer.reduceOperation = reduceOp
newExp = task.run(self.exposure, addNans=withNaNs).exposure
newMI = newExp.getMaskedImage()
newArr = newMI.getImage().getArray()
isnan = np.isnan(newArr)
if not withNaNs:
self.assertEqual(np.sum(isnan), 0,
msg='Failed on withNaNs: %s' % str(withNaNs))
mi = self.exposure.getMaskedImage().getImage().getArray()
if reduceOp != 'sum':
self.assertFloatsAlmostEqual(mi[~isnan], newArr[~isnan] - 5.,
msg='Failed on withNaNs: %s' % str(withNaNs))
else: # We don't construct a new PSF if reduceOperation == 'copy'.
self._testCoaddPsf(newExp)
def _testAverageWithOverlaps(self, withNaNs=False):
"""Test sample grid task that adds 5.0 to input image and uses
'average' `reduceOperation`. Optionally add NaNs to subimages.
"""
config = AddAmountImageMapReduceConfig()
config.gridStepX = config.gridStepY = 8.
config.reducer.reduceOperation = 'average'
task = ImageMapReduceTask(config)
config.mapper.addAmount = 5.
newExp = task.run(self.exposure, addNans=withNaNs).exposure
newMI = newExp.getMaskedImage()
newArr = newMI.getImage().getArray()
mi = self.exposure.getMaskedImage()
isnan = np.isnan(newArr)
if not withNaNs:
self.assertEqual(np.sum(isnan), 0,
msg='Failed on withNaNs: %s' % str(withNaNs))
mi = self.exposure.getMaskedImage().getImage().getArray()
self.assertFloatsAlmostEqual(mi[~isnan], newArr[~isnan] - 5.,
msg='Failed on withNaNs: %s' % str(withNaNs))
self._testCoaddPsf(newExp)
def testCellCentroidsWrongLength(self):
"""Test sample grid task which is provided a set of `cellCentroids` and
returns the mean of the subimages surrounding those centroids using 'none'
for `reduceOperation`. In this case, we ensure that len(task.boxes0) !=
len(task.boxes1) and check for ValueError.
"""
config = GetMeanImageMapReduceConfig()
config.reducer.reduceOperation = 'none'
config.cellCentroidsX = [i for i in np.linspace(0, 128, 50)]
config.cellCentroidsY = [i for i in np.linspace(0, 128, 50)]
task = ImageMapReduceTask(config)
task._generateGrid(self.exposure)
del task.boxes0[-1] # remove the last box
with self.assertRaises(ValueError):
task.run(self.exposure)