def testCreate(self): rngs = [] for name in afwMath.Random.getAlgorithmNames(): rngs.append(afwMath.Random(name)) for r in rngs: assert afwMath.Random(r.getAlgorithmName()).uniform() == r.uniform() r2 = afwMath.Random(r.getAlgorithm()) r2.uniform() assert r2.uniform() == r.uniform()
def testPolicy(self): """ Tests that policy files and environment variables can override user specified algorithms and seed values. """ pol = pexPolicy.Policy() seed = getSeed() pol.set("rngSeed", str(seed)) pol.set("rngAlgorithm", afwMath.Random.getAlgorithmNames()[afwMath.Random.RANLXD2]) r1 = afwMath.Random(afwMath.Random.RANLXD2, seed) r2 = afwMath.Random(pol) checkRngEquivalence(r1, r2)
def testWeightedStats(self): """Test that bug from #1697 (weighted stats returning NaN) stays fixed.""" rand = afwMath.Random() mu = 10000 edgeMask = afwImage.MaskU.getPlaneBitMask("EDGE") badPixelMask = afwImage.MaskU.getPlaneBitMask("EDGE") statsCtrl = afwMath.StatisticsControl() statsCtrl.setNumSigmaClip(3.0) statsCtrl.setNumIter(2) statsCtrl.setAndMask(badPixelMask) for weight in (300.0, 10.0, 1.0): print "Testing with weight=%0.1f" % (weight,) maskedImageList = afwImage.vectorMaskedImageF() # [] is rejected by afwMath.statisticsStack weightList = [] nx, ny = 256, 256 for i in range(3): print "Processing ", i maskedImage = afwImage.MaskedImageF(nx, ny) maskedImageList.append(maskedImage) afwMath.randomPoissonImage(maskedImage.getImage(), rand, mu) maskedImage.getVariance().set(mu) weightList.append(weight) self.reportBadPixels(maskedImage, badPixelMask) print "Stack: ", coaddMaskedImage = afwMath.statisticsStack( maskedImageList, afwMath.MEANCLIP, statsCtrl, weightList) self.reportBadPixels(coaddMaskedImage, badPixelMask)
def _testAverageVersusCopy(self, withNaNs=False): """Re-run `testExampleTaskNoOverlaps` and `testExampleTaskWithOverlaps` on a more complex image (with random noise). Ensure that the results are identical (within between 'copy' and 'average' reduceOperation. """ exposure1 = self.exposure.clone() img = exposure1.getMaskedImage().getImage() afwMath.randomGaussianImage(img, afwMath.Random()) exposure2 = exposure1.clone() config = AddAmountImageMapReduceConfig() task = ImageMapReduceTask(config) config.mapper.addAmount = 5. newExp = task.run(exposure1, addNans=withNaNs).exposure newMI1 = newExp.getMaskedImage() config.gridStepX = config.gridStepY = 8. config.reducer.reduceOperation = 'average' task = ImageMapReduceTask(config) newExp = task.run(exposure2, addNans=withNaNs).exposure newMI2 = newExp.getMaskedImage() newMA1 = newMI1.getImage().getArray() isnan = np.isnan(newMA1) if not withNaNs: self.assertEqual(np.sum(isnan), 0) newMA2 = newMI2.getImage().getArray() # Because the average uses a float accumulator, we can have differences, set a tolerance. # Turns out (in practice for this test), only 7 pixels seem to have a small difference. self.assertFloatsAlmostEqual(newMA1[~isnan], newMA2[~isnan], rtol=1e-7)
def testCopy(self): """ Test that the generator returned by deepCopy() produces an identical random number sequence to its prototype """ rng1 = afwMath.Random(afwMath.Random.MT19937, getSeed()) rng2 = rng1.deepCopy() checkRngEquivalence(rng1, rng2)
def addNoise(self, mi): img = mi.getImage() seed = int(afwMath.makeStatistics(mi.getVariance(), afwMath.MEDIAN).getValue()) rdm = afwMath.Random(afwMath.Random.MT19937, seed) rdmImage = img.Factory(img.getDimensions()) afwMath.randomGaussianImage(rdmImage, rdm) img += rdmImage return afwMath.makeStatistics(rdmImage, afwMath.MEAN).getValue(afwMath.MEAN)
def makeFlatNoiseImage(mi, seedStat=afwMath.MAX): img = mi.getImage() seed = int(10. * afwMath.makeStatistics(mi.getImage(), seedStat).getValue() + 1) rdm = afwMath.Random(afwMath.Random.MT19937, seed) rdmImage = img.Factory(img.getDimensions()) afwMath.randomGaussianImage(rdmImage, rdm) return rdmImage
def __init__(self, **kwargs): BaseFakeSourcesTask.__init__(self, **kwargs) print("RNG seed:", self.config.seed) self.rng = afwMath.Random(seed=self.config.seed) self.npRand = np.random.RandomState(self.config.seed) try: self.starData = fits.open(self.config.starList)[1].data except Exception: raise
def testChipGapHorizontalBackground(self): """ Test able to match image with horizontal chip gap (row of nans) with .Background""" self.matcher.config.usePolynomial = False self.matcher.config.binSize = 64 chipGapHorizontal = afwImage.ExposureF(600, 600) im = chipGapHorizontal.getMaskedImage().getImage() afwMath.randomGaussianImage(im, afwMath.Random()) im += 10 im.getArray()[200:300, :] = np.nan # simulate 100pix chip gap horizontal chipGapHorizontal.getMaskedImage().getVariance().set(1.0) self.checkAccuracy(self.vanilla, chipGapHorizontal)
def addNoise(mi): img = mi.getImage() seed = int(afwMath.makeStatistics(mi.getVariance(), afwMath.MAX).getValue())+1 rdm = afwMath.Random(afwMath.Random.MT19937, seed) rdmImage = img.Factory(img.getDimensions()) afwMath.randomGaussianImage(rdmImage, rdm) rdmImage *= num.sqrt(seed) img += rdmImage # and don't forget to add to the variance var = mi.getVariance() var += 1.0
def setUp(self): np.random.seed(1) # Make a few test images here # 1) full coverage (plain vanilla image) has mean = 50counts self.vanilla = afwImage.ExposureF(600, 600) im = self.vanilla.getMaskedImage().getImage() afwMath.randomGaussianImage(im, afwMath.Random('MT19937', 1)) im += 50 self.vanilla.getMaskedImage().getVariance().set(1.0) # 2) has chip gap and mean = 10 counts self.chipGap = afwImage.ExposureF(600, 600) im = self.chipGap.getMaskedImage().getImage() afwMath.randomGaussianImage(im, afwMath.Random('MT19937', 2)) im += 10 im.getArray()[:, 200:300] = np.nan # simulate 100pix chip gap self.chipGap.getMaskedImage().getVariance().set(1.0) # 3) has low coverage and mean = 20 counts self.lowCover = afwImage.ExposureF(600, 600) im = self.lowCover.getMaskedImage().getImage() afwMath.randomGaussianImage(im, afwMath.Random('MT19937', 3)) im += 20 self.lowCover.getMaskedImage().getImage().getArray()[:, 200:] = np.nan self.lowCover.getMaskedImage().getVariance().set(1.0) # make a matchBackgrounds object self.matcher = MatchBackgroundsTask() self.matcher.config.usePolynomial = True self.matcher.binSize = 64 self.matcher.debugDataIdString = 'Test Visit' self.sctrl = afwMath.StatisticsControl() self.sctrl.setNanSafe(True) self.sctrl.setAndMask( afwImage.Mask.getPlaneBitMask([ "NO_DATA", "DETECTED", "DETECTED_NEGATIVE", "SAT", "BAD", "INTRP", "CR" ]))
def testRampApproximate(self): """Test basic matching of a linear gradient with Approximate.""" self.matcher.config.binSize = 64 testExp = afwImage.ExposureF(self.vanilla, True) testIm = testExp.getMaskedImage().getImage() afwMath.randomGaussianImage(testIm, afwMath.Random()) nx, ny = testExp.getDimensions() dzdx, dzdy, z0 = 1, 2, 0.0 for x in range(nx): for y in range(ny): z = testIm.get(x, y) testIm.set(x, y, z + dzdx * x + dzdy * y + z0) self.checkAccuracy(testExp, self.vanilla)
def testRampBackground(self): """Test basic matching of a linear gradient with .Background.""" self.matcher.config.usePolynomial = False self.matcher.config.binSize = 64 testExp = afwImage.ExposureF(self.vanilla, True) testIm = testExp.getMaskedImage().getImage() afwMath.randomGaussianImage(testIm, afwMath.Random()) nx, ny = testExp.getDimensions() dzdx, dzdy, z0 = 1, 2, 0.0 for x in range(nx): for y in range(ny): z = testIm[x, y, afwImage.LOCAL] testIm[x, y, afwImage.LOCAL] = z + dzdx * x + dzdy * y + z0 self.checkAccuracy(testExp, self.vanilla)
def makeImage(width=500, height=1000): mi = afwImage.MaskedImageF(width, height) var = 50 mi.set(1000, 0x0, var) addSaturated(mi, addCrosstalk=True) ralg, rseed = "MT19937", int(time.time()) if True else 1234 noise = afwImage.ImageF(width, height) afwMath.randomGaussianImage(noise, afwMath.Random(ralg, rseed)) noise *= math.sqrt(var) mi += noise return mi
def setUp(self): self.random = afwMath.Random() self.imWidth = 200 self.imHeight = 200 self.maskedImage0 = afwImage.MaskedImageF( lsst.geom.Extent2I(self.imWidth, self.imHeight)) afwMath.randomUniformImage(self.maskedImage0.getImage(), self.random) afwMath.randomUniformImage(self.maskedImage0.getVariance(), self.random) # afwMath.randomUniformImage(self.maskedImage0.getMask(), self.random) self.maskedImage1 = afwImage.MaskedImageF( lsst.geom.Extent2I(self.imWidth, self.imHeight)) afwMath.randomUniformImage(self.maskedImage1.getImage(), self.random) afwMath.randomUniformImage(self.maskedImage1.getVariance(), self.random)
def test1(self): task = measAlg.ReplaceWithNoiseTask() schema = afwTable.SourceTable.makeMinimalSchema() table = afwTable.SourceTable.make(schema) sources = afwTable.SourceCatalog(table) im = afwImage.ImageF(200, 50) seed = 42 rand = afwMath.Random(afwMath.Random.MT19937, seed) afwMath.randomGaussianImage(im, rand) s = sources.addNew() s.setId(1) fp = afwDet.Footprint() y, x0, x1 = (10, 10, 190) im.getArray()[y, x0:x1] = 10 fp.addSpan(y, x0, x1) s.setFootprint(fp) s = sources.addNew() s.setId(2) fp = afwDet.Footprint() y, x0, x1 = (40, 10, 190) im.getArray()[y, x0:x1] = 10 fp.addSpan(y, x0, x1) s.setFootprint(fp) mi = afwImage.MaskedImageF(im) exposure = afwImage.makeExposure(mi) self._save(mi, 'a') task.begin(exposure, sources) self._save(mi, 'b') sourcei = 0 task.insertSource(exposure, sourcei) self._save(mi, 'c') # do something task.removeSource(exposure, sources, sources[sourcei]) self._save(mi, 'd') sourcei = 1 task.insertSource(exposure, sourcei) self._save(mi, 'e') # do something task.removeSource(exposure, sources, sources[sourcei]) self._save(mi, 'f') task.end(exposure, sources) self._save(mi, 'g')
def addCosmicRays(image, nCR=100, emin=800, emax=1000, seed=None): """Add nCR fake cosmic rays to a frame, with pixel values between emin and emax (and some extra associated fainter pixels too)""" # if seed is None: seed = int(afwMath.makeStatistics(image, afwMath.MAX).getValue()) if seed == 0: seed = 1 width = image.getWidth() height = image.getHeight() rand = afwMath.Random(afwMath.Random.RANLUX, seed) for i in range(nCR): # # Initial point in CR # x = rand.uniformInt(width) y = rand.uniformInt(height) amp = emin + rand.uniformInt(emax - emin) # # Extra contamination at about the initial amplitude # badPixels = [] while True: badPixels.append([x, y, amp + 0.1*(emax - emin)*rand.uniform()]) if rand.uniform() > 0.5: break x += rand.uniformInt(3) - 1 y += rand.uniformInt(3) - 1 # # Add a little extra CR flux to the pixels surrounding badPixels # for x, y, amp in badPixels: while rand.uniform() < 0.5: image.set(x, y, amp*rand.uniform()) x += rand.uniformInt(3) - 1 y += rand.uniformInt(3) - 1 # # And set the initial badPixels themselves # for x, y, amp in badPixels: if x >= 0 and x < width and y >= 0 and y < height: image.set(x, y, amp)
def setUp(self): self.min, self.range, self.Q = 0, 5, 20 # asinh width, height = 85, 75 self.images = [] self.images.append(afwImage.ImageF(lsst.geom.ExtentI(width, height))) self.images.append(afwImage.ImageF(lsst.geom.ExtentI(width, height))) self.images.append(afwImage.ImageF(lsst.geom.ExtentI(width, height))) for (x, y, A, g_r, r_i) in [ (15, 15, 1000, 1.0, 2.0), (50, 45, 5500, -1.0, -0.5), (30, 30, 600, 1.0, 2.5), (45, 15, 20000, 1.0, 1.0), ]: for i in range(len(self.images)): if i == B: amp = A elif i == G: amp = A * math.pow(10, 0.4 * g_r) elif i == R: amp = A * math.pow(10, 0.4 * r_i) self.images[i][x, y, afwImage.LOCAL] = amp psf = afwMath.AnalyticKernel(15, 15, afwMath.GaussianFunction2D(2.5, 1.5, 0.5)) convolvedImage = type(self.images[0])(self.images[0].getDimensions()) randomImage = type(self.images[0])(self.images[0].getDimensions()) rand = afwMath.Random("MT19937", 666) convolutionControl = afwMath.ConvolutionControl() convolutionControl.setDoNormalize(True) convolutionControl.setDoCopyEdge(True) for i in range(len(self.images)): afwMath.convolve(convolvedImage, self.images[i], psf, convolutionControl) afwMath.randomGaussianImage(randomImage, rand) randomImage *= 2 convolvedImage += randomImage self.images[i][:] = convolvedImage del convolvedImage del randomImage
def addCosmicRays(image, nCR=100, emin=100, emax=1000): seed = int(afwMath.makeStatistics(image, afwMath.MAX).getValue()) if seed == 0: seed = 1 width = image.getWidth() height = image.getHeight() rand = afwMath.Random(afwMath.Random.RANLUX, seed) for i in range(nCR): # # Initial point in CR # x = rand.uniformInt(width) y = rand.uniformInt(height) amp = emin + rand.uniformInt(emax - emin) badPixels = [] while True: badPixels.append([x, y, amp + 0.1*(emax - emin)*rand.uniform()]) if rand.uniform() > 0.5: break x += rand.uniformInt(3) - 1 y += rand.uniformInt(3) - 1 for x, y, amp in badPixels: if x >= 0 and x < width and y >= 0 and y < height: image.set(x, y, image.get(x,y)+amp) while rand.uniform() < 0.1: image.set(x, y, image.get(x,y)+(emax - emin)*rand.uniform()) x += rand.uniformInt(3) - 1 y += rand.uniformInt(3) - 1 for x, y, amp in badPixels: if x >= 0 and x < width and y >= 0 and y < height: image.set(x, y, image.get(x,y)+amp)
def __init__(self, **kwargs): FakeSourcesTask.__init__(self, **kwargs) print "RNG seed:", self.config.seed self.rng = afwMath.Random(self.config.seed) self.npRand = np.random.RandomState(self.config.seed) self.starData = fits.open(self.config.starList)[1].data
def setUp(self): self.schema = afwTable.SourceTable.makeMinimalSchema() config = measBase.SingleFrameMeasurementConfig() config.algorithms.names = [ "base_PixelFlags", "base_SdssCentroid", "base_GaussianFlux", "base_SdssShape", "base_CircularApertureFlux", "base_PsfFlux", ] config.algorithms["base_CircularApertureFlux"].radii = [3.0] config.slots.centroid = "base_SdssCentroid" config.slots.psfFlux = "base_PsfFlux" config.slots.apFlux = "base_CircularApertureFlux_3_0" config.slots.modelFlux = None config.slots.instFlux = None config.slots.calibFlux = None config.slots.shape = "base_SdssShape" self.measureTask = measBase.SingleFrameMeasurementTask(self.schema, config=config) width, height = 110, 301 self.mi = afwImage.MaskedImageF(afwGeom.ExtentI(width, height)) self.mi.set(0) sd = 3 # standard deviation of image self.mi.getVariance().set(sd * sd) self.mi.getMask().addMaskPlane("DETECTED") self.FWHM = 5 self.ksize = 31 # size of desired kernel sigma1 = 1.75 sigma2 = 2 * sigma1 self.exposure = afwImage.makeExposure(self.mi) self.exposure.setPsf( measAlg.DoubleGaussianPsf(self.ksize, self.ksize, 1.5 * sigma1, 1, 0.1)) self.exposure.setDetector(DetectorWrapper().detector) # # Make a kernel with the exactly correct basis functions. Useful for debugging # basisKernelList = [] for sigma in (sigma1, sigma2): basisKernel = afwMath.AnalyticKernel( self.ksize, self.ksize, afwMath.GaussianFunction2D(sigma, sigma)) basisImage = afwImage.ImageD(basisKernel.getDimensions()) basisKernel.computeImage(basisImage, True) basisImage /= np.sum(basisImage.getArray()) if sigma == sigma1: basisImage0 = basisImage else: basisImage -= basisImage0 basisKernelList.append(afwMath.FixedKernel(basisImage)) order = 1 # 1 => up to linear spFunc = afwMath.PolynomialFunction2D(order) exactKernel = afwMath.LinearCombinationKernel(basisKernelList, spFunc) exactKernel.setSpatialParameters([[1.0, 0, 0], [0.0, 0.5 * 1e-2, 0.2e-2]]) self.exactPsf = measAlg.PcaPsf(exactKernel) rand = afwMath.Random() # make these tests repeatable by setting seed addNoise = True if addNoise: im = self.mi.getImage() afwMath.randomGaussianImage(im, rand) # N(0, 1) im *= sd # N(0, sd^2) del im xarr, yarr = [], [] for x, y in [ (20, 20), (60, 20), (30, 35), (50, 50), (20, 90), (70, 160), (25, 265), (75, 275), (85, 30), (50, 120), (70, 80), (60, 210), (20, 210), ]: xarr.append(x) yarr.append(y) for x, y in zip(xarr, yarr): dx = rand.uniform() - 0.5 # random (centered) offsets dy = rand.uniform() - 0.5 k = exactKernel.getSpatialFunction(1)( x, y) # functional variation of Kernel ... b = (k * sigma1**2 / ((1 - k) * sigma2**2) ) # ... converted double Gaussian's "b" # flux = 80000 - 20*x - 10*(y/float(height))**2 flux = 80000 * (1 + 0.1 * (rand.uniform() - 0.5)) I0 = flux * (1 + b) / (2 * np.pi * (sigma1**2 + b * sigma2**2)) for iy in range(y - self.ksize // 2, y + self.ksize // 2 + 1): if iy < 0 or iy >= self.mi.getHeight(): continue for ix in range(x - self.ksize // 2, x + self.ksize // 2 + 1): if ix < 0 or ix >= self.mi.getWidth(): continue intensity = I0 * psfVal(ix, iy, x + dx, y + dy, sigma1, sigma2, b) Isample = rand.poisson( intensity) if addNoise else intensity self.mi.getImage().set( ix, iy, self.mi.getImage().get(ix, iy) + Isample) self.mi.getVariance().set( ix, iy, self.mi.getVariance().get(ix, iy) + intensity) # bbox = afwGeom.BoxI(afwGeom.PointI(0, 0), afwGeom.ExtentI(width, height)) self.cellSet = afwMath.SpatialCellSet(bbox, 100) self.footprintSet = afwDetection.FootprintSet( self.mi, afwDetection.Threshold(100), "DETECTED") self.catalog = self.measure(self.footprintSet, self.exposure) for source in self.catalog: try: cand = measAlg.makePsfCandidate(source, self.exposure) self.cellSet.insertCandidate(cand) except Exception as e: print(e) continue
import lsst.afw.image as afwImage import lsst.afw.geom as afwGeom import lsst.afw.math as afwMath import lsst.ip.diffim as ipDiffim import numpy as num import lsst.pex.logging as pexLogging import lsst.pex.config as pexConfig import lsst.afw.display.ds9 as ds9 verbosity = 5 pexLogging.Trace_setVerbosity("lsst.ip.diffim", verbosity) imSize = 2**7 kSize = 2**5 + 1 rdm = afwMath.Random(afwMath.Random.MT19937, 10101) scaling = 10000 doNorm = True #gScale = 1.0 # FFT works! gScale = 5.0 # FFT fails!? doAddNoise = False writeFits = False def makeTest1(doAddNoise): gaussian1 = afwMath.GaussianFunction2D(1. * gScale, 1. * gScale, 0.) kernel1 = afwMath.AnalyticKernel(imSize, imSize, gaussian1) image1 = afwImage.ImageD(kernel1.getDimensions()) kernel1.computeImage(image1, doNorm) image1 *= scaling # total counts = scaling image1 = image1.convertF() mask1 = afwImage.MaskU(kernel1.getDimensions())
def showPsfCandidates(exposure, psfCellSet, psf=None, display=None, normalize=True, showBadCandidates=True, fitBasisComponents=False, variance=None, chi=None): """Display the PSF candidates. If psf is provided include PSF model and residuals; if normalize is true normalize the PSFs (and residuals) If chi is True, generate a plot of residuals/sqrt(variance), i.e. chi If fitBasisComponents is true, also find the best linear combination of the PSF's components (if they exist) """ if not display: display = afwDisplay.Display() if chi is None: if variance is not None: # old name for chi chi = variance # # Show us the ccandidates # mos = displayUtils.Mosaic() # candidateCenters = [] candidateCentersBad = [] candidateIndex = 0 for cell in psfCellSet.getCellList(): for cand in cell.begin(False): # include bad candidates rchi2 = cand.getChi2() if rchi2 > 1e100: rchi2 = numpy.nan if not showBadCandidates and cand.isBad(): continue if psf: im_resid = displayUtils.Mosaic(gutter=0, background=-5, mode="x") try: im = cand.getMaskedImage() # copy of this object's image xc, yc = cand.getXCenter(), cand.getYCenter() margin = 0 if True else 5 w, h = im.getDimensions() bbox = lsst.geom.BoxI(lsst.geom.PointI(margin, margin), im.getDimensions()) if margin > 0: bim = im.Factory(w + 2*margin, h + 2*margin) stdev = numpy.sqrt(afwMath.makeStatistics(im.getVariance(), afwMath.MEAN).getValue()) afwMath.randomGaussianImage(bim.getImage(), afwMath.Random()) bim.getVariance().set(stdev**2) bim.assign(im, bbox) im = bim xc += margin yc += margin im = im.Factory(im, True) im.setXY0(cand.getMaskedImage().getXY0()) except Exception: continue if not variance: im_resid.append(im.Factory(im, True)) if True: # tweak up centroids mi = im psfIm = mi.getImage() config = measBase.SingleFrameMeasurementTask.ConfigClass() config.slots.centroid = "base_SdssCentroid" schema = afwTable.SourceTable.makeMinimalSchema() measureSources = measBase.SingleFrameMeasurementTask(schema, config=config) catalog = afwTable.SourceCatalog(schema) extra = 10 # enough margin to run the sdss centroider miBig = mi.Factory(im.getWidth() + 2*extra, im.getHeight() + 2*extra) miBig[extra:-extra, extra:-extra, afwImage.LOCAL] = mi miBig.setXY0(mi.getX0() - extra, mi.getY0() - extra) mi = miBig del miBig exp = afwImage.makeExposure(mi) exp.setPsf(psf) footprintSet = afwDet.FootprintSet(mi, afwDet.Threshold(0.5*numpy.max(psfIm.getArray())), "DETECTED") footprintSet.makeSources(catalog) if len(catalog) == 0: raise RuntimeError("Failed to detect any objects") measureSources.run(catalog, exp) if len(catalog) == 1: source = catalog[0] else: # more than one source; find the once closest to (xc, yc) dmin = None # an invalid value to catch logic errors for i, s in enumerate(catalog): d = numpy.hypot(xc - s.getX(), yc - s.getY()) if i == 0 or d < dmin: source, dmin = s, d xc, yc = source.getCentroid() # residuals using spatial model try: subtractPsf(psf, im, xc, yc) except Exception: continue resid = im if variance: resid = resid.getImage() var = im.getVariance() var = var.Factory(var, True) numpy.sqrt(var.getArray(), var.getArray()) # inplace sqrt resid /= var im_resid.append(resid) # Fit the PSF components directly to the data (i.e. ignoring the spatial model) if fitBasisComponents: im = cand.getMaskedImage() im = im.Factory(im, True) im.setXY0(cand.getMaskedImage().getXY0()) try: noSpatialKernel = psf.getKernel() except Exception: noSpatialKernel = None if noSpatialKernel: candCenter = lsst.geom.PointD(cand.getXCenter(), cand.getYCenter()) fit = fitKernelParamsToImage(noSpatialKernel, im, candCenter) params = fit[0] kernels = afwMath.KernelList(fit[1]) outputKernel = afwMath.LinearCombinationKernel(kernels, params) outImage = afwImage.ImageD(outputKernel.getDimensions()) outputKernel.computeImage(outImage, False) im -= outImage.convertF() resid = im if margin > 0: bim = im.Factory(w + 2*margin, h + 2*margin) afwMath.randomGaussianImage(bim.getImage(), afwMath.Random()) bim *= stdev bim.assign(resid, bbox) resid = bim if variance: resid = resid.getImage() resid /= var im_resid.append(resid) im = im_resid.makeMosaic() else: im = cand.getMaskedImage() if normalize: im /= afwMath.makeStatistics(im, afwMath.MAX).getValue() objId = splitId(cand.getSource().getId(), True)["objId"] if psf: lab = "%d chi^2 %.1f" % (objId, rchi2) ctype = afwDisplay.RED if cand.isBad() else afwDisplay.GREEN else: lab = "%d flux %8.3g" % (objId, cand.getSource().getPsfInstFlux()) ctype = afwDisplay.GREEN mos.append(im, lab, ctype) if False and numpy.isnan(rchi2): display.mtv(cand.getMaskedImage().getImage(), title="showPsfCandidates: candidate") print("amp", cand.getAmplitude()) im = cand.getMaskedImage() center = (candidateIndex, xc - im.getX0(), yc - im.getY0()) candidateIndex += 1 if cand.isBad(): candidateCentersBad.append(center) else: candidateCenters.append(center) if variance: title = "chi(Psf fit)" else: title = "Stars & residuals" mosaicImage = mos.makeMosaic(display=display, title=title) with display.Buffering(): for centers, color in ((candidateCenters, afwDisplay.GREEN), (candidateCentersBad, afwDisplay.RED)): for cen in centers: bbox = mos.getBBox(cen[0]) display.dot("+", cen[1] + bbox.getMinX(), cen[2] + bbox.getMinY(), ctype=color) return mosaicImage
def setUp(self): config = SingleFrameMeasurementTask.ConfigClass() config.slots.apFlux = 'base_CircularApertureFlux_12_0' self.schema = afwTable.SourceTable.makeMinimalSchema() self.measureSources = SingleFrameMeasurementTask(self.schema, config=config) width, height = 110, 301 self.mi = afwImage.MaskedImageF(geom.ExtentI(width, height)) self.mi.set(0) sd = 3 # standard deviation of image self.mi.getVariance().set(sd * sd) self.mi.getMask().addMaskPlane("DETECTED") self.ksize = 31 # size of desired kernel sigma1 = 1.75 sigma2 = 2 * sigma1 self.exposure = afwImage.makeExposure(self.mi) self.exposure.setPsf( measAlg.DoubleGaussianPsf(self.ksize, self.ksize, 1.5 * sigma1, 1, 0.1)) cdMatrix = np.array([1.0, 0.0, 0.0, 1.0]) cdMatrix.shape = (2, 2) wcs = afwGeom.makeSkyWcs(crpix=geom.PointD(0, 0), crval=geom.SpherePoint( 0.0, 0.0, geom.degrees), cdMatrix=cdMatrix) self.exposure.setWcs(wcs) # # Make a kernel with the exactly correct basis functions. # Useful for debugging # basisKernelList = [] for sigma in (sigma1, sigma2): basisKernel = afwMath.AnalyticKernel( self.ksize, self.ksize, afwMath.GaussianFunction2D(sigma, sigma)) basisImage = afwImage.ImageD(basisKernel.getDimensions()) basisKernel.computeImage(basisImage, True) basisImage /= np.sum(basisImage.getArray()) if sigma == sigma1: basisImage0 = basisImage else: basisImage -= basisImage0 basisKernelList.append(afwMath.FixedKernel(basisImage)) order = 1 # 1 => up to linear spFunc = afwMath.PolynomialFunction2D(order) exactKernel = afwMath.LinearCombinationKernel(basisKernelList, spFunc) exactKernel.setSpatialParameters([[1.0, 0, 0], [0.0, 0.5 * 1e-2, 0.2e-2]]) rand = afwMath.Random() # make these tests repeatable by setting seed addNoise = True if addNoise: im = self.mi.getImage() afwMath.randomGaussianImage(im, rand) # N(0, 1) im *= sd # N(0, sd^2) del im xarr, yarr = [], [] for x, y in [ (20, 20), (60, 20), (30, 35), (50, 50), (20, 90), (70, 160), (25, 265), (75, 275), (85, 30), (50, 120), (70, 80), (60, 210), (20, 210), ]: xarr.append(x) yarr.append(y) for x, y in zip(xarr, yarr): dx = rand.uniform() - 0.5 # random (centered) offsets dy = rand.uniform() - 0.5 k = exactKernel.getSpatialFunction(1)( x, y) # functional variation of Kernel ... b = (k * sigma1**2 / ((1 - k) * sigma2**2) ) # ... converted double Gaussian's "b" # flux = 80000 - 20*x - 10*(y/float(height))**2 flux = 80000 * (1 + 0.1 * (rand.uniform() - 0.5)) I0 = flux * (1 + b) / (2 * np.pi * (sigma1**2 + b * sigma2**2)) for iy in range(y - self.ksize // 2, y + self.ksize // 2 + 1): if iy < 0 or iy >= self.mi.getHeight(): continue for ix in range(x - self.ksize // 2, x + self.ksize // 2 + 1): if ix < 0 or ix >= self.mi.getWidth(): continue II = I0 * psfVal(ix, iy, x + dx, y + dy, sigma1, sigma2, b) Isample = rand.poisson(II) if addNoise else II self.mi.image[ix, iy, afwImage.LOCAL] += Isample self.mi.variance[ix, iy, afwImage.LOCAL] += II bbox = geom.BoxI(geom.PointI(0, 0), geom.ExtentI(width, height)) self.cellSet = afwMath.SpatialCellSet(bbox, 100) self.footprintSet = afwDetection.FootprintSet( self.mi, afwDetection.Threshold(100), "DETECTED") self.catalog = self.measure(self.footprintSet, self.exposure) for source in self.catalog: try: cand = measAlg.makePsfCandidate(source, self.exposure) self.cellSet.insertCandidate(cand) except Exception as e: print(e) continue
def setUp(self): width, height = 110, 301 self.mi = afwImage.MaskedImageF(afwGeom.ExtentI(width, height)) self.mi.set(0) sd = 3 # standard deviation of image self.mi.getVariance().set(sd*sd) self.mi.getMask().addMaskPlane("DETECTED") self.FWHM = 5 self.ksize = 31 # size of desired kernel sigma1 = 1.75 sigma2 = 2*sigma1 self.exposure = afwImage.makeExposure(self.mi) self.exposure.setPsf(measAlg.DoubleGaussianPsf(self.ksize, self.ksize, 1.5*sigma1, 1, 0.1)) crval = afwCoord.makeCoord(afwCoord.ICRS, 0.0*afwGeom.degrees, 0.0*afwGeom.degrees) wcs = afwImage.makeWcs(crval, afwGeom.PointD(0, 0), 1.0, 0, 0, 1.0) self.exposure.setWcs(wcs) ccd = cameraGeom.Ccd(cameraGeom.Id(1)) ccd.addAmp(cameraGeom.Amp(cameraGeom.Id(0), afwGeom.BoxI(afwGeom.PointI(0,0), self.exposure.getDimensions()), afwGeom.BoxI(afwGeom.PointI(0,0), afwGeom.ExtentI(0,0)), afwGeom.BoxI(afwGeom.PointI(0,0), self.exposure.getDimensions()), cameraGeom.ElectronicParams(1.0, 100.0, 65535))) self.exposure.setDetector(ccd) self.exposure.getDetector().setDistortion(None) # # Make a kernel with the exactly correct basis functions. Useful for debugging # basisKernelList = afwMath.KernelList() for sigma in (sigma1, sigma2): basisKernel = afwMath.AnalyticKernel(self.ksize, self.ksize, afwMath.GaussianFunction2D(sigma, sigma)) basisImage = afwImage.ImageD(basisKernel.getDimensions()) basisKernel.computeImage(basisImage, True) basisImage /= np.sum(basisImage.getArray()) if sigma == sigma1: basisImage0 = basisImage else: basisImage -= basisImage0 basisKernelList.append(afwMath.FixedKernel(basisImage)) order = 1 # 1 => up to linear spFunc = afwMath.PolynomialFunction2D(order) exactKernel = afwMath.LinearCombinationKernel(basisKernelList, spFunc) exactKernel.setSpatialParameters([[1.0, 0, 0], [0.0, 0.5*1e-2, 0.2e-2]]) rand = afwMath.Random() # make these tests repeatable by setting seed addNoise = True if addNoise: im = self.mi.getImage() afwMath.randomGaussianImage(im, rand) # N(0, 1) im *= sd # N(0, sd^2) del im xarr, yarr = [], [] for x, y in [(20, 20), (60, 20), (30, 35), (50, 50), (20, 90), (70, 160), (25, 265), (75, 275), (85, 30), (50, 120), (70, 80), (60, 210), (20, 210), ]: xarr.append(x) yarr.append(y) for x, y in zip(xarr, yarr): dx = rand.uniform() - 0.5 # random (centered) offsets dy = rand.uniform() - 0.5 k = exactKernel.getSpatialFunction(1)(x, y) # functional variation of Kernel ... b = (k*sigma1**2/((1 - k)*sigma2**2)) # ... converted double Gaussian's "b" #flux = 80000 - 20*x - 10*(y/float(height))**2 flux = 80000*(1 + 0.1*(rand.uniform() - 0.5)) I0 = flux*(1 + b)/(2*np.pi*(sigma1**2 + b*sigma2**2)) for iy in range(y - self.ksize//2, y + self.ksize//2 + 1): if iy < 0 or iy >= self.mi.getHeight(): continue for ix in range(x - self.ksize//2, x + self.ksize//2 + 1): if ix < 0 or ix >= self.mi.getWidth(): continue I = I0*psfVal(ix, iy, x + dx, y + dy, sigma1, sigma2, b) Isample = rand.poisson(I) if addNoise else I self.mi.getImage().set(ix, iy, self.mi.getImage().get(ix, iy) + Isample) self.mi.getVariance().set(ix, iy, self.mi.getVariance().get(ix, iy) + I) # bbox = afwGeom.BoxI(afwGeom.PointI(0,0), afwGeom.ExtentI(width, height)) self.cellSet = afwMath.SpatialCellSet(bbox, 100) self.footprintSet = afwDetection.FootprintSet(self.mi, afwDetection.Threshold(100), "DETECTED") self.catalog = SpatialModelPsfTestCase.measure(self.footprintSet, self.exposure) for source in self.catalog: try: cand = measAlg.makePsfCandidate(source, self.exposure) self.cellSet.insertCandidate(cand) except Exception, e: print e continue
def __init__(self, rand=None): if rand is None: rand = afwMath.Random() self.rand = rand
def getNoiseGenerator(self, exposure, noiseImage, noiseMeanVar, exposureId=None): """Return a generator of artificial noise. Returns ------- noiseGenerator : `lsst.afw.image.noiseReplacer.NoiseGenerator` """ if noiseImage is not None: return ImageNoiseGenerator(noiseImage) rand = None if self.noiseSeedMultiplier: # default plugin, our seed if exposureId is not None and exposureId != 0: seed = exposureId * self.noiseSeedMultiplier else: seed = self.noiseSeedMultiplier rand = afwMath.Random(afwMath.Random.MT19937, seed) if noiseMeanVar is not None: try: # Assume noiseMeanVar is an iterable of floats noiseMean, noiseVar = noiseMeanVar noiseMean = float(noiseMean) noiseVar = float(noiseVar) noiseStd = math.sqrt(noiseVar) if self.log: self.log.debug( 'Using passed-in noise mean = %g, variance = %g -> stdev %g', noiseMean, noiseVar, noiseStd) return FixedGaussianNoiseGenerator(noiseMean, noiseStd, rand=rand) except Exception: if self.log: self.log.debug( 'Failed to cast passed-in noiseMeanVar to floats: %s', str(noiseMeanVar)) offset = self.noiseOffset noiseSource = self.noiseSource if noiseSource == 'meta': # check the exposure metadata meta = exposure.getMetadata() # this key name correspond to SubtractBackgroundTask() in meas_algorithms try: bgMean = meta.getAsDouble('BGMEAN') # We would have to adjust for GAIN if ip_isr didn't make it 1.0 noiseStd = math.sqrt(bgMean) if self.log: self.log.debug( 'Using noise variance = (BGMEAN = %g) from exposure metadata', bgMean) return FixedGaussianNoiseGenerator(offset, noiseStd, rand=rand) except Exception: if self.log: self.log.debug( 'Failed to get BGMEAN from exposure metadata') if noiseSource == 'variance': if self.log: self.log.debug( 'Will draw noise according to the variance plane.') var = exposure.getMaskedImage().getVariance() return VariancePlaneNoiseGenerator(var, mean=offset, rand=rand) if noiseSource == 'variance_median': if self.log: self.log.debug( 'Will draw noise using the median of the variance plane.') var = exposure.getMaskedImage().getVariance() s = afwMath.makeStatistics(var, afwMath.MEDIAN) varMedian = s.getValue(afwMath.MEDIAN) if self.log: self.log.debug("Measured from variance: median variance = %g", varMedian) return FixedGaussianNoiseGenerator(offset, math.sqrt(varMedian), rand=rand) # Compute an image-wide clipped variance. im = exposure.getMaskedImage().getImage() s = afwMath.makeStatistics(im, afwMath.MEANCLIP | afwMath.STDEVCLIP) noiseMean = s.getValue(afwMath.MEANCLIP) noiseStd = s.getValue(afwMath.STDEVCLIP) if self.log: self.log.debug( "Measured from image: clipped mean = %g, stdev = %g", noiseMean, noiseStd) return FixedGaussianNoiseGenerator(noiseMean + offset, noiseStd, rand=rand)
def test2(self): # Check that doReplaceWithNoise works with deblended source # hierarchies. seed = 42 rand = afwMath.Random(afwMath.Random.MT19937, seed) psf = self.getpsf() im = afwImage.ImageF(200, 50) skystd = 100 afwMath.randomGaussianImage(im, rand) im *= skystd imorig = afwImage.ImageF(im, True) noiseim = imorig mi = afwImage.MaskedImageF(im) mi.getVariance().set(skystd**2) exposure = afwImage.makeExposure(mi) exposure.setPsf(psf) detconf = measAlg.SourceDetectionConfig() detconf.returnOriginalFootprints = True detconf.reEstimateBackground = False measconf = measAlg.SourceMeasurementConfig() measconf.doReplaceWithNoise = True measconf.replaceWithNoise.noiseSeed = 42 schema = afwTable.SourceTable.makeMinimalSchema() detect = measAlg.SourceDetectionTask(config=detconf, schema=schema) measure = MySourceMeasurementTask(config=measconf, schema=schema, doplot=plots) table = afwTable.SourceTable.make(schema) table.preallocate(10) # We're going to fake up a perfect deblend hierarchy here, by # creating individual images containing single sources and # measuring them, and then creating a deblend hierarchy where # the children have the correct HeavyFootprints. We want to # find that the measurements on the deblend hierarchy and the # blended image are equal to the individual images. # # Note that in the normal setup we don't expect the # measurements to be *identical* because of the faint wings of # the objects; when measuring a deblended child, we pick up # the wings of the other objects. # # In order to get exactly equal measurements, we'll fake some # sources that have no wings -- we'll copy just the source # pixels within the footprint. This means that all the # footprints are the same, and the pixels inside the footprint # are the same. fullim = None sources = None # "normal" measurements xx0, yy0, vx0, vy0 = [], [], [], [] # "no-wing" measurements xx1, yy1, vx1, vy1 = [], [], [], [] y = 25 for i in range(5): # no-noise source image sim = afwImage.ImageF(imorig.getWidth(), imorig.getHeight()) # Put all four sources in the parent (i==0), and one # source in each child (i=[1 to 4]) if i in [0, 1]: addPsf(sim, psf, 20, y, 1000) if i in [0, 2]: addGaussian(sim, 40, y, 10, 3, 2e5) if i in [0, 3]: addGaussian(sim, 75, y, 10, 3, 2e5) if i in [0, 4]: addPsf(sim, psf, 95, y, 1000) imcopy = afwImage.ImageF(imorig, True) imcopy += sim # copy the pixels into the exposure object im <<= imcopy if i == 0: detected = detect.makeSourceCatalog(table, exposure) sources = detected.sources print 'detected', len(sources), 'sources' self.assertEqual(len(sources), 1) else: fpSets = detect.detectFootprints(exposure) print 'detected', fpSets.numPos, 'sources' fpSets.positive.makeSources(sources) self.assertEqual(fpSets.numPos, 1) print len(sources), 'sources total' measure.plotpat = 'single-%i.png' % i measure.run(exposure, sources[-1:]) s = sources[-1] fp = s.getFootprint() if i == 0: # This is the blended image fullim = imcopy else: print 'Creating heavy footprint...' heavy = afwDet.makeHeavyFootprint(fp, mi) s.setFootprint(heavy) # Record the single-source measurements. xx0.append(s.getX()) yy0.append(s.getY()) vx0.append(s.getIxx()) vy0.append(s.getIyy()) # "no-wings": add just the source pixels within the footprint im <<= sim h = afwDet.makeHeavyFootprint(fp, mi) sim2 = afwImage.ImageF(imorig.getWidth(), imorig.getHeight()) h.insert(sim2) imcopy = afwImage.ImageF(imorig, True) imcopy += sim2 im <<= imcopy measure.plotpat = 'single2-%i.png' % i measure.run(exposure, sources[i:i + 1], noiseImage=noiseim) s = sources[i] xx1.append(s.getX()) yy1.append(s.getY()) vx1.append(s.getIxx()) vy1.append(s.getIyy()) if i == 0: fullim2 = imcopy # Now we'll build the fake deblended hierarchy. parent = sources[0] kids = sources[1:] # Ensure that the parent footprint contains all the child footprints pfp = parent.getFootprint() for s in kids: for span in s.getFootprint().getSpans(): pfp.addSpan(span) pfp.normalize() #parent.setFootprint(pfp) # The parent-child relationship is established through the IDs parentid = parent.getId() for s in kids: s.setParent(parentid) # Reset all the measurements shkey = sources.getTable().getShapeKey() ckey = sources.getTable().getCentroidKey() for s in sources: sh = s.get(shkey) sh.setIxx(np.nan) sh.setIyy(np.nan) sh.setIxy(np.nan) s.set(shkey, sh) c = s.get(ckey) c.setX(np.nan) c.setY(np.nan) s.set(ckey, c) # Measure the "deblended" normal sources im <<= fullim measure.plotpat = 'joint-%(sourcenum)i.png' measure.run(exposure, sources) xx2, yy2, vx2, vy2 = [], [], [], [] for s in sources: xx2.append(s.getX()) yy2.append(s.getY()) vx2.append(s.getIxx()) vy2.append(s.getIyy()) # Measure the "deblended" no-wings sources im <<= fullim2 measure.plotpat = 'joint2-%(sourcenum)i.png' measure.run(exposure, sources, noiseImage=noiseim) xx3, yy3, vx3, vy3 = [], [], [], [] for s in sources: xx3.append(s.getX()) yy3.append(s.getY()) vx3.append(s.getIxx()) vy3.append(s.getIyy()) print 'Normal:' print 'xx ', xx0 print ' vs', xx2 print 'yy ', yy0 print ' vs', yy2 print 'vx ', vx0 print ' vs', vx2 print 'vy ', vy0 print ' vs', vy2 print 'No wings:' print 'xx ', xx1 print ' vs', xx3 print 'yy ', yy1 print ' vs', yy3 print 'vx ', vx1 print ' vs', vx3 print 'vy ', vy1 print ' vs', vy3 # These "normal" tests are not very stringent. # 0.1-pixel centroids self.assertTrue(all([abs(v1 - v2) < 0.1 for v1, v2 in zip(xx0, xx2)])) self.assertTrue(all([abs(v1 - v2) < 0.1 for v1, v2 in zip(yy0, yy2)])) # 10% variances self.assertTrue( all([ abs(v1 - v2) / ((v1 + v2) / 2.) < 0.1 for v1, v2 in zip(vx0, vx2) ])) self.assertTrue( all([ abs(v1 - v2) / ((v1 + v2) / 2.) < 0.1 for v1, v2 in zip(vy0, vy2) ])) # The "no-wings" tests should be exact. self.assertTrue(xx1 == xx3) self.assertTrue(yy1 == yy3) self.assertTrue(vx1 == vx3) self.assertTrue(vy1 == vy3) # Reset sources for s in sources: sh = s.get(shkey) sh.setIxx(np.nan) sh.setIyy(np.nan) sh.setIxy(np.nan) s.set(shkey, sh) c = s.get(ckey) c.setX(np.nan) c.setY(np.nan) s.set(ckey, c) # Test that the parent/child order is unimportant. im <<= fullim2 measure.doplot = False sources2 = sources.copy() perm = [2, 1, 0, 3, 4] for i, j in enumerate(perm): sources2[i] = sources[j] # I'm not convinced that HeavyFootprints get copied correctly... sources2[i].setFootprint(sources[j].getFootprint()) measure.run(exposure, sources2, noiseImage=noiseim) # "measure.run" reorders the sources! xx3, yy3, vx3, vy3 = [], [], [], [] for s in sources: xx3.append(s.getX()) yy3.append(s.getY()) vx3.append(s.getIxx()) vy3.append(s.getIyy()) self.assertTrue(xx1 == xx3) self.assertTrue(yy1 == yy3) self.assertTrue(vx1 == vx3) self.assertTrue(vy1 == vy3) # Reset sources for s in sources: sh = s.get(shkey) sh.setIxx(np.nan) sh.setIyy(np.nan) sh.setIxy(np.nan) s.set(shkey, sh) c = s.get(ckey) c.setX(np.nan) c.setY(np.nan) s.set(ckey, c) # Test that it still works when the parent ID falls in the middle of # the child IDs. im <<= fullim2 measure.doplot = False sources2 = sources.copy() parentid = 3 ids = [parentid, 1, 2, 4, 5] for i, s in enumerate(sources2): s.setId(ids[i]) if i != 0: s.setParent(parentid) s.setFootprint(sources[i].getFootprint()) measure.run(exposure, sources2, noiseImage=noiseim) # The sources get reordered! xx3, yy3, vx3, vy3 = [], [], [], [] xx3, yy3, vx3, vy3 = [0] * 5, [0] * 5, [0] * 5, [0] * 5 for i, j in enumerate(ids): xx3[i] = sources2[j - 1].getX() yy3[i] = sources2[j - 1].getY() vx3[i] = sources2[j - 1].getIxx() vy3[i] = sources2[j - 1].getIyy() self.assertTrue(xx1 == xx3) self.assertTrue(yy1 == yy3) self.assertTrue(vx1 == vx3) self.assertTrue(vy1 == vy3)
def test1(self): seed = 42 rand = afwMath.Random(afwMath.Random.MT19937, seed) #for k in range(5): self.runone(1, rand)
def setUp(self): self.rand = afwMath.Random() self.image = afwImage.ImageF(afwGeom.Extent2I(1000, 1000))