def readMatchListFits(baseName, outputStyle="hsc"): ################################################# # input the matchlist # Unless basename is an instance of str, # it is assumed to be an `file-like' object ################################################ matchList = [] if isinstance(baseName, str): if os.path.exists("%s.match.fits" % (baseName)): fits = pyfits.open("%s.match.fits" % (baseName)) else: fits = None else: # baseName is actually an file-like object fits = pyfits.open(baseName) if fits: hdu = 1 data = fits[hdu].data fits.close() outputs = getOutputList(outputStyle) nOut = len(outputs) for i in range(len(data)): s1 = afwDet.Source() s2 = afwDet.Source() matchList.append(afwDet.SourceMatch(s1, s2, 0.0)) for j in range(nOut): if outputs[j]['label'] in ['ra', 'dec']: s = s1 else: s = s2 setMethod = getattr(s, outputs[j]["set"]) thistype = outputs[j]["pytype"] value = data[i].field(outputs[j]["label"]) if outputs[j]['angle']: setMethod(value * afwGeom.degrees) else: setMethod(thistype(value)) return matchList
def setUp(self): random.seed(1369) ##Load sample input from disk path = os.path.join(eups.productDir("meas_pipeline"), "tests") srcSet = readSourceSet(os.path.join(path, "v695833-e0-c000.xy.txt")) #Make a copy, with different positions #The exact choice doesn't matter ,we just want to make sure the code returns an answer catSet = [] for s in srcSet: s1 = afwDet.Source(s) s1.setXAstrom(s1.getXAstrom() + random.uniform(-.1, .1)) catSet.append(s1) #Make a SourceMatch object maxDist = 1 / 3600. #matches must be this close together srcMatchSet = afwDet.matchXy(catSet, srcSet, maxDist) #Put them on the clipboard filename = pexPolicy.DefaultPolicyFile( "meas_pipeline", "WcsVerificationStageDictionary.paf", "policy") self.policy = pexPolicy.Policy.createPolicy(filename) self.clipboard = pexClipboard.Clipboard() self.clipboard.put(self.policy.get("sourceMatchSetKey"), srcMatchSet) self.clipboard.put(self.policy.get("inputExposureKey"), afwImage.ExposureF(afwGeom.Extent2I(4000, 4000)))
def setUp(self): ##Load sample input from disk path = os.path.join(eups.productDir("meas_pipeline"), "tests") srcSet = readSourceSet(os.path.join(path, "v695833-e0-c000.xy.txt")) #Make a copy, with different fluxes. #The exact choice doesn't matter ,we just want to make sure the code returns an answer #Also need to specify that each source is a star catSet = [] flags = malgUtil.getDetectionFlags() goodFlag = flags['BINNED1'] | flags['STAR'] for s in srcSet: s1 = afwDet.Source(s) s1.setPsfFlux(s1.getPsfFlux() * .281) catSet.append(s1) s.setFlagForDetection(goodFlag) #Make a SourceMatch object maxDist = 1 / 3600. #matches must be this close together srcMatchSet = afwDet.matchXy(catSet, srcSet, maxDist) #Put them on the clipboard filename = pexPolicy.DefaultPolicyFile("meas_pipeline", "PhotoCalStageDictionary.paf", "policy") self.policy = pexPolicy.Policy.createPolicy(filename) self.clipboard = pexClipboard.Clipboard() self.clipboard.put(self.policy.get("sourceMatchSetKey"), srcMatchSet) self.clipboard.put(self.policy.get("inputExposureKey"), afwImage.ExposureF(afwGeom.Extent2I(10, 10)))
def testRadialDistortion(self): for raft in self.camera: for ccdIndex, ccd in enumerate(cameraGeom.cast_Raft(raft)): dist = pipDist.createDistortion(ccd, self.config) size = ccd.getSize() height, width = size.getX(), size.getY() for x, y in ((0.0, 0.0), (0.0, height), (0.0, width), (height, width), (height / 2.0, width / 2.0)): src = afwDet.Source() src.setXAstrom(x) src.setYAstrom(y) forward = dist.actualToIdeal(src) backward = dist.idealToActual(forward) trueForward = bickDistortion(x, y, ccdIndex) self.assertTrue( compare(backward.getXAstrom(), backward.getYAstrom(), x, y, REVERSE_TOL), "Undistorted distorted position is not original: %f,%f vs %f,%f for %d" % (backward.getXAstrom(), backward.getYAstrom(), x, y, ccdIndex)) self.assertTrue( compare(forward.getXAstrom(), forward.getYAstrom(), trueForward[0], trueForward[1], ABSOLUTE_TOL), "Distorted position is not correct: %f,%f vs %f,%f for %d %f,%f" % (forward.getXAstrom(), forward.getYAstrom(), trueForward[0], trueForward[1], ccdIndex, x, y))
def setUp(self): self.sources = afwDet.SourceSet() for i in xrange(1, 10): s = afwDet.Source() s.setXFlux(i * 10.0) s.setXFluxErr(i * 0.01) s.setYFlux(i * 10.0) s.setYFluxErr(i * 0.01) s.setXAstrom(i * 10.0) s.setXAstromErr(i * 0.01) s.setYAstrom(i * 10.0) s.setYAstromErr(i * 0.01) s.setXPeak(i * 10.0) s.setYPeak(i * 10.0) self.sources.append(s) imagePath = os.path.join(eups.productDir("meas_pipeline"), "tests", "v695833-e0-c000-a00.sci_img.fits") self.wcs = afwImage.makeWcs(afwImage.readMetadata(imagePath))
def testValidClipboard(self): dataPolicy1 = pexPolicy.Policy() dataPolicy1.add("inputKey", "sourceSet0") dataPolicy1.add("outputKey", "diaSourceSet0") dataPolicy2 = pexPolicy.Policy() dataPolicy2.add("inputKey", "sourceSet1") dataPolicy2.add("outputKey", "diaSourceSet1") stagePolicy = pexPolicy.Policy() stagePolicy.add("data", dataPolicy1) stagePolicy.add("data", dataPolicy2) stagePolicy.add("ccdWcsKey", "ccdWcs") stagePolicy.add("ampBBoxKey", "ampBBox") sourceSet = afwDet.SourceSet() for i in xrange(5): sourceSet.append(afwDet.Source()) point = afwGeom.Point2D(0.0, 0.0) wcs = afwImage.makeWcs(afwCoord.Coord(point, afwGeom.degrees), point, 1., 0., 0., 1.) ampBBox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1)) tester = SimpleStageTester( measPipe.SourceToDiaSourceStage(stagePolicy)) clipboard = pexClipboard.Clipboard() clipboard.put(dataPolicy1.get("inputKey"), sourceSet) clipboard.put(dataPolicy2.get("inputKey"), sourceSet) clipboard.put(stagePolicy.get("ccdWcsKey"), wcs) clipboard.put(stagePolicy.get("ampBBoxKey"), ampBBox) outWorker = tester.runWorker(clipboard) for policy in stagePolicy.getPolicyArray("data"): assert (outWorker.contains(policy.get("inputKey"))) assert (outWorker.contains(policy.get("outputKey"))) assert (outWorker.contains("persistable_" + policy.get("outputKey"))) diaSourceSet = outWorker.get(policy.get("outputKey")) assert (diaSourceSet.size() == sourceSet.size())
def readSourcesetFromFits(baseName, hdrKeys=[], outputStyle="hsc"): fits = pyfits.open("%s.fits" % (baseName)) hdu = 1 data = fits[hdu].data hdr = fits[0].header fits.close() outputs = getOutputList(outputStyle) nOut = len(outputs) # get any requested hdrs hdrInfo = {} for hdrKey in hdrKeys: hdrInfo[hdrKey] = hdr[hdrKey] sourceSet = afwDet.SourceSet() for i in range(len(data)): source = afwDet.Source() sourceSet.append(source) for j in range(nOut): setMethod = getattr(source, outputs[j]["set"]) thistype = outputs[j]["pytype"] value = data[i].field(outputs[j]["label"]) if outputs[j]['angle']: setMethod(value * afwGeom.degrees) else: try: setMethod(thistype(value)) except: setMethod(thistype("nan")) ################################################# # input the matchlist ################################################ matchList = readMatchListFits(baseName, outputStyle) return sourceSet, matchList, hdrInfo
def readStandards(filename): fd = open(filename, "r") sourceSet = afwDetection.SourceSet() lineno = 0 for line in fd.readlines(): lineno += 1 try: id, flags, ra, dec, cts = line.split() except Exception as e: print("Line %d: %s: %s" % (lineno, e, line), end=' ') s = afwDetection.Source() sourceSet.append(s) s.setId(int(id)) s.setFlagForDetection(int(flags)) s.setRa(float(ra)) s.setDec(float(dec)) s.setPsfFlux(float(cts)) return sourceSet
def readSourceSet(fileName): fd = open(fileName, "r") sourceSet = afwDet.SourceSet() lineno = 0 for line in fd.readlines(): lineno += 1 try: id, x, y, ra, dec, cts, flags = line.split() except Exception, e: print "Line %d: %s: %s" % (lineno, e, line), s = afwDet.Source() sourceSet.append(s) s.setId(int(id)) s.setFlagForDetection(int(flags)) s.setRa(float(ra) * afwGeom.radians) s.setDec(float(dec) * afwGeom.radians) s.setXAstrom(float(x)) s.setYAstrom(float(y)) s.setPsfFlux(float(cts))
def read(self, basename, pixscale = 0.18390): self.exposure = afwImage.ExposureF(basename) fd = open("%s.out" % basename) self.pixscale = pixscale self.sourceList = afwDetection.SourceSet() for line in fd.readlines(): source = afwDetection.Source() self.sourceList.append(source) vals = line.split() i = 0 source.setId(int(vals[i])); i += 1 source.setXAstrom(float(vals[i])); i += 1 source.setXAstromErr(float(vals[i])); i += 1 source.setYAstrom(float(vals[i])); i += 1 source.setYAstromErr(float(vals[i])); i += 1 source.setIxx(float(vals[i])); i += 1 source.setIxy(float(vals[i])); i += 1 source.setIyy(float(vals[i])); i += 1 source.setPsfFlux(float(vals[i])); i += 1 source.setFlagForDetection(int(vals[i], 16)); i += 1
def detectSources(exposure, threshold, psf=None): """Detect sources above positiveThreshold in the provided exposure returning the sourceList """ if not psf: FWHM = 5 psf = algorithms.createPSF("DoubleGaussian", 15, 15, FWHM / (2 * math.sqrt(2 * math.log(2)))) # # Subtract background # mi = exposure.getMaskedImage() bctrl = afwMath.BackgroundControl(afwMath.NATURAL_SPLINE) bctrl.setNxSample(int(mi.getWidth() / 256) + 1) bctrl.setNySample(int(mi.getHeight() / 256) + 1) backobj = afwMath.makeBackground(mi.getImage(), bctrl) img = mi.getImage() img -= backobj.getImageF() del img if display: disp = afwDisplay.Display() disp.mtv(exposure) ds = detectFootprints(exposure, threshold) objects = ds.getFootprints() # # Time to actually measure # measPipelineDir = lsst.utils.getPackageDir('meas_pipeline') moPolicy = policy.Policy.createPolicy( os.path.join(measPipelineDir, "policy", "MeasureSources.paf")) moPolicy = moPolicy.getPolicy("measureObjects") measureSources = algorithms.makeMeasureSources(exposure, moPolicy, psf) sourceList = afwDetection.SourceSet() for i in range(len(objects)): source = afwDetection.Source() sourceList.append(source) source.setId(i) source.setFlagForDetection(source.getFlagForDetection() | algorithms.Flags.BINNED1) try: measureSources.apply(source, objects[i]) except Exception: pass if source.getFlagForDetection() & algorithms.Flags.EDGE: continue if display: xc, yc = source.getXAstrom() - mi.getX0(), source.getYAstrom( ) - mi.getY0() if False: display.dot( "%.1f %d" % (source.getPsfInstFlux(), source.getId()), xc, yc + 1) disp.dot("+", xc, yc, size=1) return sourceList
def checkMatches(srcMatchSet, exposure, log=None): """Check astrometric matches and assess Wcs quality by computing statics over spacial cells in the image. Parameters ---------- srcMatchSet : `list` of `lsst.afw.table.ReferenceMatch` List of matched sources to a reference catalog. exposure : `lsst.afw.image.Exposure` Image the sources in srcMatchSet were detected/measured in. log : `lsst.log.Log` Logger object. Returns ------- values : `dict` Result dictionary with fields: - ``minObjectsPerCell`` : (`int`) - ``maxObjectsPerCell`` : (`int`) - ``meanObjectsPerCell`` : (`float`) - ``stdObjectsPerCell`` : (`float`) """ if not exposure: return {} if log is None: log = Log.getLogger("meas.astrom.verifyWcs.checkMatches") im = exposure.getMaskedImage().getImage() width, height = im.getWidth(), im.getHeight() nx, ny = 3, 3 w, h = width // nx, height // ny if w == 0: w = 1 while nx * w < width: w += 1 if h == 0: h = 1 while ny * h < height: h += 1 cellSet = afwMath.SpatialCellSet( lsst.geom.Box2I(lsst.geom.Point2I(0, 0), lsst.geom.Extent2I(width, height)), w, h) # # Populate cellSet # i = -1 for m in srcMatchSet: i += 1 src = m.second csrc = afwDetection.Source() csrc.setId(i) csrc.setXAstrom(src.getXAstrom()) csrc.setYAstrom(src.getYAstrom()) try: cellSet.insertCandidate( measAlg.PsfCandidateF(csrc, exposure.getMaskedImage())) except Exception as e: log.warn(str(e)) ncell = len(cellSet.getCellList()) nobj = np.ndarray(ncell, dtype='i') for i in range(ncell): cell = cellSet.getCellList()[i] nobj[i] = cell.size() dx = np.ndarray(cell.size()) dy = np.ndarray(cell.size()) j = 0 for cand in cell: # # Swig doesn't know that we're a SpatialCellImageCandidate; all it knows is that we have # a SpatialCellCandidate so we need an explicit (dynamic) cast # mid = cand.getSource().getId() dx[j] = srcMatchSet[mid].first.getXAstrom( ) - srcMatchSet[mid].second.getXAstrom() dy[j] = srcMatchSet[mid].first.getYAstrom( ) - srcMatchSet[mid].second.getYAstrom() j += 1 log.debug("%s %-30s %8s dx,dy = %5.2f,%5.2f rms_x,y = %5.2f,%5.2f", cell.getLabel(), cell.getBBox(), ("nobj=%d" % cell.size()), dx.mean(), dy.mean(), dx.std(), dy.std()) nobj.sort() values = {} values["minObjectsPerCell"] = int( nobj[0]) # otherwise it's a numpy integral type values["maxObjectsPerCell"] = int(nobj[-1]) values["meanObjectsPerCell"] = nobj.mean() values["stdObjectsPerCell"] = nobj.std() return values
def run2(self, exposure, defects=None, background=None): assert exposure is not None, "No exposure provided" print('creating fake PSF...') psf, wcs = self.fakePsf(exposure) print('wcs is', wcs) if False: import lsst.meas.utils.sourceDetection as muDetection import lsst.meas.utils.sourceMeasurement as muMeasurement # phot.py : detect() print('Simulating detect()...') policy = self.config['detect'] posSources, negSources = muDetection.detectSources( exposure, psf, policy.getPolicy()) numPos = len( posSources.getFootprints()) if posSources is not None else 0 numNeg = len( negSources.getFootprints()) if negSources is not None else 0 print('found', numPos, 'pos and', numNeg, 'neg footprints') footprintSet = posSources # phot.py : measure() print('Simulating measure()...') policy = self.config['measure'].getPolicy() footprints = [] num = len(footprintSet.getFootprints()) self.log.log(self.log.INFO, "Measuring %d positive footprints" % num) footprints.append([footprintSet.getFootprints(), False]) sources = muMeasurement.sourceMeasurement(exposure, psf, footprints, policy) print('sources:', sources) for s in sources: print(' ', s, s.getXAstrom(), s.getYAstrom(), s.getPsfFlux(), s.getIxx(), s.getIyy(), s.getIxy()) # sourceMeasurement(): import lsst.meas.algorithms as measAlg import lsst.afw.detection as afwDetection print('Simulating sourceMeasurement()...') exposure.setPsf(psf) measureSources = measAlg.makeMeasureSources(exposure, policy) # print('ms policy', str(measureSources.getPolicy())) # print('ms astrom:', measureSources.getMeasureAstrom()) # print('ms photom:', measureSources.getMeasurePhotom()) # print('ms shape:', measureSources.getMeasureShape()) F = footprints[0][0] for i in range(len(F)): # create a new source, and add it to the list, initialize ... s = afwDetection.Source() f = F[i] print('measuring footprint', f) measureSources.apply(s, f) print('got', s) print(' ', s, s.getXAstrom(), s.getYAstrom(), s.getPsfFlux(), s.getIxx(), s.getIyy(), s.getIxy()) print('initial photometry...') sources, footprints = self.phot(exposure, psf) # print('got sources', str(sources)) # print('got footprints', str(footprints)) # print('sources:', sources) print('got sources:', len(sources)) for s in sources: print(' ', s, s.getXAstrom(), s.getYAstrom(), s.getPsfFlux(), s.getIxx(), s.getIyy(), s.getIxy()) print('re-photometry...') sources = self.rephot(exposure, footprints, psf) # print('got sources', str(sources)) print('sources:', len(sources)) for s in sources: print(' ', s, s.getXAstrom(), s.getYAstrom(), s.getPsfFlux(), s.getIxx(), s.getIyy(), s.getIxy()) return psf, sources, footprints
def checkMatches(srcMatchSet, exposure, log=None): if not exposure: return {} if log is None: log = Log.getLogger("meas.astrom.verifyWcs.checkMatches") im = exposure.getMaskedImage().getImage() width, height = im.getWidth(), im.getHeight() nx, ny = 3, 3 w, h = width // nx, height // ny if w == 0: w = 1 while nx * w < width: w += 1 if h == 0: h = 1 while ny * h < height: h += 1 cellSet = afwMath.SpatialCellSet( afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(width, height)), w, h) # # Populate cellSet # i = -1 for m in srcMatchSet: i += 1 src = m.second csrc = afwDetection.Source() csrc.setId(i) csrc.setXAstrom(src.getXAstrom()) csrc.setYAstrom(src.getYAstrom()) try: cellSet.insertCandidate( measAlg.PsfCandidateF(csrc, exposure.getMaskedImage())) except Exception as e: log.warn(str(e)) ncell = len(cellSet.getCellList()) nobj = np.ndarray(ncell, dtype='i') for i in range(ncell): cell = cellSet.getCellList()[i] nobj[i] = cell.size() dx = np.ndarray(cell.size()) dy = np.ndarray(cell.size()) j = 0 for cand in cell: # # Swig doesn't know that we're a SpatialCellImageCandidate; all it knows is that we have # a SpatialCellCandidate so we need an explicit (dynamic) cast # mid = cand.getSource().getId() dx[j] = srcMatchSet[mid].first.getXAstrom( ) - srcMatchSet[mid].second.getXAstrom() dy[j] = srcMatchSet[mid].first.getYAstrom( ) - srcMatchSet[mid].second.getYAstrom() j += 1 log.debug("%s %-30s %8s dx,dy = %5.2f,%5.2f rms_x,y = %5.2f,%5.2f", cell.getLabel(), cell.getBBox(), ("nobj=%d" % cell.size()), dx.mean(), dy.mean(), dx.std(), dy.std()) nobj.sort() values = {} values["minObjectsPerCell"] = int( nobj[0]) # otherwise it's a numpy integral type values["maxObjectsPerCell"] = int(nobj[-1]) values["meanObjectsPerCell"] = nobj.mean() values["stdObjectsPerCell"] = nobj.std() return values
def measure(self): """Detect and measure sources""" mi = self.exposure.getMaskedImage() # # We do a pretty good job of interpolating, so don't propagagate the convolved CR/INTRP bits # (we'll keep them for the original CR/INTRP pixels) # savedMask = mi.getMask().Factory(mi.getMask(), True) saveBits = savedMask.getPlaneBitMask("CR") | \ savedMask.getPlaneBitMask("BAD") | \ savedMask.getPlaneBitMask("INTRP") # Bits to not convolve savedMask &= saveBits msk = mi.getMask(); msk &= ~saveBits; del msk # Clear the saved bits # # Smooth image # cnvImage = mi.Factory(mi.getBBox(afwImage.PARENT)) afwMath.convolve(cnvImage, mi, self.psf.getKernel(), afwMath.ConvolutionControl()) msk = cnvImage.getMask(); msk |= savedMask; del msk # restore the saved bits threshold = afwDetection.Threshold(3, afwDetection.Threshold.STDEV) # # Only search the part of the frame that was PSF-smoothed # llc = afwGeom.PointI(self.psf.getKernel().getWidth()/2, self.psf.getKernel().getHeight()/2) urc = afwGeom.PointI(cnvImage.getWidth() - 1, cnvImage.getHeight() - 1) - afwGeom.ExtentI(llc[0], llc[1]); middle = cnvImage.Factory(cnvImage, afwGeom.BoxI(llc, urc), afwImage.LOCAL) ds = afwDetection.FootprintSetF(middle, threshold, "DETECTED") del middle # # ds only searched the middle but it belongs to the entire MaskedImage # ds.setRegion(mi.getBBox(afwImage.PARENT)) # # We want to grow the detections into the edge by at least one pixel so that it sees the EDGE bit # grow, isotropic = 1, False ds = afwDetection.FootprintSetF(ds, grow, isotropic) ds.setMask(mi.getMask(), "DETECTED") # # Reinstate the saved (e.g. BAD) (and also the DETECTED | EDGE) bits in the unsmoothed image # savedMask <<= cnvImage.getMask() msk = mi.getMask(); msk |= savedMask; del msk del savedMask; savedMask = None #msk = mi.getMask(); msk &= ~0x10; del msk # XXXX if self.display: ds9.mtv(mi, frame = 0, lowOrderBits = True) ds9.mtv(cnvImage, frame = 1) objects = ds.getFootprints() # # Time to actually measure # msPolicy = policy.Policy.createPolicy(policy.DefaultPolicyFile("meas_algorithms", "examples/measureSources.paf")) msPolicy = msPolicy.getPolicy("measureSources") measureSources = measAlg.makeMeasureSources(self.exposure, msPolicy) self.sourceList = afwDetection.SourceSet() for i in range(len(objects)): source = afwDetection.Source() self.sourceList.append(source) source.setId(i) source.setFlagForDetection(source.getFlagForDetection() | measAlg.Flags.BINNED1); try: measureSources.apply(source, objects[i]) except Exception, e: try: print e except Exception, ee: print ee