def readMatchListFits(baseName, outputStyle="hsc"):
#################################################
# input the matchlist
# Unless basename is an instance of str,
# it is assumed to be an `file-like' object
################################################
matchList = []
if isinstance(baseName, str):
if os.path.exists("%s.match.fits" % (baseName)):
fits = pyfits.open("%s.match.fits" % (baseName))
else:
fits = None
else:
# baseName is actually an file-like object
fits = pyfits.open(baseName)
if fits:
hdu = 1
data = fits[hdu].data
fits.close()
outputs = getOutputList(outputStyle)
nOut = len(outputs)
for i in range(len(data)):
s1 = afwDet.Source()
s2 = afwDet.Source()
matchList.append(afwDet.SourceMatch(s1, s2, 0.0))
for j in range(nOut):
if outputs[j]['label'] in ['ra', 'dec']:
s = s1
else:
s = s2
setMethod = getattr(s, outputs[j]["set"])
thistype = outputs[j]["pytype"]
value = data[i].field(outputs[j]["label"])
if outputs[j]['angle']:
setMethod(value * afwGeom.degrees)
else:
setMethod(thistype(value))
return matchList
def setUp(self):
random.seed(1369)
##Load sample input from disk
path = os.path.join(eups.productDir("meas_pipeline"), "tests")
srcSet = readSourceSet(os.path.join(path, "v695833-e0-c000.xy.txt"))
#Make a copy, with different positions
#The exact choice doesn't matter ,we just want to make sure the code returns an answer
catSet = []
for s in srcSet:
s1 = afwDet.Source(s)
s1.setXAstrom(s1.getXAstrom() + random.uniform(-.1, .1))
catSet.append(s1)
#Make a SourceMatch object
maxDist = 1 / 3600. #matches must be this close together
srcMatchSet = afwDet.matchXy(catSet, srcSet, maxDist)
#Put them on the clipboard
filename = pexPolicy.DefaultPolicyFile(
"meas_pipeline", "WcsVerificationStageDictionary.paf", "policy")
self.policy = pexPolicy.Policy.createPolicy(filename)
self.clipboard = pexClipboard.Clipboard()
self.clipboard.put(self.policy.get("sourceMatchSetKey"), srcMatchSet)
self.clipboard.put(self.policy.get("inputExposureKey"),
afwImage.ExposureF(afwGeom.Extent2I(4000, 4000)))
def setUp(self):
##Load sample input from disk
path = os.path.join(eups.productDir("meas_pipeline"), "tests")
srcSet = readSourceSet(os.path.join(path, "v695833-e0-c000.xy.txt"))
#Make a copy, with different fluxes.
#The exact choice doesn't matter ,we just want to make sure the code returns an answer
#Also need to specify that each source is a star
catSet = []
flags = malgUtil.getDetectionFlags()
goodFlag = flags['BINNED1'] | flags['STAR']
for s in srcSet:
s1 = afwDet.Source(s)
s1.setPsfFlux(s1.getPsfFlux() * .281)
catSet.append(s1)
s.setFlagForDetection(goodFlag)
#Make a SourceMatch object
maxDist = 1 / 3600. #matches must be this close together
srcMatchSet = afwDet.matchXy(catSet, srcSet, maxDist)
#Put them on the clipboard
filename = pexPolicy.DefaultPolicyFile("meas_pipeline",
"PhotoCalStageDictionary.paf",
"policy")
self.policy = pexPolicy.Policy.createPolicy(filename)
self.clipboard = pexClipboard.Clipboard()
self.clipboard.put(self.policy.get("sourceMatchSetKey"), srcMatchSet)
self.clipboard.put(self.policy.get("inputExposureKey"),
afwImage.ExposureF(afwGeom.Extent2I(10, 10)))
def testRadialDistortion(self):
for raft in self.camera:
for ccdIndex, ccd in enumerate(cameraGeom.cast_Raft(raft)):
dist = pipDist.createDistortion(ccd, self.config)
size = ccd.getSize()
height, width = size.getX(), size.getY()
for x, y in ((0.0, 0.0), (0.0, height), (0.0, width),
(height, width), (height / 2.0, width / 2.0)):
src = afwDet.Source()
src.setXAstrom(x)
src.setYAstrom(y)
forward = dist.actualToIdeal(src)
backward = dist.idealToActual(forward)
trueForward = bickDistortion(x, y, ccdIndex)
self.assertTrue(
compare(backward.getXAstrom(), backward.getYAstrom(),
x, y, REVERSE_TOL),
"Undistorted distorted position is not original: %f,%f vs %f,%f for %d"
% (backward.getXAstrom(), backward.getYAstrom(), x, y,
ccdIndex))
self.assertTrue(
compare(forward.getXAstrom(), forward.getYAstrom(),
trueForward[0], trueForward[1], ABSOLUTE_TOL),
"Distorted position is not correct: %f,%f vs %f,%f for %d %f,%f"
% (forward.getXAstrom(), forward.getYAstrom(),
trueForward[0], trueForward[1], ccdIndex, x, y))
def setUp(self):
self.sources = afwDet.SourceSet()
for i in xrange(1, 10):
s = afwDet.Source()
s.setXFlux(i * 10.0)
s.setXFluxErr(i * 0.01)
s.setYFlux(i * 10.0)
s.setYFluxErr(i * 0.01)
s.setXAstrom(i * 10.0)
s.setXAstromErr(i * 0.01)
s.setYAstrom(i * 10.0)
s.setYAstromErr(i * 0.01)
s.setXPeak(i * 10.0)
s.setYPeak(i * 10.0)
self.sources.append(s)
imagePath = os.path.join(eups.productDir("meas_pipeline"), "tests",
"v695833-e0-c000-a00.sci_img.fits")
self.wcs = afwImage.makeWcs(afwImage.readMetadata(imagePath))
def testValidClipboard(self):
dataPolicy1 = pexPolicy.Policy()
dataPolicy1.add("inputKey", "sourceSet0")
dataPolicy1.add("outputKey", "diaSourceSet0")
dataPolicy2 = pexPolicy.Policy()
dataPolicy2.add("inputKey", "sourceSet1")
dataPolicy2.add("outputKey", "diaSourceSet1")
stagePolicy = pexPolicy.Policy()
stagePolicy.add("data", dataPolicy1)
stagePolicy.add("data", dataPolicy2)
stagePolicy.add("ccdWcsKey", "ccdWcs")
stagePolicy.add("ampBBoxKey", "ampBBox")
sourceSet = afwDet.SourceSet()
for i in xrange(5):
sourceSet.append(afwDet.Source())
point = afwGeom.Point2D(0.0, 0.0)
wcs = afwImage.makeWcs(afwCoord.Coord(point, afwGeom.degrees), point,
1., 0., 0., 1.)
ampBBox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(1, 1))
tester = SimpleStageTester(
measPipe.SourceToDiaSourceStage(stagePolicy))
clipboard = pexClipboard.Clipboard()
clipboard.put(dataPolicy1.get("inputKey"), sourceSet)
clipboard.put(dataPolicy2.get("inputKey"), sourceSet)
clipboard.put(stagePolicy.get("ccdWcsKey"), wcs)
clipboard.put(stagePolicy.get("ampBBoxKey"), ampBBox)
outWorker = tester.runWorker(clipboard)
for policy in stagePolicy.getPolicyArray("data"):
assert (outWorker.contains(policy.get("inputKey")))
assert (outWorker.contains(policy.get("outputKey")))
assert (outWorker.contains("persistable_" +
policy.get("outputKey")))
diaSourceSet = outWorker.get(policy.get("outputKey"))
assert (diaSourceSet.size() == sourceSet.size())
def readSourcesetFromFits(baseName, hdrKeys=[], outputStyle="hsc"):
fits = pyfits.open("%s.fits" % (baseName))
hdu = 1
data = fits[hdu].data
hdr = fits[0].header
fits.close()
outputs = getOutputList(outputStyle)
nOut = len(outputs)
# get any requested hdrs
hdrInfo = {}
for hdrKey in hdrKeys:
hdrInfo[hdrKey] = hdr[hdrKey]
sourceSet = afwDet.SourceSet()
for i in range(len(data)):
source = afwDet.Source()
sourceSet.append(source)
for j in range(nOut):
setMethod = getattr(source, outputs[j]["set"])
thistype = outputs[j]["pytype"]
value = data[i].field(outputs[j]["label"])
if outputs[j]['angle']:
setMethod(value * afwGeom.degrees)
else:
try:
setMethod(thistype(value))
except:
setMethod(thistype("nan"))
#################################################
# input the matchlist
################################################
matchList = readMatchListFits(baseName, outputStyle)
return sourceSet, matchList, hdrInfo
def readStandards(filename):
fd = open(filename, "r")
sourceSet = afwDetection.SourceSet()
lineno = 0
for line in fd.readlines():
lineno += 1
try:
id, flags, ra, dec, cts = line.split()
except Exception as e:
print("Line %d: %s: %s" % (lineno, e, line), end=' ')
s = afwDetection.Source()
sourceSet.append(s)
s.setId(int(id))
s.setFlagForDetection(int(flags))
s.setRa(float(ra))
s.setDec(float(dec))
s.setPsfFlux(float(cts))
return sourceSet
def readSourceSet(fileName):
fd = open(fileName, "r")
sourceSet = afwDet.SourceSet()
lineno = 0
for line in fd.readlines():
lineno += 1
try:
id, x, y, ra, dec, cts, flags = line.split()
except Exception, e:
print "Line %d: %s: %s" % (lineno, e, line),
s = afwDet.Source()
sourceSet.append(s)
s.setId(int(id))
s.setFlagForDetection(int(flags))
s.setRa(float(ra) * afwGeom.radians)
s.setDec(float(dec) * afwGeom.radians)
s.setXAstrom(float(x))
s.setYAstrom(float(y))
s.setPsfFlux(float(cts))
def read(self, basename, pixscale = 0.18390):
self.exposure = afwImage.ExposureF(basename)
fd = open("%s.out" % basename)
self.pixscale = pixscale
self.sourceList = afwDetection.SourceSet()
for line in fd.readlines():
source = afwDetection.Source()
self.sourceList.append(source)
vals = line.split()
i = 0
source.setId(int(vals[i])); i += 1
source.setXAstrom(float(vals[i])); i += 1
source.setXAstromErr(float(vals[i])); i += 1
source.setYAstrom(float(vals[i])); i += 1
source.setYAstromErr(float(vals[i])); i += 1
source.setIxx(float(vals[i])); i += 1
source.setIxy(float(vals[i])); i += 1
source.setIyy(float(vals[i])); i += 1
source.setPsfFlux(float(vals[i])); i += 1
source.setFlagForDetection(int(vals[i], 16)); i += 1
def detectSources(exposure, threshold, psf=None):
"""Detect sources above positiveThreshold in the provided exposure returning the sourceList
"""
if not psf:
FWHM = 5
psf = algorithms.createPSF("DoubleGaussian", 15, 15,
FWHM / (2 * math.sqrt(2 * math.log(2))))
#
# Subtract background
#
mi = exposure.getMaskedImage()
bctrl = afwMath.BackgroundControl(afwMath.NATURAL_SPLINE)
bctrl.setNxSample(int(mi.getWidth() / 256) + 1)
bctrl.setNySample(int(mi.getHeight() / 256) + 1)
backobj = afwMath.makeBackground(mi.getImage(), bctrl)
img = mi.getImage()
img -= backobj.getImageF()
del img
if display:
disp = afwDisplay.Display()
disp.mtv(exposure)
ds = detectFootprints(exposure, threshold)
objects = ds.getFootprints()
#
# Time to actually measure
#
measPipelineDir = lsst.utils.getPackageDir('meas_pipeline')
moPolicy = policy.Policy.createPolicy(
os.path.join(measPipelineDir, "policy", "MeasureSources.paf"))
moPolicy = moPolicy.getPolicy("measureObjects")
measureSources = algorithms.makeMeasureSources(exposure, moPolicy, psf)
sourceList = afwDetection.SourceSet()
for i in range(len(objects)):
source = afwDetection.Source()
sourceList.append(source)
source.setId(i)
source.setFlagForDetection(source.getFlagForDetection()
| algorithms.Flags.BINNED1)
try:
measureSources.apply(source, objects[i])
except Exception:
pass
if source.getFlagForDetection() & algorithms.Flags.EDGE:
continue
if display:
xc, yc = source.getXAstrom() - mi.getX0(), source.getYAstrom(
) - mi.getY0()
if False:
display.dot(
"%.1f %d" % (source.getPsfInstFlux(), source.getId()), xc,
yc + 1)
disp.dot("+", xc, yc, size=1)
return sourceList
def checkMatches(srcMatchSet, exposure, log=None):
"""Check astrometric matches and assess Wcs quality by computing statics
over spacial cells in the image.
Parameters
----------
srcMatchSet : `list` of `lsst.afw.table.ReferenceMatch`
List of matched sources to a reference catalog.
exposure : `lsst.afw.image.Exposure`
Image the sources in srcMatchSet were detected/measured in.
log : `lsst.log.Log`
Logger object.
Returns
-------
values : `dict`
Result dictionary with fields:
- ``minObjectsPerCell`` : (`int`)
- ``maxObjectsPerCell`` : (`int`)
- ``meanObjectsPerCell`` : (`float`)
- ``stdObjectsPerCell`` : (`float`)
"""
if not exposure:
return {}
if log is None:
log = Log.getLogger("meas.astrom.verifyWcs.checkMatches")
im = exposure.getMaskedImage().getImage()
width, height = im.getWidth(), im.getHeight()
nx, ny = 3, 3
w, h = width // nx, height // ny
if w == 0:
w = 1
while nx * w < width:
w += 1
if h == 0:
h = 1
while ny * h < height:
h += 1
cellSet = afwMath.SpatialCellSet(
lsst.geom.Box2I(lsst.geom.Point2I(0, 0),
lsst.geom.Extent2I(width, height)), w, h)
#
# Populate cellSet
#
i = -1
for m in srcMatchSet:
i += 1
src = m.second
csrc = afwDetection.Source()
csrc.setId(i)
csrc.setXAstrom(src.getXAstrom())
csrc.setYAstrom(src.getYAstrom())
try:
cellSet.insertCandidate(
measAlg.PsfCandidateF(csrc, exposure.getMaskedImage()))
except Exception as e:
log.warn(str(e))
ncell = len(cellSet.getCellList())
nobj = np.ndarray(ncell, dtype='i')
for i in range(ncell):
cell = cellSet.getCellList()[i]
nobj[i] = cell.size()
dx = np.ndarray(cell.size())
dy = np.ndarray(cell.size())
j = 0
for cand in cell:
#
# Swig doesn't know that we're a SpatialCellImageCandidate; all it knows is that we have
# a SpatialCellCandidate so we need an explicit (dynamic) cast
#
mid = cand.getSource().getId()
dx[j] = srcMatchSet[mid].first.getXAstrom(
) - srcMatchSet[mid].second.getXAstrom()
dy[j] = srcMatchSet[mid].first.getYAstrom(
) - srcMatchSet[mid].second.getYAstrom()
j += 1
log.debug("%s %-30s %8s dx,dy = %5.2f,%5.2f rms_x,y = %5.2f,%5.2f",
cell.getLabel(), cell.getBBox(), ("nobj=%d" % cell.size()),
dx.mean(), dy.mean(), dx.std(), dy.std())
nobj.sort()
values = {}
values["minObjectsPerCell"] = int(
nobj[0]) # otherwise it's a numpy integral type
values["maxObjectsPerCell"] = int(nobj[-1])
values["meanObjectsPerCell"] = nobj.mean()
values["stdObjectsPerCell"] = nobj.std()
return values
def run2(self, exposure, defects=None, background=None):
assert exposure is not None, "No exposure provided"
print('creating fake PSF...')
psf, wcs = self.fakePsf(exposure)
print('wcs is', wcs)
if False:
import lsst.meas.utils.sourceDetection as muDetection
import lsst.meas.utils.sourceMeasurement as muMeasurement
# phot.py : detect()
print('Simulating detect()...')
policy = self.config['detect']
posSources, negSources = muDetection.detectSources(
exposure, psf, policy.getPolicy())
numPos = len(
posSources.getFootprints()) if posSources is not None else 0
numNeg = len(
negSources.getFootprints()) if negSources is not None else 0
print('found', numPos, 'pos and', numNeg, 'neg footprints')
footprintSet = posSources
# phot.py : measure()
print('Simulating measure()...')
policy = self.config['measure'].getPolicy()
footprints = []
num = len(footprintSet.getFootprints())
self.log.log(self.log.INFO,
"Measuring %d positive footprints" % num)
footprints.append([footprintSet.getFootprints(), False])
sources = muMeasurement.sourceMeasurement(exposure, psf,
footprints, policy)
print('sources:', sources)
for s in sources:
print(' ', s, s.getXAstrom(), s.getYAstrom(), s.getPsfFlux(),
s.getIxx(), s.getIyy(), s.getIxy())
# sourceMeasurement():
import lsst.meas.algorithms as measAlg
import lsst.afw.detection as afwDetection
print('Simulating sourceMeasurement()...')
exposure.setPsf(psf)
measureSources = measAlg.makeMeasureSources(exposure, policy)
# print('ms policy', str(measureSources.getPolicy()))
# print('ms astrom:', measureSources.getMeasureAstrom())
# print('ms photom:', measureSources.getMeasurePhotom())
# print('ms shape:', measureSources.getMeasureShape())
F = footprints[0][0]
for i in range(len(F)):
# create a new source, and add it to the list, initialize ...
s = afwDetection.Source()
f = F[i]
print('measuring footprint', f)
measureSources.apply(s, f)
print('got', s)
print(' ', s, s.getXAstrom(), s.getYAstrom(), s.getPsfFlux(),
s.getIxx(), s.getIyy(), s.getIxy())
print('initial photometry...')
sources, footprints = self.phot(exposure, psf)
# print('got sources', str(sources))
# print('got footprints', str(footprints))
# print('sources:', sources)
print('got sources:', len(sources))
for s in sources:
print(' ', s, s.getXAstrom(), s.getYAstrom(), s.getPsfFlux(),
s.getIxx(), s.getIyy(), s.getIxy())
print('re-photometry...')
sources = self.rephot(exposure, footprints, psf)
# print('got sources', str(sources))
print('sources:', len(sources))
for s in sources:
print(' ', s, s.getXAstrom(), s.getYAstrom(), s.getPsfFlux(),
s.getIxx(), s.getIyy(), s.getIxy())
return psf, sources, footprints
def checkMatches(srcMatchSet, exposure, log=None):
if not exposure:
return {}
if log is None:
log = Log.getLogger("meas.astrom.verifyWcs.checkMatches")
im = exposure.getMaskedImage().getImage()
width, height = im.getWidth(), im.getHeight()
nx, ny = 3, 3
w, h = width // nx, height // ny
if w == 0:
w = 1
while nx * w < width:
w += 1
if h == 0:
h = 1
while ny * h < height:
h += 1
cellSet = afwMath.SpatialCellSet(
afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(width, height)),
w, h)
#
# Populate cellSet
#
i = -1
for m in srcMatchSet:
i += 1
src = m.second
csrc = afwDetection.Source()
csrc.setId(i)
csrc.setXAstrom(src.getXAstrom())
csrc.setYAstrom(src.getYAstrom())
try:
cellSet.insertCandidate(
measAlg.PsfCandidateF(csrc, exposure.getMaskedImage()))
except Exception as e:
log.warn(str(e))
ncell = len(cellSet.getCellList())
nobj = np.ndarray(ncell, dtype='i')
for i in range(ncell):
cell = cellSet.getCellList()[i]
nobj[i] = cell.size()
dx = np.ndarray(cell.size())
dy = np.ndarray(cell.size())
j = 0
for cand in cell:
#
# Swig doesn't know that we're a SpatialCellImageCandidate; all it knows is that we have
# a SpatialCellCandidate so we need an explicit (dynamic) cast
#
mid = cand.getSource().getId()
dx[j] = srcMatchSet[mid].first.getXAstrom(
) - srcMatchSet[mid].second.getXAstrom()
dy[j] = srcMatchSet[mid].first.getYAstrom(
) - srcMatchSet[mid].second.getYAstrom()
j += 1
log.debug("%s %-30s %8s dx,dy = %5.2f,%5.2f rms_x,y = %5.2f,%5.2f",
cell.getLabel(), cell.getBBox(), ("nobj=%d" % cell.size()),
dx.mean(), dy.mean(), dx.std(), dy.std())
nobj.sort()
values = {}
values["minObjectsPerCell"] = int(
nobj[0]) # otherwise it's a numpy integral type
values["maxObjectsPerCell"] = int(nobj[-1])
values["meanObjectsPerCell"] = nobj.mean()
values["stdObjectsPerCell"] = nobj.std()
return values
def measure(self):
"""Detect and measure sources"""
mi = self.exposure.getMaskedImage()
#
# We do a pretty good job of interpolating, so don't propagagate the convolved CR/INTRP bits
# (we'll keep them for the original CR/INTRP pixels)
#
savedMask = mi.getMask().Factory(mi.getMask(), True)
saveBits = savedMask.getPlaneBitMask("CR") | \
savedMask.getPlaneBitMask("BAD") | \
savedMask.getPlaneBitMask("INTRP") # Bits to not convolve
savedMask &= saveBits
msk = mi.getMask(); msk &= ~saveBits; del msk # Clear the saved bits
#
# Smooth image
#
cnvImage = mi.Factory(mi.getBBox(afwImage.PARENT))
afwMath.convolve(cnvImage, mi, self.psf.getKernel(), afwMath.ConvolutionControl())
msk = cnvImage.getMask(); msk |= savedMask; del msk # restore the saved bits
threshold = afwDetection.Threshold(3, afwDetection.Threshold.STDEV)
#
# Only search the part of the frame that was PSF-smoothed
#
llc = afwGeom.PointI(self.psf.getKernel().getWidth()/2, self.psf.getKernel().getHeight()/2)
urc = afwGeom.PointI(cnvImage.getWidth() - 1, cnvImage.getHeight() - 1) - afwGeom.ExtentI(llc[0], llc[1]);
middle = cnvImage.Factory(cnvImage, afwGeom.BoxI(llc, urc), afwImage.LOCAL)
ds = afwDetection.FootprintSetF(middle, threshold, "DETECTED")
del middle
#
# ds only searched the middle but it belongs to the entire MaskedImage
#
ds.setRegion(mi.getBBox(afwImage.PARENT))
#
# We want to grow the detections into the edge by at least one pixel so that it sees the EDGE bit
#
grow, isotropic = 1, False
ds = afwDetection.FootprintSetF(ds, grow, isotropic)
ds.setMask(mi.getMask(), "DETECTED")
#
# Reinstate the saved (e.g. BAD) (and also the DETECTED | EDGE) bits in the unsmoothed image
#
savedMask <<= cnvImage.getMask()
msk = mi.getMask(); msk |= savedMask; del msk
del savedMask; savedMask = None
#msk = mi.getMask(); msk &= ~0x10; del msk # XXXX
if self.display:
ds9.mtv(mi, frame = 0, lowOrderBits = True)
ds9.mtv(cnvImage, frame = 1)
objects = ds.getFootprints()
#
# Time to actually measure
#
msPolicy = policy.Policy.createPolicy(policy.DefaultPolicyFile("meas_algorithms",
"examples/measureSources.paf"))
msPolicy = msPolicy.getPolicy("measureSources")
measureSources = measAlg.makeMeasureSources(self.exposure, msPolicy)
self.sourceList = afwDetection.SourceSet()
for i in range(len(objects)):
source = afwDetection.Source()
self.sourceList.append(source)
source.setId(i)
source.setFlagForDetection(source.getFlagForDetection() | measAlg.Flags.BINNED1);
try:
measureSources.apply(source, objects[i])
except Exception, e:
try:
print e
except Exception, ee:
print ee
