def overscan(self, exposure):
"""Overscan subtraction
@param exposure Exposure to process
"""
assert exposure, "No exposure provided"
ccd = pipUtil.getCcd(exposure)
mi = exposure.getMaskedImage()
MaskedImage = type(mi)
for amp in ccd:
if not pipUtil.haveAmp(exposure, amp):
continue
biassec = amp.getDiskBiasSec()
# XXX lsst.ip.isr.overscanCorrection doesn't allow for the exposure to contain multiple amps, so
# we'll do this ourselves. The options for overscan correction are currently quite limited, so
# we're not missing out on anything.
#ipIsr.overscanCorrection(exposure, biassec, "MEDIAN")
datasec = amp.getDiskDataSec()
overscan = MaskedImage(mi, biassec, afwImage.LOCAL)
image = MaskedImage(mi, datasec, afwImage.LOCAL)
offset = afwMath.makeStatistics(overscan, afwMath.MEDIAN).getValue(
afwMath.MEDIAN)
self.log.log(
self.log.INFO, "Overscan correction on amp %s, %s: %f" %
(amp.getId(), biassec, offset))
image -= offset
return
def assembly(self, exposureList):
"""Assembly of amplifiers into a CCD
@param exposure List of exposures to be assembled (each is an amp from the same exposure)
@return Assembled exposure
"""
if not hasattr(exposureList, "__iter__"):
# This is not a list; presumably it's a single item needing no assembly
return exposureList
assert len(exposureList) > 0, "Nothing in exposureList"
if len(exposureList) == 1 and exposureList[0].getMaskedImage().getDimensions() == \
pipUtil.getCcd(exposureList[0]).getAllPixelsNoRotation(True).getDimensions():
# Special case: single exposure of the correct size
return exposureList[0]
egExp = exposureList[0] # The (assumed) model for exposures
egMi = egExp.getMaskedImage() # The (assumed) model for masked images
Exposure = type(egExp)
MaskedImage = type(egMi)
ccd = pipUtil.getCcd(egExp)
miCcd = MaskedImage(ccd.getAllPixelsNoRotation(True).getDimensions())
for exp in exposureList:
mi = exp.getMaskedImage()
if pipUtil.detectorIsCcd(exp):
for amp in ccd:
self._assembleAmp(miCcd, mi, amp)
exp.setMaskedImage(miCcd)
else:
amp = pipUtil.getAmp(exp)
self._assembleAmp(miCcd, mi, amp)
exp = afwImage.makeExposure(miCcd, egExp.getWcs())
exp.setWcs(egExp.getWcs())
exp.setMetadata(egExp.getMetadata())
md = exp.getMetadata()
if md.exists('DATASEC'):
md.remove('DATASEC')
exp.setFilter(egExp.getFilter())
exp.setDetector(ccd)
exp.getCalib().setExptime(egExp.getCalib().getExptime())
exp.getCalib().setMidTime(egExp.getCalib().getMidTime())
return exp
def distortion(self, exposure):
"""Generate appropriate optical distortion
@param exposure Exposure from which to get CCD
@return Distortion
"""
assert exposure, "No exposure provided"
ccd = pipUtil.getCcd(exposure)
dist = pipDist.createDistortion(ccd, self.config['distortion'])
return dist
def defects(self, exposure):
"""Mask defects and trim guider shadow
@param exposure Exposure to process
@return Defect list
"""
assert exposure, "No exposure provided"
defects = super(IsrSuprimeCam, self).defects(exposure)
ccd = pipUtil.getCcd(exposure)
ccdNum = ccd.getId().getSerial()
if ccdNum not in [0, 1, 2, 6, 7]:
# No need to mask
return defects
md = exposure.getMetadata()
if not md.exists("S_AG-X"):
self.log.log(self.log.WARN, "No autoguider position in exposure metadata.")
return defects
xGuider = md.get("S_AG-X")
if ccdNum in [1, 2, 7]:
maskLimit = int(60.0 * xGuider - 2300.0) # From SDFRED
elif ccdNum in [0, 6]:
maskLimit = int(60.0 * xGuider - 2000.0) # From SDFRED
mi = exposure.getMaskedImage()
height = mi.getHeight()
if height < maskLimit:
# Nothing to mask!
return defects
if False:
# XXX This mask plane isn't respected by background subtraction or source detection or measurement
self.log.log(self.log.INFO, "Masking autoguider shadow at y > %d" % maskLimit)
mask = mi.getMask()
bbox = afwGeom.Box2I(afwGeom.Point2I(0, maskLimit - 1),
afwGeom.Point2I(mask.getWidth() - 1, height - 1))
badMask = mask.Factory(mask, bbox, afwImage.LOCAL)
mask.addMaskPlane("GUIDER")
badBitmask = mask.getPlaneBitMask("GUIDER")
badMask |= badBitmask
else:
# XXX Temporary solution until a mask plane is respected by downstream processes
self.log.log(self.log.INFO, "Removing pixels affected by autoguider shadow at y > %d" % maskLimit)
bbox = afwGeom.Box2I(afwGeom.Point2I(0, 0), afwGeom.Extent2I(mi.getWidth(), maskLimit))
good = mi.Factory(mi, bbox, afwImage.LOCAL)
exposure.setMaskedImage(good)
return defects
def linearize(self, exposure):
"""Correct for non-linearity
@param exposure Exposure to process
"""
assert exposure, "No exposure provided"
image = exposure.getMaskedImage().getImage()
ccd = pipUtil.getCcd(exposure)
for amp in ccd:
if not pipUtil.haveAmp(exposure, amp):
continue
if False:
linear_threshold = amp.getElectronicParams(
).getLinearizationThreshold()
linear_c = amp.getElectronicParams(
).getLinearizationCoefficient()
else:
linear_threshold = self.config["linearize"]["threshold"]
linear_c = self.config["linearize"]["coefficient"]
if linear_c == 0.0: # nothing to do
continue
self.log.log(
self.log.INFO,
"Applying linearity corrections to Ccd %s Amp %s" %
(ccd.getId(), amp.getId()))
log10_thresh = math.log10(linear_threshold)
ampImage = image.Factory(image, amp.getDiskDataSec(),
afwImage.LOCAL)
width, height = ampImage.getDimensions()
for y in range(height):
for x in range(width):
val = ampImage.get(x, y)
if True:
val += linear_c * val * val
ampImage.set(x, y, val)
else:
if val > linear_threshold:
val += val * linear_c * (math.log10(val) -
log10_thresh)
ampImage.set(x, y, val)
def variance(self, exposure):
"""Set variance from gain
@param exposure Exposure to process
"""
assert exposure, "No exposure provided"
mi = exposure.getMaskedImage()
if pipUtil.detectorIsCcd(exposure):
ccd = pipUtil.getCcd(exposure)
MaskedImage = type(mi)
for amp in ccd:
miAmp = MaskedImage(mi, amp.getDataSec(True), afwImage.LOCAL)
self._varianceAmp(miAmp, amp)
else:
amp = cameraGeom.cast_Amp(exposure.getDetector())
self._varianceAmp(mi, amp)
return
def saturation(self, exposure):
"""Mask saturated pixels
@param exposure Exposure to process
"""
assert exposure, "No exposure provided"
ccd = pipUtil.getCcd(exposure)
mi = exposure.getMaskedImage()
Exposure = type(exposure)
MaskedImage = type(mi)
for amp in ccd:
if not pipUtil.haveAmp(exposure, amp):
continue
saturation = amp.getElectronicParams().getSaturationLevel()
miAmp = MaskedImage(mi, amp.getDiskDataSec(), afwImage.PARENT)
expAmp = Exposure(miAmp)
bboxes = ipIsr.saturationDetection(expAmp, saturation, doMask=True)
self.log.log(
self.log.INFO, "Masked %d saturated objects on amp %s: %f" %
(len(bboxes), amp.getId(), saturation))
return
def trim(self, exposure):
"""Trim overscan out of exposure
@param exposure Exposure to process
"""
assert exposure, "No exposure provided"
mi = exposure.getMaskedImage()
MaskedImage = type(mi)
if pipUtil.detectorIsCcd(exposure):
# Effectively doing CCD assembly since we have all amplifiers
ccd = pipUtil.getCcd(exposure)
miCcd = MaskedImage(ccd.getAllPixels(True).getDimensions())
for amp in ccd:
diskDataSec = amp.getDiskDataSec()
trimDataSec = amp.getElectronicDataSec(True)
miTrim = MaskedImage(mi, diskDataSec, afwImage.LOCAL)
miTrim = MaskedImage(amp.prepareAmpData(miTrim.getImage()),
amp.prepareAmpData(miTrim.getMask()),
amp.prepareAmpData(miTrim.getVariance()))
miAmp = MaskedImage(miCcd, trimDataSec, afwImage.LOCAL)
self.log.log(
self.log.INFO, "Trimming amp %s: %s --> %s" %
(amp.getId(), diskDataSec, trimDataSec))
miAmp <<= miTrim
amp.setTrimmed(True)
exposure.setMaskedImage(miCcd)
else:
# AFW doesn't provide a useful target datasec, so we just make an image that has the useful pixels
amp = cameraGeom.cast_Amp(exposure.getDetector())
diskDataSec = amp.getDiskDataSec()
self.log.log(self.log.INFO,
"Trimming amp %s: %s" % (amp.getId(), diskDataSec))
miTrim = MaskedImage(mi, diskDataSec, afwImage.LOCAL)
amp.setTrimmed(True)
miAmp = MaskedImage(amp.prepareAmpData(miTrim.getImage()),
amp.prepareAmpData(miTrim.getMask()),
amp.prepareAmpData(miTrim.getVariance()))
exposure.setMaskedImage(miAmp)
return
def defects(self, exposure):
"""Mask defects
@param exposure Exposure to process
@return Defect list
"""
assert exposure, "No exposure provided"
policy = self.config['defects']
defects = measAlg.DefectListT()
ccd = pipUtil.getCcd(exposure)
statics = ccd.getDefects() # Static defects
for defect in statics:
bbox = defect.getBBox()
new = measAlg.Defect(bbox)
defects.append(new)
ipIsr.maskBadPixelsDef(exposure, defects, fwhm=None, interpolate=False, maskName='BAD')
self.log.log(self.log.INFO, "Masked %d static defects." % len(statics))
grow = policy['grow']
sat = ipIsr.defectListFromMask(exposure, growFootprints=grow, maskName='SAT') # Saturated defects
self.log.log(self.log.INFO, "Added %d saturation defects." % len(sat))
for defect in sat:
bbox = defect.getBBox()
new = measAlg.Defect(bbox)
defects.append(new)
exposure.getMaskedImage().getMask().addMaskPlane("UNMASKEDNAN")
nanMasker = ipIsr.UnmaskedNanCounterF()
nanMasker.apply(exposure.getMaskedImage())
nans = ipIsr.defectListFromMask(exposure, maskName='UNMASKEDNAN')
self.log.log(self.log.INFO, "Added %d unmasked NaNs." % nanMasker.getNpix())
for defect in nans:
bbox = defect.getBBox()
new = measAlg.Defect(bbox)
defects.append(new)
return defects
def run(ioMgr, frameId, ccdId, config, display=False):
data = {'visit': frameId, 'ccd':ccdId}
raws = ioMgr.readRaw(data)[0]
wcsIn = raws.getWcs()
exposure = ioMgr.read('calexp', data, ignore=True)[0]
wcsTrue = exposure.getWcs()
exposure.setWcs(wcsIn)
if display:
ds9.mtv(exposure, frame=1)
bscSet = ioMgr.read('bsc', data, ignore=True)[0].getSources()
srcSet = [s for s in bscSet if goodStar(s)]
print "# of srcSet: ", len(srcSet)
distConfig = config['distortion']
ccd = pipUtil.getCcd(exposure)
dist = pipDist.createDistortion(ccd, distConfig)
distSrc = dist.actualToIdeal(srcSet)
if display:
for (s, d) in zip(srcSet, distSrc):
xs = s.getXAstrom()
ys = s.getYAstrom()
xd = d.getXAstrom()
yd = d.getYAstrom()
#ds9.dot("o", xs, ys, size=10, frame=1)
#ds9.dot("o", xd, yd, size=10, frame=1, ctype=ds9.RED)
#ds9.line([[xs,ys], [xd,yd]], frame=1)
xMin, xMax, yMin, yMax = float("INF"), float("-INF"), float("INF"), float("-INF")
for x, y in ((0.0, 0.0), (0.0, exposure.getHeight()), (exposure.getWidth(), 0.0),
(exposure.getWidth(), exposure.getHeight())):
point = afwGeom.Point2D(x, y)
point = dist.actualToIdeal(point)
x, y = point.getX(), point.getY()
if x < xMin: xMin = x
if x > xMax: xMax = x
if y < yMin: yMin = y
if y > yMax: yMax = y
xMin = int(xMin)
yMin = int(yMin)
size = (int(xMax - xMin + 0.5),
int(yMax - yMin + 0.5))
for s in distSrc:
s.setXAstrom(s.getXAstrom() - xMin)
s.setYAstrom(s.getYAstrom() - yMin)
sortedDistSrc = sorted(distSrc, key=lambda x:x.getPsfFlux(), reverse=True)
if display:
for i, d in enumerate(sortedDistSrc):
#ds9.dot('o', d.getXAstrom(), d.getYAstrom(), size=15, frame=1)
if i >= 99: break
ds9.line([[0, 0], [0+size[0], 0]], frame=1)
ds9.line([[0+size[0], 0], [0+size[0], 0+size[1]]], frame=1)
ds9.line([[0+size[0], 0+size[1]], [0, 0+size[1]]], frame=1)
ds9.line([[0, 0+size[1]], [0, 0]], frame=1)
log = Log.getDefaultLog()
pol = policy.Policy()
pol.set('matchThreshold', 30)
solver = measAst.createSolver(pol, log)
(W,H) = size
filterName = 'r'
idName = 'id'
wcsIn.shiftReferencePixel(-xMin, -yMin)
catSet = hscAst.queryReferenceCatalog(solver, distSrc, wcsIn, (W,H),
filterName, idName)
print "# of catSet: ", len(catSet)
if display:
for c in catSet:
ds9.dot("o", c.getXAstrom()+xMin, c.getYAstrom()+yMin, size=15, ctype=ds9.RED, frame=1)
start = datetime.datetime.today()
matchList = hscAst.match(distSrc, catSet)
end = datetime.datetime.today()
wcsIn.shiftReferencePixel(xMin, yMin)
matchList2 = []
for m in matchList:
for s in srcSet:
if (s.getId() == m.second.getId()):
x0 = m.first.getXAstrom() + xMin
y0 = m.first.getYAstrom() + yMin
x1 = m.second.getXAstrom()
y1 = m.second.getYAstrom()
x1 = s.getXAstrom()
y1 = s.getYAstrom()
#print x0, y0, x1, y1, m.second.getId()
if display:
ds9.dot("o", x1, y1, size=10, frame=1)
ds9.line([[x0, y0], [x1, y1]], ctype=ds9.RED, frame=1)
m2 = afwDet.SourceMatch(m.first, s, 0.0)
matchList2.append(m2)
elapsedtime = end - start
print "%6d %d %3d %6.2f" % (frameId, ccdId, len(matchList), elapsedtime.seconds + elapsedtime.microseconds/1000000.)
def display(self,
name,
exposure=None,
sources=[],
matches=None,
pause=None,
prompt=None):
"""Display image and/or sources
@param name Name of product to display
@param exposure Exposure to display, or None
@param sources list of sets of Sources to display, or []
@param matches Matches to display, or None
@param pause Pause execution?
"""
if not self._display or not self._display.has_key(name) or self._display < 0 or \
self._display in (False, None) or \
self._display[name] in (False, None) or self._display[name] < 0:
return
if isinstance(self._display, int):
frame = self._display
elif isinstance(self._display, dict):
frame = self._display[name]
else:
frame = 1
if exposure:
if isinstance(exposure, list):
if len(exposure) == 1:
exposure = exposure[0]
else:
exposure = ipIsr.assembleCcd(exposure,
pipUtil.getCcd(exposure[0]))
mi = exposure.getMaskedImage()
ds9.mtv(mi, frame=frame, title=name)
x0, y0 = mi.getX0(), mi.getY0()
else:
x0, y0 = 0, 0
try:
sources[0][0]
except IndexError: # empty list
pass
except TypeError: # not a list of sets of sources
sources = [sources]
ctypes = [ds9.GREEN, ds9.RED, ds9.BLUE]
for i, ss in enumerate(sources):
try:
ds9.buffer()
except AttributeError:
ds9.cmdBuffer.pushSize()
for source in ss:
xc, yc = source.getXAstrom() - x0, source.getYAstrom() - y0
ds9.dot("o",
xc,
yc,
size=4,
frame=frame,
ctype=ctypes[i % len(ctypes)])
#try:
# mag = 25-2.5*math.log10(source.getPsfFlux())
# if mag > 15: continue
#except: continue
#ds9.dot("%.1f" % mag, xc, yc, frame=frame, ctype="red")
try:
ds9.buffer(False)
except AttributeError:
ds9.cmdBuffer.popSize()
if matches:
try:
ds9.buffer()
except AttributeError:
ds9.cmdBuffer.pushSize()
for match in matches:
first = match.first
x1, y1 = first.getXAstrom() - x0, first.getYAstrom() - y0
ds9.dot("+", x1, y1, size=8, frame=frame, ctype="yellow")
second = match.second
x2, y2 = second.getXAstrom() - x0, second.getYAstrom() - y0
ds9.dot("x", x2, y2, size=8, frame=frame, ctype="red")
try:
ds9.buffer(False)
except AttributeError:
ds9.cmdBuffer.popSize()
if pause:
if prompt is None:
prompt = "%s: Enter or c to continue [chp]: " % name
while True:
ans = raw_input(prompt).lower()
if ans in (
"",
"c",
):
break
if ans in ("p", ):
import pdb
pdb.set_trace()
elif ans in ("h", ):
print "h[elp] c[ontinue] p[db]"
return