def testAssemble(self):
"""Test the assembly of E2V and ITL sensors
"""
task = AssembleCcdTask()
# exclude LATISS for this test since we don't have an expected output
for root, did, expected in zip(self.roots, self.ids, self.expecteds):
butler = Butler(root)
raw = butler.get("raw", dataId=did)
assembled = task.assembleCcd(raw)
count = numpy.sum(expected['expected'].read().array -
assembled.getImage().array)
self.assertEqual(count, 0)
def assemble_raw(dataId, componentInfo, cls):
"""Called by the butler to construct the composite type "raw".
Note that we still need to define "_raw" and copy various fields over.
Parameters
----------
dataId : `lsst.daf.persistence.dataId.DataId`
the data ID
componentInfo : `dict`
dict containing the components, as defined by the composite definition in the mapper policy
cls : 'object'
unused
Returns
-------
exposure : `lsst.afw.image.exposure.exposure`
The assembled exposure
"""
from lsst.ip.isr import AssembleCcdTask
config = AssembleCcdTask.ConfigClass()
config.doTrim = False
assembleTask = AssembleCcdTask(config=config)
ampExps = componentInfo['raw_amp'].obj
if len(ampExps) == 0:
raise RuntimeError("Unable to read raw_amps for %s" % dataId)
ccd = ampExps[0].getDetector(
) # the same (full, CCD-level) Detector is attached to all ampExps
ampDict = {}
for amp, ampExp in zip(ccd, ampExps):
ampDict[amp.getName()] = ampExp
exposure = assembleTask.assembleCcd(ampDict)
md = componentInfo['raw_hdu'].obj
exposure.setMetadata(md)
#
# We need to standardize, but have no legal way to call std_raw. The butler should do this for us.
#
ccm = ComCamMapper()
exposure = ccm.std_raw(exposure, dataId)
return exposure
def runAssembler():
'''Run the task to assemble amps into a ccd'''
#Create the assemble task with default config
assembleConfig = AssembleCcdTask.ConfigClass()
assembleTask = AssembleCcdTask(config=assembleConfig)
frame = 0
#The assemble task can operate in two ways:
#1. On a dictionary of amp size images
#2. On a single amp mosaic image
#Both methods should produce the same output.
for isPerAmp in (True, False):
assemblyInput = exampleUtils.makeAssemblyInput(isPerAmp)
assembledExposure = assembleTask.assembleCcd(assemblyInput)
ds9.mtv(assembledExposure.getMaskedImage(), frame=frame, title="Per amp input is %s"%(isPerAmp))
frame += 1
def AssembleImage(input_file, doGainCorrection = False, trim = True):
import os
imDict = {}
afilelist = []
dfilename = '%s[%s]' % (input_file, 0)
for i in range(16):
filename = '%s[%i]' % (input_file, i+1)
md = afwImage.readMetadata(filename)
afilelist.append(filename)
imDict[md.get('EXTNAME')] = afwImage.ImageF(filename)
db = TestCamDetectorBuilder(dfilename, afilelist, inAmpCoords=True, clobberMetadata=True)
det = db.buildDetector()
assembleInput = {}
for amp in det:
im = imDict[amp.getName()]
oscanim = im.Factory(im, amp.getRawHorizontalOverscanBBox())
oscan = numpy.median(oscanim.getArray())
imArr = im.getArray()
imArr -= oscan
#Calculate and correct for gain
if doGainCorrection:
# Buffer so edge rolloff doesn't interfere
buffImArr = imArr[30:-30][30:-30]
medCounts = numpy.median(buffImArr)
stdCounts = numpy.std(buffImArr)
gain = medCounts/stdCounts**2
imArr *= gain
assembleInput[amp.getName()] = db.makeExposure(im)
assembleConfig = AssembleCcdTask.ConfigClass()
if trim:
assembleConfig.doTrim = True
assembler = AssembleCcdTask(config=assembleConfig)
resultExp = assembler.assembleCcd(assembleInput)
# camGeomUtils.showCcd(resultExp.getDetector(), imageSource(resultExp), frame=0)
return resultExp
else:
assembleConfig.doTrim = False
assembler = AssembleCcdTask(config=assembleConfig)
resultExp = assembler.assembleCcd(assembleInput)
# camGeomUtils.showCcd(resultExp.getDetector(), imageSource(resultExp), frame=1)
return resultExp
def runAssembler():
'''Run the task to assemble amps into a ccd'''
# Create the assemble task with default config
assembleConfig = AssembleCcdTask.ConfigClass()
assembleTask = AssembleCcdTask(config=assembleConfig)
frame = 0
# The assemble task can operate in two ways:
# 1. On a dictionary of amp size images
# 2. On a single amp mosaic image
# Both methods should produce the same output.
for isPerAmp in (True, False):
assemblyInput = exampleUtils.makeAssemblyInput(isPerAmp)
assembledExposure = assembleTask.assembleCcd(assemblyInput)
ds9.mtv(assembledExposure.getMaskedImage(),
frame=frame,
title="Per amp input is %s" % (isPerAmp))
frame += 1
def assembleUntrimmedCcd(ccd, exposures):
"""Assemble an untrimmmed CCD from per-amp Exposure objects.
Parameters
----------
ccd : `~lsst.afw.cameraGeom.Detector`
The detector geometry for this ccd that will be used as the
framework for the assembly of the input amplifier exposures.
exposures : sequence of `lsst.afw.image.Exposure`
Per-amplifier images, in the same order as ``amps``.
Returns
-------
ccd : `lsst.afw.image.Exposure`
Assembled CCD image.
"""
ampDict = {}
for amp, exposure in zip(ccd, exposures):
ampDict[amp.getName()] = exposure
config = AssembleCcdTask.ConfigClass()
config.doTrim = False
assembleTask = AssembleCcdTask(config=config)
return assembleTask.assembleCcd(ampDict)
def AssembleImage(input_file, doGainCorrection=False, trim=True):
import os
imDict = {}
afilelist = []
dfilename = '%s[%s]' % (input_file, 0)
for i in range(16):
filename = '%s[%i]' % (input_file, i + 1)
md = afwImage.readMetadata(filename)
afilelist.append(filename)
imDict[md.get('EXTNAME')] = afwImage.ImageF(filename)
db = TestCamDetectorBuilder(dfilename,
afilelist,
inAmpCoords=True,
clobberMetadata=True)
det = db.buildDetector()
assembleInput = {}
for amp in det:
im = imDict[amp.getName()]
oscanim = im.Factory(im, amp.getRawHorizontalOverscanBBox())
oscan = numpy.median(oscanim.getArray())
imArr = im.getArray()
imArr -= oscan
#Calculate and correct for gain
if doGainCorrection:
# Buffer so edge rolloff doesn't interfere
buffImArr = imArr[30:-30][30:-30]
medCounts = numpy.median(buffImArr)
stdCounts = numpy.std(buffImArr)
gain = medCounts / stdCounts**2
imArr *= gain
assembleInput[amp.getName()] = db.makeExposure(im)
assembleConfig = AssembleCcdTask.ConfigClass()
if trim:
assembleConfig.doTrim = True
assembler = AssembleCcdTask(config=assembleConfig)
resultExp = assembler.assembleCcd(assembleInput)
# camGeomUtils.showCcd(resultExp.getDetector(), imageSource(resultExp), frame=0)
return resultExp
else:
assembleConfig.doTrim = False
assembler = AssembleCcdTask(config=assembleConfig)
resultExp = assembler.assembleCcd(assembleInput)
# camGeomUtils.showCcd(resultExp.getDetector(), imageSource(resultExp), frame=1)
return resultExp
def bypass_raw(self, datasetType, pythonType, location, dataId):
"""Magic method that is called automatically if it exists.
"""
from lsst.ip.isr import AssembleCcdTask
config = AssembleCcdTask.ConfigClass()
config.doTrim = False
assembleTask = AssembleCcdTask(config=config)
logger = lsst.log.Log.getLogger("ZtfMapper")
# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
assert len(location.getLocations()
) == 1 # KTL says that this is always true, but check
relativeFileName = location.getLocations()[0]
if location.storage.exists(relativeFileName):
fileName = location.storage.locationWithRoot(relativeFileName)
data = afwImage.ImageF(fileName, hdu=1)
data = pythonType(afwImage.MaskedImageF(data))
else:
raise IOError(f"Unable to find {relativeFileName}")
# Get the Detector
detector = self.camera[self._extractDetectorName(dataId)]
#
# Read all the data
#
ampDict = {}
for i, amp in enumerate(detector, 1):
ampExp = pythonType(amp.getRawBBox())
ampExp.setDetector(detector)
ampDict[amp.getName()] = ampExp
hdu = amp.get('hdu')
if i == 1:
assert i == hdu # don't read twice
data = data.image
else:
data = afwImage.ImageF(fileName, hdu=hdu)
ampExp[amp.getRawDataBBox()].image = data
bias = afwImage.ImageF(fileName, hdu=hdu + 4)
ampExp[amp.getRawHorizontalOverscanBBox()].image = bias
exposure = assembleTask.assembleCcd(ampDict)
md = afwImage.readMetadata(fileName, hdu=0)
fix_header(md, translator_class=self.translatorClass)
exposure.setMetadata(md)
visitInfo = ZtfMakeRawVisitInfo(logger)(md)
exposure.getInfo().setVisitInfo(visitInfo)
boresight = visitInfo.getBoresightRaDec()
if boresight.isFinite():
exposure.setWcs(
getWcsFromDetector(exposure.getDetector(),
boresight,
180 * geom.degrees,
flipX=True))
else:
logger.warn(
f"Unable to set WCS for {dataId} from header as RA/Dec/Angle are unavailable"
)
return exposure
def assemble_raw(dataId, componentInfo, cls):
"""Called by the butler to construct the composite type "raw".
Note that we still need to define "_raw" and copy various fields over.
Parameters
----------
dataId : `lsst.daf.persistence.dataId.DataId`
The data ID.
componentInfo : `dict`
dict containing the components, as defined by the composite definition
in the mapper policy.
cls : 'object'
Unused.
Returns
-------
exposure : `lsst.afw.image.Exposure`
The assembled exposure.
"""
from lsst.ip.isr import AssembleCcdTask
config = AssembleCcdTask.ConfigClass()
config.doTrim = False
assembleTask = AssembleCcdTask(config=config)
ampExps = componentInfo['raw_amp'].obj
if len(ampExps) == 0:
raise RuntimeError("Unable to read raw_amps for %s" % dataId)
ccd = ampExps[0].getDetector() # the same (full, CCD-level) Detector is attached to all ampExps
#
# Check that the geometry in the metadata matches cameraGeom
#
logger = lsst.log.Log.getLogger("LsstCamMapper")
warned = False
for i, (amp, ampExp) in enumerate(zip(ccd, ampExps)):
ampMd = ampExp.getMetadata().toDict()
if amp.getRawBBox() != ampExp.getBBox(): # Oh dear. cameraGeom is wrong -- probably overscan
if amp.getRawDataBBox().getDimensions() != amp.getBBox().getDimensions():
raise RuntimeError("Active area is the wrong size: %s v. %s" %
(amp.getRawDataBBox().getDimensions(), amp.getBBox().getDimensions()))
if not warned:
logger.warn("amp.getRawBBox() != data.getBBox(); patching. (%s v. %s)",
amp.getRawBBox(), ampExp.getBBox())
warned = True
w, h = ampExp.getBBox().getDimensions()
ow, oh = amp.getRawBBox().getDimensions() # "old" (cameraGeom) dimensions
#
# We could trust the BIASSEC keyword, or we can just assume that
# they've changed the number of overscan pixels (serial and/or
# parallel). As Jim Chiang points out, the latter is safer
#
bbox = amp.getRawHorizontalOverscanBBox()
hOverscanBBox = geom.BoxI(bbox.getBegin(),
geom.ExtentI(w - bbox.getBeginX(), bbox.getHeight()))
bbox = amp.getRawVerticalOverscanBBox()
vOverscanBBox = geom.BoxI(bbox.getBegin(),
geom.ExtentI(bbox.getWidth(), h - bbox.getBeginY()))
amp.setRawBBox(ampExp.getBBox())
amp.setRawHorizontalOverscanBBox(hOverscanBBox)
amp.setRawVerticalOverscanBBox(vOverscanBBox)
#
# This gets all the geometry right for the amplifier, but the size
# of the untrimmed image will be wrong and we'll put the amp
# sections in the wrong places, i.e.
# amp.getRawXYOffset()
# will be wrong. So we need to recalculate the offsets.
#
xRawExtent, yRawExtent = amp.getRawBBox().getDimensions()
x0, y0 = amp.getRawXYOffset()
ix, iy = x0//ow, y0/oh
x0, y0 = ix*xRawExtent, iy*yRawExtent
amp.setRawXYOffset(geom.ExtentI(ix*xRawExtent, iy*yRawExtent))
#
# Check the "IRAF" keywords, but don't abort if they're wrong
#
# Only warn about the first amp, use debug for the others
#
detsec = bboxFromIraf(ampMd["DETSEC"]) if "DETSEC" in ampMd else None
datasec = bboxFromIraf(ampMd["DATASEC"]) if "DATASEC" in ampMd else None
biassec = bboxFromIraf(ampMd["BIASSEC"]) if "BIASSEC" in ampMd else None
logCmd = logger.warn if i == 0 else logger.debug
if detsec and amp.getBBox() != detsec:
logCmd("DETSEC doesn't match for %s (%s != %s)", dataId, amp.getBBox(), detsec)
if datasec and amp.getRawDataBBox() != datasec:
logCmd("DATASEC doesn't match for %s (%s != %s)", dataId, amp.getRawDataBBox(), detsec)
if biassec and amp.getRawHorizontalOverscanBBox() != biassec:
logCmd("BIASSEC doesn't match for %s (%s != %s)",
dataId, amp.getRawHorizontalOverscanBBox(), detsec)
ampDict = {}
for amp, ampExp in zip(ccd, ampExps):
ampDict[amp.getName()] = ampExp
exposure = assembleTask.assembleCcd(ampDict)
md = componentInfo['raw_hdu'].obj
fix_header(md) # No mapper so cannot specify the translator class
exposure.setMetadata(md)
visitInfo = LsstCamMakeRawVisitInfo(logger)(md)
exposure.getInfo().setVisitInfo(visitInfo)
boresight = visitInfo.getBoresightRaDec()
rotangle = visitInfo.getBoresightRotAngle()
if boresight.isFinite():
exposure.setWcs(getWcsFromDetector(exposure.getDetector(), boresight,
90*geom.degrees - rotangle))
else:
# Should only warn for science observations but VisitInfo does not know
logger.warn("Unable to set WCS for %s from header as RA/Dec/Angle are unavailable" %
(dataId,))
return exposure
db = TestCamDetectorBuilder(dfilename, afilelist, inAmpCoords=True, clobberMetadata=True)
det = db.buildDetector()
assembleInput = {}
for amp in det:
im = imDict[amp.getName()]
oscanim = im.Factory(im, amp.getRawHorizontalOverscanBBox())
oscan = numpy.median(oscanim.getArray())
imArr = im.getArray()
imArr -= oscan
#Calculate and correct for gain
if args.doGainCorrection:
# Buffer so edge rolloff doesn't interfere
buffImArr = imArr[30:-30][30:-30]
medCounts = numpy.median(buffImArr)
stdCounts = numpy.std(buffImArr)
gain = medCounts/stdCounts**2
imArr *= gain
assembleInput[amp.getName()] = db.makeExposure(im)
assembleConfig = AssembleCcdTask.ConfigClass()
if args.displayTrimmed:
assembleConfig.doTrim = True
assembler = AssembleCcdTask(config=assembleConfig)
resultExp = assembler.assembleCcd(assembleInput)
camGeomUtils.showCcd(resultExp.getDetector(), imageSource(resultExp), frame=0)
if args.displayUnTrimmed:
assembleConfig.doTrim = False
assembler = AssembleCcdTask(config=assembleConfig)
resultExp = assembler.assembleCcd(assembleInput)
camGeomUtils.showCcd(resultExp.getDetector(), imageSource(resultExp), frame=1)
assembleInput = {}
for amp in det:
im = imDict[amp.getName()]
oscanim = im.Factory(im, amp.getRawHorizontalOverscanBBox())
oscan = numpy.median(oscanim.getArray())
imArr = im.getArray()
imArr -= oscan
#Calculate and correct for gain
if args.doGainCorrection:
# Buffer so edge rolloff doesn't interfere
medCounts = numpy.median(imArr[30:-30][30:-30])
stdCounts = numpy.std(imArr[30:-30][30:-30])
gain = medCounts / stdCounts**2
imArr *= gain
assembleInput[amp.getName()] = db.makeExposure(im)
assembleConfig = AssembleCcdTask.ConfigClass()
if args.displayTrimmed:
assembleConfig.doTrim = True
assembler = AssembleCcdTask(config=assembleConfig)
resultExp = assembler.assembleCcd(assembleInput)
camGeomUtils.showCcd(resultExp.getDetector(),
imageSource(resultExp),
frame=0)
if args.displayUnTrimmed:
assembleConfig.doTrim = False
assembler = AssembleCcdTask(config=assembleConfig)
resultExp = assembler.assembleCcd(assembleInput)
camGeomUtils.showCcd(resultExp.getDetector(),
imageSource(resultExp),