def addAmp(ampCatalog, i, eparams):
""" Add an amplifier to an AmpInfoCatalog
@param ampCatalog: An instance of an AmpInfoCatalog object to fill with amp properties
@param i which amplifier? (i == 0 ? left : right)
@param eparams: Electronic parameters. This is a list of tuples with (i, params),
where params is a dictionary of electronic parameters.
"""
#
# Layout of active and overclock pixels in the as-readout data The layout is:
# Amp0 || extended | overclock | data || data | overclock | extended || Amp1
# for each row; all rows are identical in drift-scan data
#
height = 1361 # number of rows in a frame
width = 1024 # number of data pixels read out through one amplifier
nExtended = 8 # number of pixels in the extended register
nOverclock = 32 # number of (horizontal) overclock pixels
#
# Construct the needed bounding boxes given that geometrical information.
#
# Note that all the offsets are relative to the origin of this amp, not to its eventual
# position in the CCD
#
record = ampCatalog.addNew()
xtot = width + nExtended + nOverclock
allPixels = geom.BoxI(geom.PointI(0, 0), geom.ExtentI(xtot, height))
biasSec = geom.BoxI(geom.PointI(nExtended if i == 0 else width, 0),
geom.ExtentI(nOverclock, height))
dataSec = geom.BoxI(
geom.PointI(nExtended + nOverclock if i == 0 else 0, 0),
geom.ExtentI(width, height))
emptyBox = geom.BoxI()
bbox = geom.BoxI(geom.PointI(0, 0), geom.ExtentI(width, height))
bbox.shift(geom.Extent2I(width * i, 0))
shiftp = geom.Extent2I(xtot * i, 0)
allPixels.shift(shiftp)
biasSec.shift(shiftp)
dataSec.shift(shiftp)
record.setBBox(bbox)
record.setRawXYOffset(geom.ExtentI(0, 0))
record.setName('left' if i == 0 else 'right')
record.setReadoutCorner(afwTable.LL if i == 0 else afwTable.LR)
record.setGain(eparams['gain'])
record.setReadNoise(eparams['readNoise'])
record.setSaturation(eparams['fullWell'])
record.setSuspectLevel(float("nan"))
record.setLinearityType(NullLinearityType)
record.setLinearityCoeffs([
1.,
])
record.setHasRawInfo(True)
record.setRawFlipX(False)
record.setRawFlipY(False)
record.setRawBBox(allPixels)
record.setRawDataBBox(dataSec)
record.setRawHorizontalOverscanBBox(biasSec)
record.setRawVerticalOverscanBBox(emptyBox)
record.setRawPrescanBBox(emptyBox)
def testValid(self):
test_data = {
"[1:1084,1:1024]":
geom.BoxI(geom.PointI(0, 0), geom.PointI(1083, 1023)),
"[0:0,0:0]":
geom.BoxI(geom.PointI(-1, -1), geom.PointI(-1, -1))
}
for val, result in test_data.items():
self.assertEqual(obsBase.bboxFromIraf(val), result)
def findBrightestStarRHL(exp, minArea=100, maxRadialOffset=1500):
import lsst.afw.detection as afwDetect
import lsst.afw.image as afwImage
import lsst.afw.math as afwMath
cexp = exp.clone()
sigma = 4
ksize = 1 + 4*int(sigma + 1)
gauss1d = afwMath.GaussianFunction1D(sigma)
kernel = afwMath.SeparableKernel(ksize, ksize, gauss1d, gauss1d)
convolvedImage = cexp.maskedImage.Factory(cexp.maskedImage.getBBox())
afwMath.convolve(convolvedImage, cexp.maskedImage, kernel)
bbox = geom.BoxI(geom.PointI(ksize//2, ksize//2), geom.PointI(cexp.getWidth() - ksize//2 - 1, cexp.getHeight() - ksize//2 - 1))
tmp = cexp.maskedImage[bbox]
cexp.image *= 0
tmp[bbox, afwImage.PARENT] = convolvedImage[bbox]
del convolvedImage; del tmp
if False:
defectBBoxes = [
geom.BoxI(geom.PointI(548, 3562), geom.PointI(642, 3999)),
geom.BoxI(geom.PointI(474, 3959), geom.PointI(1056, 3999)),
geom.BoxI(geom.PointI(500, 0), geom.PointI(680, 40)),
]
for bbox in defectBBoxes:
cexp.maskedImage[bbox] = 0
threshold = 1000
feet = afwDetect.FootprintSet(cexp.maskedImage, afwDetect.Threshold(threshold), "DETECTED").getFootprints()
maxVal = None
centroid = None
for foot in feet:
peak = foot.peaks[0]
if minArea > 0 and foot.getArea() < minArea:
continue
x, y = peak.getCentroid()
if np.hypot(x - 2036, y - 2000) > maxRadialOffset:
continue
if maxVal is None or peak.getPeakValue() > maxVal:
maxVal = peak.getPeakValue()
centroid = peak.getCentroid()
return centroid
def testSubMap(self):
bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400))
mapper = MinMapper2(root=ROOT)
loc = mapper.map("raw_sub", {"ccd": 13, "bbox": bbox}, write=True)
self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU")
self.assertEqual(loc.getCppType(), "ImageU")
self.assertEqual(loc.getStorageName(), "FitsStorage")
self.assertEqual(loc.getLocations(), ["foo-13.fits"])
self.assertEqual(loc.getStorage().root, ROOT)
self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
self.assertEqual(loc.getAdditionalData().getScalar("width"), 300)
self.assertEqual(loc.getAdditionalData().getScalar("height"), 400)
self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200)
self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100)
checkCompression(self, loc.getAdditionalData())
loc = mapper.map("raw_sub", {
"ccd": 13,
"bbox": bbox,
"imageOrigin": "PARENT"
},
write=True)
self.assertEqual(loc.getPythonType(), "lsst.afw.image.ExposureU")
self.assertEqual(loc.getCppType(), "ImageU")
self.assertEqual(loc.getStorageName(), "FitsStorage")
self.assertEqual(loc.getLocations(), ["foo-13.fits"])
self.assertEqual(loc.getStorage().root, ROOT)
self.assertEqual(loc.getAdditionalData().getScalar("ccd"), 13)
self.assertEqual(loc.getAdditionalData().getScalar("width"), 300)
self.assertEqual(loc.getAdditionalData().getScalar("height"), 400)
self.assertEqual(loc.getAdditionalData().getScalar("llcX"), 200)
self.assertEqual(loc.getAdditionalData().getScalar("llcY"), 100)
self.assertEqual(loc.getAdditionalData().getScalar("imageOrigin"),
"PARENT")
checkCompression(self, loc.getAdditionalData())
def loadExposure(self, sensorRef):
"""Load SDSS data as a post-ISR exposure
- Image is from fpC
- Mask is from fpM
- Wcs is from asTrans
- PhotoCalib is from tsField
- Psf is from psField
"""
originalExp = sensorRef.get("fpC").convertF()
image = originalExp.getMaskedImage().getImage()
if self.config.removePedestal:
image -= self.config.pedestalVal
mask = sensorRef.get("fpM")
wcs = sensorRef.get("asTrans")
tsField = sensorRef.get("tsField")
photoCalib = tsField.photoCalib
gain = tsField.gain
var = afwImage.ImageF(image, True)
var /= gain
mi = afwImage.MaskedImageF(image, mask, var)
if self.config.removeOverlap:
bbox = mi.getBBox()
begin = bbox.getBegin()
extent = bbox.getDimensions()
extent -= geom.Extent2I(0, self.config.overlapSize)
tbbox = geom.BoxI(begin, extent)
mi = afwImage.MaskedImageF(mi, tbbox)
exposure = afwImage.ExposureF(mi, wcs)
expInfo = exposure.getInfo()
expInfo.setPhotoCalib(photoCalib)
camera = sensorRef.get('camera')
detector = camera["%(filter)s%(camcol)d" % sensorRef.dataId]
expInfo.setDetector(detector)
expInfo.setFilter(afwImage.Filter(sensorRef.dataId['filter']))
visitInfo = afwImage.VisitInfo(
exposureTime=tsField.exptime,
date=tsField.dateAvg,
boresightAirmass=tsField.airmass,
)
expInfo.setVisitInfo(visitInfo)
# Install the SDSS PSF here; if we want to overwrite it later, we can.
psf = sensorRef.get('psField')
exposure.setPsf(psf)
return exposure
def bboxFromIraf(irafBBoxStr):
"""Return a Box2I corresponding to an IRAF-style BBOX
[x0:x1,y0:y1] where x0 and x1 are the one-indexed start and end columns, and correspondingly
y0 and y1 are the start and end rows.
"""
mat = re.search(r"^\[([-\d]+):([-\d]+),([-\d]+):([-\d]+)\]$", irafBBoxStr)
if not mat:
raise RuntimeError("Unable to parse IRAF-style bbox \"%s\"" %
irafBBoxStr)
x0, x1, y0, y1 = [int(_) for _ in mat.groups()]
return geom.BoxI(geom.PointI(x0 - 1, y0 - 1), geom.PointI(x1 - 1, y1 - 1))
def testParentTrailingSlash2527(self):
"""Just like testParentNormal, but put a trailing slash on the root
paths.
Test that an object can be found at root location and put into an
output location. Then test that when the output locaiton is used as
an input location, and with a new output location, that the object is
found in the first output location."""
# todo these shouldn't be commented out, I think the test wants the
# trailing slash.
testOutput = self.mkdtemp("testOutput") + '/'
butler = dafPersist.Butler(inputs={
'root': ROOT + '/',
'mapper': MinMapper1
},
outputs=testOutput)
mapper1 = butler._repos.inputs()[0].repo._mapper
loc = mapper1.map("x", dict(sensor="1,1"), write=True)
self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI")
self.assertEqual(loc.getCppType(), "BoxI")
self.assertEqual(loc.getStorageName(), "PickleStorage")
self.assertEqual(loc.getLocations(), ["foo-1,1.pickle"])
self.assertEqual(loc.getStorage().root, ROOT)
self.assertEqual(loc.getAdditionalData().toString(),
"sensor = \"1,1\"\n")
box = geom.BoxI(geom.PointI(0, 1), geom.PointI(2, 3))
butler.put(box, "x", sensor="1,1")
self.assertTrue(
os.path.exists(os.path.join(testOutput,
loc.getLocations()[0])))
del butler
del mapper1
testOutput2 = self.mkdtemp("testOutput2") + '/'
butler = dafPersist.Butler(inputs={
'root': testOutput,
'mapper': MinMapper1
},
outputs=testOutput2)
mapper2 = butler._repos.inputs()[0].repo._mapper
loc = mapper2.map("x", dict(sensor="1,1"))
self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI")
self.assertEqual(loc.getCppType(), "BoxI")
self.assertEqual(loc.getStorageName(), "PickleStorage")
self.assertEqual(loc.getLocations(), ["foo-1,1.pickle"])
self.assertEqual(os.path.normpath(loc.getStorage().root),
os.path.normpath(testOutput))
self.assertEqual(loc.getAdditionalData().toString(),
"sensor = \"1,1\"\n")
def testGzImage(self):
mapper = MinMapper2(root=ROOT)
loc = mapper.map("someGz", dict(ccd=35))
expectedLocations = [os.path.join("gz", "bar-35.fits.gz")]
self.assertEqual(loc.getStorage().root, ROOT)
self.assertEqual(loc.getLocations(), expectedLocations)
butler = dafPersist.ButlerFactory(mapper=mapper).create()
image = butler.get("someGz", ccd=35)
self.assertEqual(image.getFilter().getName(), "r")
bbox = geom.BoxI(geom.Point2I(200, 100), geom.Extent2I(300, 400))
image = butler.get("someGz_sub",
ccd=35,
bbox=bbox,
imageOrigin="LOCAL",
immediate=True)
self.assertEqual(image.getHeight(), 400)
self.assertEqual(image.getWidth(), 300)
def testParentNormal(self):
"""Test that an object can be found at root location and put into an
output location. Then test that when the output locaiton is used as an
input location, and with a new output location, that the object is
found in the first output location.
"""
testOutput = self.mkdtemp("testOutput")
butler = dafPersist.Butler(inputs={
'root': ROOT,
'mapper': MinMapper1
},
outputs=testOutput)
mapper1 = butler._repos.inputs()[0].repo._mapper
loc = mapper1.map("x", dict(sensor="1,1"), write=True)
self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI")
self.assertEqual(loc.getCppType(), "BoxI")
self.assertEqual(loc.getStorageName(), "PickleStorage")
self.assertEqual(loc.getLocations(), ["foo-1,1.pickle"])
self.assertEqual(loc.getAdditionalData().toString(),
"sensor = \"1,1\"\n")
box = geom.BoxI(geom.PointI(0, 1), geom.PointI(2, 3))
butler.put(box, "x", sensor="1,1")
self.assertTrue(
os.path.exists(os.path.join(testOutput,
loc.getLocations()[0])))
del butler
testOutput2 = self.mkdtemp("testOutput2")
butler = dafPersist.Butler(inputs={
'root': testOutput,
'mapper': MinMapper1
},
outputs=testOutput2)
mapper2 = butler._repos.inputs()[0].repo._mapper
loc = mapper2.map("x", dict(sensor="1,1"))
self.assertEqual(loc.getPythonType(), "lsst.afw.geom.BoxI")
self.assertEqual(loc.getCppType(), "BoxI")
self.assertEqual(loc.getStorageName(), "PickleStorage")
self.assertEqual(loc.getLocations(), ["foo-1,1.pickle"])
self.assertEqual(loc.getStorage().root, testOutput)
self.assertEqual(loc.getAdditionalData().toString(),
"sensor = \"1,1\"\n")
def plotDeblendFamilyRGB(parent, bands=['g', 'r', 'i'],
min=0.01, max=0.5, Q=8, rgbFileFmt=None):
x, y = parent.getX(), parent.getY()
fams = {}
imBbox = geom.BoxI()
for bandName in "GRI".upper():
filterName = "HSC-%s" % bandName
x0, y0 = coaddDict[filterName].getXY0()
x0, y0 = 0, 0
fams[filterName] = familiesDict[filterName].find((x + x0, y + y0), matchRadius=20)
if not fams[filterName]:
return
parent, kids = fams[filterName]
bbox = parent.getFootprint().getBBox()
# Can children extend outside parent BBox?
for kid in kids:
kim = footprintToImage(kid.getFootprint(), coaddDict[filterName].getMaskedImage())
bbox.include(kim.getBBox(afwImage.PARENT))
imBbox.include(bbox)
images = {}
for bandName in bands:
filterName = "HSC-%s" % bandName.upper()
images[bandName] = makeDeblendFamilyMosaic(coaddDict[filterName].getMaskedImage(),
*fams[filterName],
background=-0.1, imBbox=imBbox).makeMosaic(display=None)
for bands in [bands]:
B, G, R = bands
rgb = afwRgb.makeRGB(images[R], images[G], images[B], min, max - min, Q)
afwRgb.displayRGB(rgb, show=True)
if rgbFileFmt:
afwRgb.writeRGB(rgbFileFmt % "".join(bands), rgb)
def makeBBoxFromList(ylist):
"""Given a list [(x0, y0), (xsize, ysize)], probably from a yaml file,
return a BoxI
"""
(x0, y0), (xsize, ysize) = ylist
return geom.BoxI(geom.PointI(x0, y0), geom.ExtentI(xsize, ysize))
def _parallel_box(self, start, end):
llc = lsstGeom.PointI(self.imaging.getMinX(), start)
urc = lsstGeom.PointI(self.imaging.getMaxX(), end)
return lsstGeom.BoxI(llc, urc)
def _serial_box(self, start, end):
llc = lsstGeom.PointI(start, self.imaging.getMinY())
urc = lsstGeom.PointI(end, self.imaging.getMaxY())
return lsstGeom.BoxI(llc, urc)
def assemble_raw(dataId, componentInfo, cls):
"""Called by the butler to construct the composite type "raw".
Note that we still need to define "_raw" and copy various fields over.
Parameters
----------
dataId : `lsst.daf.persistence.dataId.DataId`
The data ID.
componentInfo : `dict`
dict containing the components, as defined by the composite definition
in the mapper policy.
cls : 'object'
Unused.
Returns
-------
exposure : `lsst.afw.image.Exposure`
The assembled exposure.
"""
from lsst.ip.isr import AssembleCcdTask
config = AssembleCcdTask.ConfigClass()
config.doTrim = False
assembleTask = AssembleCcdTask(config=config)
ampExps = componentInfo['raw_amp'].obj
if len(ampExps) == 0:
raise RuntimeError("Unable to read raw_amps for %s" % dataId)
ccd = ampExps[0].getDetector() # the same (full, CCD-level) Detector is attached to all ampExps
#
# Check that the geometry in the metadata matches cameraGeom
#
logger = lsst.log.Log.getLogger("LsstCamMapper")
warned = False
for i, (amp, ampExp) in enumerate(zip(ccd, ampExps)):
ampMd = ampExp.getMetadata().toDict()
if amp.getRawBBox() != ampExp.getBBox(): # Oh dear. cameraGeom is wrong -- probably overscan
if amp.getRawDataBBox().getDimensions() != amp.getBBox().getDimensions():
raise RuntimeError("Active area is the wrong size: %s v. %s" %
(amp.getRawDataBBox().getDimensions(), amp.getBBox().getDimensions()))
if not warned:
logger.warn("amp.getRawBBox() != data.getBBox(); patching. (%s v. %s)",
amp.getRawBBox(), ampExp.getBBox())
warned = True
w, h = ampExp.getBBox().getDimensions()
ow, oh = amp.getRawBBox().getDimensions() # "old" (cameraGeom) dimensions
#
# We could trust the BIASSEC keyword, or we can just assume that
# they've changed the number of overscan pixels (serial and/or
# parallel). As Jim Chiang points out, the latter is safer
#
bbox = amp.getRawHorizontalOverscanBBox()
hOverscanBBox = geom.BoxI(bbox.getBegin(),
geom.ExtentI(w - bbox.getBeginX(), bbox.getHeight()))
bbox = amp.getRawVerticalOverscanBBox()
vOverscanBBox = geom.BoxI(bbox.getBegin(),
geom.ExtentI(bbox.getWidth(), h - bbox.getBeginY()))
amp.setRawBBox(ampExp.getBBox())
amp.setRawHorizontalOverscanBBox(hOverscanBBox)
amp.setRawVerticalOverscanBBox(vOverscanBBox)
#
# This gets all the geometry right for the amplifier, but the size
# of the untrimmed image will be wrong and we'll put the amp
# sections in the wrong places, i.e.
# amp.getRawXYOffset()
# will be wrong. So we need to recalculate the offsets.
#
xRawExtent, yRawExtent = amp.getRawBBox().getDimensions()
x0, y0 = amp.getRawXYOffset()
ix, iy = x0//ow, y0/oh
x0, y0 = ix*xRawExtent, iy*yRawExtent
amp.setRawXYOffset(geom.ExtentI(ix*xRawExtent, iy*yRawExtent))
#
# Check the "IRAF" keywords, but don't abort if they're wrong
#
# Only warn about the first amp, use debug for the others
#
detsec = bboxFromIraf(ampMd["DETSEC"]) if "DETSEC" in ampMd else None
datasec = bboxFromIraf(ampMd["DATASEC"]) if "DATASEC" in ampMd else None
biassec = bboxFromIraf(ampMd["BIASSEC"]) if "BIASSEC" in ampMd else None
logCmd = logger.warn if i == 0 else logger.debug
if detsec and amp.getBBox() != detsec:
logCmd("DETSEC doesn't match for %s (%s != %s)", dataId, amp.getBBox(), detsec)
if datasec and amp.getRawDataBBox() != datasec:
logCmd("DATASEC doesn't match for %s (%s != %s)", dataId, amp.getRawDataBBox(), detsec)
if biassec and amp.getRawHorizontalOverscanBBox() != biassec:
logCmd("BIASSEC doesn't match for %s (%s != %s)",
dataId, amp.getRawHorizontalOverscanBBox(), detsec)
ampDict = {}
for amp, ampExp in zip(ccd, ampExps):
ampDict[amp.getName()] = ampExp
exposure = assembleTask.assembleCcd(ampDict)
md = componentInfo['raw_hdu'].obj
fix_header(md) # No mapper so cannot specify the translator class
exposure.setMetadata(md)
visitInfo = LsstCamMakeRawVisitInfo(logger)(md)
exposure.getInfo().setVisitInfo(visitInfo)
boresight = visitInfo.getBoresightRaDec()
rotangle = visitInfo.getBoresightRotAngle()
if boresight.isFinite():
exposure.setWcs(getWcsFromDetector(exposure.getDetector(), boresight,
90*geom.degrees - rotangle))
else:
# Should only warn for science observations but VisitInfo does not know
logger.warn("Unable to set WCS for %s from header as RA/Dec/Angle are unavailable" %
(dataId,))
return exposure
def generate_cutout(butler,
skymap,
ra,
dec,
band='i',
label='deepCoadd_skyMap',
radius=10.0 * u.arcsec,
psf=True,
verbose=False):
"""Generate a single cutout image.
"""
if not isinstance(radius, u.Quantity):
# Assume that this is in pixel
radius = int(radius)
else:
radius = int(radius.to('arcsec').value / PIXEL_SCALE)
# Width and height of the post-stamps
stamp_shape = (radius * 2 + 1, radius * 2 + 1)
# Coordinate of the image center
coord = geom.SpherePoint(ra * geom.degrees, dec * geom.degrees)
# Make a list of (RA, Dec) that covers the cutout region
radec_list = np.array(sky_cone(ra, dec, radius * PIXEL_SCALE, steps=50)).T
# Retrieve the Tracts and Patches that cover the cutout region
patches, _ = tracts_n_patches(radec_list, skymap)
# Collect the images
images = []
for t, p in patches:
data_id = {
'tract': t,
'patch': p.decode(),
'filter': 'HSC-' + band.upper()
}
if butler.datasetExists(label, data_id):
img = butler.get(label, data_id, immediate=True)
images.append(img)
if len(images) == 0:
if verbose:
print('***** No data at {:.5f} {:.5f} *****'.format(ra, dec))
return None
cutouts = []
idx, bbox_sizes, bbox_origins = [], [], []
for img_patch in images:
# Generate cutout
cut, x0, y0 = make_single_cutout(img_patch, coord, radius)
cutouts.append(cut)
# Original lower corner pixel coordinate
bbox_origins.append([x0, y0])
# New lower corner pixel coordinate
xnew, ynew = cut.getBBox().getBeginX() - x0, cut.getBBox().getBeginY(
) - y0
idx.append([
xnew, xnew + cut.getBBox().getWidth(), ynew,
ynew + cut.getBBox().getHeight()
])
# Pixel size of the cutout region
bbox_sizes.append(cut.getBBox().getWidth() * cut.getBBox().getHeight())
# Stitch cutouts together with the largest bboxes inserted last
stamp_bbox = geom.BoxI(geom.Point2I(0, 0), geom.Extent2I(*stamp_shape))
stamp = afwImage.MaskedImageF(stamp_bbox)
bbox_sorted_ind = np.argsort(bbox_sizes)
for i in bbox_sorted_ind:
masked_img = cutouts[i].getMaskedImage()
stamp[idx[i][0]:idx[i][1], idx[i][2]:idx[i][3]] = masked_img
# Build the new WCS of the cutout
stamp_wcs = build_cutout_wcs(coord, cutouts, bbox_sorted_ind[-1],
bbox_origins)
# The final product of the cutout
if psf:
return (afwImage.ExposureF(stamp, stamp_wcs),
get_psf(cutouts[bbox_sorted_ind[-1]], coord))
return afwImage.ExposureF(stamp, stamp_wcs)
def _fillVisitCatalog(self,
visitCat,
groupedDataRefs,
visitCatDataRef=None):
"""
Fill the visit catalog with visit metadata
Parameters
----------
visitCat: `afw.table.BaseCatalog`
Catalog with schema from _makeFgcmVisitSchema()
groupedDataRefs: `dict`
Dictionary with visit keys, and `list`s of
`lsst.daf.persistence.ButlerDataRef
visitCatDataRef: `lsst.daf.persistence.ButlerDataRef`, optional
Dataref to write visitCat for checkpoints
"""
bbox = geom.BoxI(geom.PointI(0, 0), geom.PointI(1, 1))
for i, visit in enumerate(sorted(groupedDataRefs)):
# We don't use the bypasses since we need the psf info which does
# not have a bypass
# TODO: When DM-15500 is implemented in the Gen3 Butler, this
# can be fixed
# Do not read those that have already been read
if visitCat['used'][i]:
continue
if (i % self.config.nVisitsPerCheckpoint) == 0:
self.log.info("Retrieving metadata for %s %d (%d/%d)" %
(self.config.visitDataRefName, visit, i,
len(groupedDataRefs)))
# Save checkpoint if desired
if visitCatDataRef is not None:
visitCatDataRef.put(visitCat)
# Note that the reference ccd is first in the list (if available).
# The first dataRef in the group will be the reference ccd (if available)
dataRef = groupedDataRefs[visit][0]
exp = dataRef.get(datasetType='calexp_sub',
bbox=bbox,
flags=afwTable.SOURCE_IO_NO_FOOTPRINTS)
visitInfo = exp.getInfo().getVisitInfo()
f = exp.getFilter()
psf = exp.getPsf()
rec = visitCat[i]
rec['visit'] = visit
rec['filtername'] = f.getName()
# TODO DM-26991: when gen2 is removed, gen3 workflow will make it
# much easier to get the wcs's necessary to recompute the pointing
# ra/dec at the center of the camera.
radec = visitInfo.getBoresightRaDec()
rec['telra'] = radec.getRa().asDegrees()
rec['teldec'] = radec.getDec().asDegrees()
rec['telha'] = visitInfo.getBoresightHourAngle().asDegrees()
rec['telrot'] = visitInfo.getBoresightRotAngle().asDegrees()
rec['mjd'] = visitInfo.getDate().get(system=DateTime.MJD)
rec['exptime'] = visitInfo.getExposureTime()
# convert from Pa to millibar
# Note that I don't know if this unit will need to be per-camera config
rec['pmb'] = visitInfo.getWeather().getAirPressure() / 100
# Flag to signify if this is a "deep" field. Not currently used
rec['deepFlag'] = 0
# Relative flat scaling (1.0 means no relative scaling)
rec['scaling'][:] = 1.0
# Median delta aperture, to be measured from stars
rec['deltaAper'] = 0.0
rec['psfSigma'] = psf.computeShape().getDeterminantRadius()
if dataRef.datasetExists(datasetType='calexpBackground'):
# Get background for reference CCD
# This approximation is good enough for now
bgStats = (bg[0].getStatsImage().getImage().array
for bg in dataRef.get(
datasetType='calexpBackground'))
rec['skyBackground'] = sum(
np.median(bg[np.isfinite(bg)]) for bg in bgStats)
else:
self.log.warn(
'Sky background not found for visit %d / ccd %d' %
(visit, dataRef.dataId[self.config.ccdDataRefName]))
rec['skyBackground'] = -1.0
rec['used'] = 1
def makeAmplifierList(ccd):
"""Construct a list of AmplifierBuilder objects
"""
# Much of this will need to be filled in when we know it.
assert len(ccd) > 0
amp = list(ccd['amplifiers'].values())[0]
rawBBox = makeBBoxFromList(amp['rawBBox']) # total in file
xRawExtent, yRawExtent = rawBBox.getDimensions()
readCorners = {"LL": ReadoutCorner.LL,
"LR": ReadoutCorner.LR,
"UL": ReadoutCorner.UL,
"UR": ReadoutCorner.UR}
amplifierList = []
for name, amp in sorted(ccd['amplifiers'].items(), key=lambda x: x[1]['hdu']):
amplifier = Amplifier.Builder()
amplifier.setName(name)
ix, iy = amp['ixy']
perAmpData = amp['perAmpData']
if perAmpData:
x0, y0 = 0, 0 # origin of data within each amp image
else:
x0, y0 = ix*xRawExtent, iy*yRawExtent
rawDataBBox = makeBBoxFromList(amp['rawDataBBox']) # Photosensitive area
xDataExtent, yDataExtent = rawDataBBox.getDimensions()
amplifier.setBBox(geom.BoxI(
geom.PointI(ix*xDataExtent, iy*yDataExtent), rawDataBBox.getDimensions()))
rawBBox = makeBBoxFromList(amp['rawBBox'])
rawBBox.shift(geom.ExtentI(x0, y0))
amplifier.setRawBBox(rawBBox)
rawDataBBox = makeBBoxFromList(amp['rawDataBBox'])
rawDataBBox.shift(geom.ExtentI(x0, y0))
amplifier.setRawDataBBox(rawDataBBox)
rawSerialOverscanBBox = makeBBoxFromList(amp['rawSerialOverscanBBox'])
rawSerialOverscanBBox.shift(geom.ExtentI(x0, y0))
amplifier.setRawHorizontalOverscanBBox(rawSerialOverscanBBox)
rawParallelOverscanBBox = makeBBoxFromList(amp['rawParallelOverscanBBox'])
rawParallelOverscanBBox.shift(geom.ExtentI(x0, y0))
amplifier.setRawVerticalOverscanBBox(rawParallelOverscanBBox)
rawSerialPrescanBBox = makeBBoxFromList(amp['rawSerialPrescanBBox'])
rawSerialPrescanBBox.shift(geom.ExtentI(x0, y0))
amplifier.setRawPrescanBBox(rawSerialPrescanBBox)
if perAmpData:
amplifier.setRawXYOffset(geom.Extent2I(ix*xRawExtent, iy*yRawExtent))
else:
amplifier.setRawXYOffset(geom.Extent2I(0, 0))
amplifier.setReadoutCorner(readCorners[amp['readCorner']])
amplifier.setGain(amp['gain'])
amplifier.setReadNoise(amp['readNoise'])
amplifier.setSaturation(amp['saturation'])
amplifier.setSuspectLevel(amp.get('suspect', np.nan))
# flip data when assembling if needs be (e.g. data from the serial at the top of a CCD)
flipX, flipY = amp.get("flipXY")
amplifier.setRawFlipX(flipX)
amplifier.setRawFlipY(flipY)
# linearity placeholder stuff
amplifier.setLinearityCoeffs([float(val) for val in amp['linearityCoeffs']])
amplifier.setLinearityType(amp['linearityType'])
amplifier.setLinearityThreshold(float(amp['linearityThreshold']))
amplifier.setLinearityMaximum(float(amp['linearityMax']))
amplifier.setLinearityUnits("DN")
amplifierList.append(amplifier)
return amplifierList
def setUp(self):
config = SingleFrameMeasurementTask.ConfigClass()
config.slots.apFlux = 'base_CircularApertureFlux_12_0'
self.schema = afwTable.SourceTable.makeMinimalSchema()
self.measureSources = SingleFrameMeasurementTask(self.schema,
config=config)
width, height = 110, 301
self.mi = afwImage.MaskedImageF(geom.ExtentI(width, height))
self.mi.set(0)
sd = 3 # standard deviation of image
self.mi.getVariance().set(sd * sd)
self.mi.getMask().addMaskPlane("DETECTED")
self.ksize = 31 # size of desired kernel
sigma1 = 1.75
sigma2 = 2 * sigma1
self.exposure = afwImage.makeExposure(self.mi)
self.exposure.setPsf(
measAlg.DoubleGaussianPsf(self.ksize, self.ksize, 1.5 * sigma1, 1,
0.1))
cdMatrix = np.array([1.0, 0.0, 0.0, 1.0])
cdMatrix.shape = (2, 2)
wcs = afwGeom.makeSkyWcs(crpix=geom.PointD(0, 0),
crval=geom.SpherePoint(
0.0, 0.0, geom.degrees),
cdMatrix=cdMatrix)
self.exposure.setWcs(wcs)
#
# Make a kernel with the exactly correct basis functions.
# Useful for debugging
#
basisKernelList = []
for sigma in (sigma1, sigma2):
basisKernel = afwMath.AnalyticKernel(
self.ksize, self.ksize,
afwMath.GaussianFunction2D(sigma, sigma))
basisImage = afwImage.ImageD(basisKernel.getDimensions())
basisKernel.computeImage(basisImage, True)
basisImage /= np.sum(basisImage.getArray())
if sigma == sigma1:
basisImage0 = basisImage
else:
basisImage -= basisImage0
basisKernelList.append(afwMath.FixedKernel(basisImage))
order = 1 # 1 => up to linear
spFunc = afwMath.PolynomialFunction2D(order)
exactKernel = afwMath.LinearCombinationKernel(basisKernelList, spFunc)
exactKernel.setSpatialParameters([[1.0, 0, 0],
[0.0, 0.5 * 1e-2, 0.2e-2]])
rand = afwMath.Random() # make these tests repeatable by setting seed
addNoise = True
if addNoise:
im = self.mi.getImage()
afwMath.randomGaussianImage(im, rand) # N(0, 1)
im *= sd # N(0, sd^2)
del im
xarr, yarr = [], []
for x, y in [
(20, 20),
(60, 20),
(30, 35),
(50, 50),
(20, 90),
(70, 160),
(25, 265),
(75, 275),
(85, 30),
(50, 120),
(70, 80),
(60, 210),
(20, 210),
]:
xarr.append(x)
yarr.append(y)
for x, y in zip(xarr, yarr):
dx = rand.uniform() - 0.5 # random (centered) offsets
dy = rand.uniform() - 0.5
k = exactKernel.getSpatialFunction(1)(
x, y) # functional variation of Kernel ...
b = (k * sigma1**2 / ((1 - k) * sigma2**2)
) # ... converted double Gaussian's "b"
# flux = 80000 - 20*x - 10*(y/float(height))**2
flux = 80000 * (1 + 0.1 * (rand.uniform() - 0.5))
I0 = flux * (1 + b) / (2 * np.pi * (sigma1**2 + b * sigma2**2))
for iy in range(y - self.ksize // 2, y + self.ksize // 2 + 1):
if iy < 0 or iy >= self.mi.getHeight():
continue
for ix in range(x - self.ksize // 2, x + self.ksize // 2 + 1):
if ix < 0 or ix >= self.mi.getWidth():
continue
II = I0 * psfVal(ix, iy, x + dx, y + dy, sigma1, sigma2, b)
Isample = rand.poisson(II) if addNoise else II
self.mi.image[ix, iy, afwImage.LOCAL] += Isample
self.mi.variance[ix, iy, afwImage.LOCAL] += II
bbox = geom.BoxI(geom.PointI(0, 0), geom.ExtentI(width, height))
self.cellSet = afwMath.SpatialCellSet(bbox, 100)
self.footprintSet = afwDetection.FootprintSet(
self.mi, afwDetection.Threshold(100), "DETECTED")
self.catalog = self.measure(self.footprintSet, self.exposure)
for source in self.catalog:
try:
cand = measAlg.makePsfCandidate(source, self.exposure)
self.cellSet.insertCandidate(cand)
except Exception as e:
print(e)
continue
def subtractXTalk(mi, coeffs, minPixelToMask=45000, crosstalkStr="CROSSTALK"):
"""Subtract the crosstalk from MaskedImage mi given a set of coefficients
The pixels affected by signal over minPixelToMask have the crosstalkStr
bit set
"""
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(mi.getMask().getPlaneBitMask("BAD"))
bkgd = afwMath.makeStatistics(mi, afwMath.MEDIAN, sctrl).getValue()
#
# These are the pixels that are bright enough to cause crosstalk (more
# precisely, the ones that we label as causing crosstalk; in reality all
# pixels cause crosstalk)
#
tempStr = "TEMP" # mask plane used to record the bright pixels that we need to mask
msk = mi.getMask()
msk.addMaskPlane(tempStr)
try:
fs = afwDetect.FootprintSet(mi, afwDetect.Threshold(minPixelToMask), tempStr)
mi.getMask().addMaskPlane(crosstalkStr)
afwDisplay.getDisplay().setMaskPlaneColor(crosstalkStr, afwDisplay.MAGENTA)
# the crosstalkStr bit will now be set whenever we subtract crosstalk
fs.setMask(mi.getMask(), crosstalkStr)
crosstalk = mi.getMask().getPlaneBitMask(crosstalkStr)
width, height = mi.getDimensions()
for i in range(nAmp):
bbox = geom.BoxI(geom.PointI(i*(width//nAmp), 0), geom.ExtentI(width//nAmp, height))
ampI = mi.Factory(mi, bbox)
for j in range(nAmp):
if i == j:
continue
bbox = geom.BoxI(geom.PointI(j*(width//nAmp), 0), geom.ExtentI(width//nAmp, height))
if (i + j)%2 == 1:
ampJ = afwMath.flipImage(mi.Factory(mi, bbox), True, False) # no need for a deep copy
else:
ampJ = mi.Factory(mi, bbox, afwImage.LOCAL, True)
msk = ampJ.getMask()
if np.all(msk.getArray() & msk.getPlaneBitMask("BAD")):
# Bad amplifier; ignore it completely --- its effect will
# come out in the bias
continue
msk &= crosstalk
ampJ -= bkgd
ampJ *= coeffs[j][i]
ampI -= ampJ
#
# Clear the crosstalkStr bit in the original bright pixels, where
# tempStr is set
#
msk = mi.getMask()
temp = msk.getPlaneBitMask(tempStr)
xtalk_temp = crosstalk | temp
np_msk = msk.getArray()
mask_indicies = np.where(np.bitwise_and(np_msk, xtalk_temp) == xtalk_temp)
np_msk[mask_indicies] &= getattr(np, np_msk.dtype.name)(~crosstalk)
finally:
msk.removeAndClearMaskPlane(tempStr, True) # added in afw #1853
def bbox(self):
"""Return the detector bounding box from the separate box endpoint
values.
"""
return geom.BoxI(geom.PointI(self.bbox_x0, self.bbox_y0),
geom.PointI(self.bbox_x1, self.bbox_y1))
def generate_cutout(butler,
skymap,
ra,
dec,
band='N708',
data_type='deepCoadd',
half_size=10.0 * u.arcsec,
psf=True,
verbose=False):
"""Generate a single cutout image.
"""
if not isinstance(half_size, u.Quantity):
# Assume that this is in pixel
half_size_pix = int(half_size)
else:
half_size_pix = int(half_size.to('arcsec').value / PIXEL_SCALE)
# Width and height of the post-stamps
stamp_shape = (half_size_pix * 2 + 1, half_size_pix * 2 + 1)
# Make a list of (RA, Dec) that covers the cutout region
radec_list = np.array(
sky_cone(ra,
dec,
half_size_pix * PIXEL_SCALE * u.Unit('arcsec'),
steps=50)).T
# Retrieve the Patches that cover the cutout region
img_patches = _get_patches(butler,
skymap,
radec_list,
band,
data_type=data_type)
if img_patches is None:
if verbose:
print('***** No data at {:.5f} {:.5f} *****'.format(ra, dec))
return None
# Coordinate of the image center
coord = geom.SpherePoint(ra * geom.degrees, dec * geom.degrees)
# Making the stacked cutout
cutouts = []
idx, bbox_sizes, bbox_origins = [], [], []
for img_p in img_patches:
# Generate cutout
cut, x0, y0 = _get_single_cutout(img_p, coord, half_size_pix)
cutouts.append(cut)
# Original lower corner pixel coordinate
bbox_origins.append([x0, y0])
# New lower corner pixel coordinate
xnew, ynew = cut.getBBox().getBeginX() - x0, cut.getBBox().getBeginY(
) - y0
idx.append([
xnew, xnew + cut.getBBox().getWidth(), ynew,
ynew + cut.getBBox().getHeight()
])
# Area of the cutout region on this patch in unit of pixels
# Will reverse rank all the overlapped images by this
bbox_sizes.append(cut.getBBox().getWidth() * cut.getBBox().getHeight())
# Stitch cutouts together with the largest bboxes inserted last
stamp = afwImage.MaskedImageF(
geom.BoxI(geom.Point2I(0, 0), geom.Extent2I(*stamp_shape)))
bbox_sorted_ind = np.argsort(bbox_sizes)
for i in bbox_sorted_ind:
masked_img = cutouts[i].getMaskedImage()
stamp[idx[i][0]:idx[i][1], idx[i][2]:idx[i][3]] = masked_img
# Build the new WCS of the cutout
stamp_wcs = _build_cutout_wcs(coord, cutouts, bbox_sorted_ind[-1],
bbox_origins)
cutout = afwImage.ExposureF(stamp, stamp_wcs)
if bbox_sizes[bbox_sorted_ind[-1]] < (half_size_pix * 2 + 1)**2:
flag = 1
else:
flag = 2
# The final product of the cutout
if psf:
psf = _get_psf(cutouts[bbox_sorted_ind[-1]], coord)
return cutout, psf, flag
return cutout, flag
