def writeThumbnail(self, dataRef, dataset, exposure):
"""Write out exposure to a snapshot file named outfile in the given size.
"""
filename = dataRef.get(dataset + "_filename")[0]
directory = os.path.dirname(filename)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
# Don't fail if directory exists due to race
if e.errno != errno.EEXIST:
raise e
binning = self.config.thumbnailBinning
binnedImage = afwMath.binImage(exposure.getMaskedImage(), binning, binning, afwMath.MEAN)
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setAndMask(binnedImage.getMask().getPlaneBitMask(["SAT", "BAD", "INTRP"]))
stats = afwMath.makeStatistics(binnedImage,
afwMath.MEDIAN | afwMath.STDEVCLIP | afwMath.MAX, statsCtrl)
low = stats.getValue(afwMath.MEDIAN) - self.config.thumbnailStdev*stats.getValue(afwMath.STDEVCLIP)
try:
makeRGB(binnedImage, binnedImage, binnedImage, minimum=low, dataRange=self.config.thumbnailRange,
Q=self.config.thumbnailQ, fileName=filename,
saturatedBorderWidth=self.config.thumbnailSatBorder,
saturatedPixelValue=stats.getValue(afwMath.MAX))
except Exception as exc:
# makeRGB can fail with:
# SystemError: <built-in method flush of _io.BufferedWriter object at 0x2b2a0081c938>
# returned a result with an error set
# This unhelpful error comes from the bowels of matplotlib, so not much we can do about it
# except keep it from being fatal.
self.log.warn("Unable to write thumbnail for %s: %s", dataRef.dataId, exc)
def testMakeRGBResize(self):
"""Test the function that does it all, including rescaling"""
rgb.makeRGB(self.images[R], self.images[G], self.images[B], xSize=40, ySize=60)
with utilsTests.getTempFilePath(".png") as fileName:
rgb.makeRGB(self.images[R], self.images[G], self.images[B], fileName=fileName, rescaleFactor=0.5)
self.assertTrue(os.path.exists(fileName))
def writeThumbnail(self, dataRef, dataset, exposure):
"""Write out exposure to a snapshot file named outfile in the given size.
"""
filename = dataRef.get(dataset + "_filename")[0]
directory = os.path.dirname(filename)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError as e:
# Don't fail if directory exists due to race
if e.errno != errno.EEXIST:
raise e
binning = self.config.thumbnailBinning
binnedImage = afwMath.binImage(exposure.getMaskedImage(), binning,
binning, afwMath.MEAN)
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setAndMask(binnedImage.getMask().getPlaneBitMask(
["SAT", "BAD", "INTRP"]))
stats = afwMath.makeStatistics(
binnedImage, afwMath.MEDIAN | afwMath.STDEVCLIP | afwMath.MAX,
statsCtrl)
low = stats.getValue(
afwMath.MEDIAN) - self.config.thumbnailStdev * stats.getValue(
afwMath.STDEVCLIP)
makeRGB(binnedImage,
binnedImage,
binnedImage,
minimum=low,
dataRange=self.config.thumbnailRange,
Q=self.config.thumbnailQ,
fileName=filename,
saturatedBorderWidth=self.config.thumbnailSatBorder,
saturatedPixelValue=stats.getValue(afwMath.MAX))
def testMakeRGBResize(self):
"""Test the function that does it all, including rescaling"""
rgb.makeRGB(self.images[R], self.images[G],
self.images[B], xSize=40, ySize=60)
with lsst.utils.tests.getTempFilePath(".png") as fileName:
rgb.makeRGB(self.images[R], self.images[G],
self.images[B], fileName=fileName, rescaleFactor=0.5)
self.assertTrue(os.path.exists(fileName))
def testMakeRGBResize(self):
"""Test the function that does it all, including rescaling"""
fileName = "makeRGB.png"
rgb.makeRGB(self.images[R], self.images[G], self.images[B], xSize=40, ySize=60)
with Tempfile(fileName, remove=True):
rgb.makeRGB(self.images[R], self.images[G], self.images[B], fileName=fileName, rescaleFactor=0.5)
self.assertTrue(os.path.exists(fileName))
def testMakeRGB(self):
"""Test the function that does it all"""
satValue = 1000.0
with utilsTests.getTempFilePath(".png") as fileName:
red = saturate(self.images[R], satValue)
green = saturate(self.images[G], satValue)
blue = saturate(self.images[B], satValue)
rgb.makeRGB(red, green, blue, self.min, self.range, self.Q, fileName=fileName,
saturatedBorderWidth=1, saturatedPixelValue=2000)
def testMakeRGB(self):
"""Test the function that does it all"""
fileName = "makeRGB.png"
assert "imageLib.ImageF" in repr(self.images[R]) # check that ImageF is supported
with Tempfile(fileName, remove=True):
rgb.makeRGB(self.images[R], self.images[G], self.images[B],
self.min, self.range, self.Q, fileName=fileName)
self.assertTrue(os.path.exists(fileName))
def testMakeRGBSaturated(self):
"""Test the function that does it all when there are saturated pixels"""
fileName = "makeRGB.png"
satValue = 1000.0
with Tempfile(fileName, remove=True):
red = saturate(self.images[R], satValue)
green = saturate(self.images[G], satValue)
blue = saturate(self.images[B], satValue)
assert "imageLib.MaskedImageF" in repr(red) # check that MaskedImageF is supported
rgb.makeRGB(red, green, blue, self.min, self.range, self.Q, fileName=fileName,
saturatedBorderWidth=1, saturatedPixelValue=2000)
self.assertTrue(os.path.exists(fileName))
def testMakeRGBU(self):
"""Test the function that does it all works with unsigned short images"""
rgbImages = [afwImage.ImageU(_.getArray().astype('uint16')) for _ in [
self.images[R], self.images[G], self.images[B]]]
assert "imageLib.ImageU" in repr(rgbImages[0]) # check that ImageU is supported
rgbImage = rgb.makeRGB(*rgbImages, minimum=self.min, range=self.range, Q=self.Q)
def plotDeblendFamilyRGB(parent,
bands=['g', 'r', 'i'],
min=0.01,
max=0.5,
Q=8,
rgbFileFmt=None):
x, y = parent.getX(), parent.getY()
fams = {}
imBbox = afwGeom.BoxI()
for bandName in "GRI".upper():
filterName = "HSC-%s" % bandName
x0, y0 = coaddDict[filterName].getXY0()
x0, y0 = 0, 0
fams[filterName] = familiesDict[filterName].find((x + x0, y + y0),
matchRadius=20)
if not fams[filterName]:
return
parent, kids = fams[filterName]
bbox = parent.getFootprint().getBBox()
for kid in kids:
kim = footprintToImage(kid.getFootprint(),
coaddDict[filterName].getMaskedImage())
bbox.include(kim.getBBox(afwImage.PARENT))
imBbox.include(bbox)
images = {}
for bandName in bands:
filterName = "HSC-%s" % bandName.upper()
images[bandName] = makeDeblendFamilyMosaic(
coaddDict[filterName].getMaskedImage(),
*fams[filterName],
background=-0.1,
imBbox=imBbox).makeMosaic(display=None)
for bands in [bands]:
B, G, R = bands
rgb = afwRgb.makeRGB(images[R], images[G], images[B], min, max - min,
Q)
afwRgb.displayRGB(rgb, show=True)
if rgbFileFmt:
afwRgb.writeRGB(rgbFileFmt % "".join(bands), rgb)
def make_rgb_image(ra,
dec,
radius,
butler=None,
skymap=None,
rgb='irg',
Q=8,
dataRange=0.6,
root=ROOT,
img_size=None,
return_wcs=False,
coadd_label='deepCoadd_calexp'):
if butler is None:
butler = lsst.daf.persistence.Butler(root)
if skymap is None:
skymap = butler.get('deepCoadd_skyMap', immediate=True)
if type(radius) == float or type(radius) == int:
radius *= u.arcsec
colors = []
for band in rgb:
stamp = make_stamp(ra,
dec,
radius,
band=band,
butler=butler,
skymap=skymap,
coadd_label=coadd_label)
if stamp:
colors.append(stamp.getMaskedImage())
if return_wcs and band == rgb[0]:
wcs = stamp.getWcs()
if len(colors) == 0:
return None
rgb_kws = {'Q': Q, 'dataRange': dataRange}
if img_size is not None:
rgb_kws['xSize'] = img_size
img = afwRgb.makeRGB(*colors, **rgb_kws)
return (img, wcs) if return_wcs else img
def plotDeblendFamilyRGB(parent, bands=['g', 'r', 'i'],
min=0.01, max=0.5, Q=8, rgbFileFmt=None):
x, y = parent.getX(), parent.getY()
fams = {}
imBbox = afwGeom.BoxI()
for bandName in "GRI".upper():
filterName = "HSC-%s" % bandName
x0, y0 = coaddDict[filterName].getXY0()
x0, y0 = 0, 0
fams[filterName] = familiesDict[filterName].find((x + x0, y + y0), matchRadius=20)
if not fams[filterName]:
return
parent, kids = fams[filterName]
bbox = parent.getFootprint().getBBox()
for kid in kids:
kim = footprintToImage(kid.getFootprint(), coaddDict[filterName].getMaskedImage())
bbox.include(kim.getBBox(afwImage.PARENT))
imBbox.include(bbox)
images = {}
for bandName in bands:
filterName = "HSC-%s" % bandName.upper()
images[bandName] = makeDeblendFamilyMosaic(coaddDict[filterName].getMaskedImage(),
*fams[filterName],
background=-0.1, imBbox=imBbox).makeMosaic(display=None)
for bands in [bands]:
B, G, R = bands
rgb = afwRgb.makeRGB(images[R], images[G], images[B], min, max - min, Q)
afwRgb.displayRGB(rgb, show=True)
if rgbFileFmt:
afwRgb.writeRGB(rgbFileFmt % "".join(bands), rgb)
def coaddColourImageFull(root, ra, dec, size, filt='gri',
prefix='hsc_coadd_cutout',
info1=None, info2=None, info3=None,
min=-0.0, max=0.70, Q=10, name=None, localMax=True,
scaleBar=10, butler=None, verbose=False):
"""General full colored picture of cutout."""
pipeVersion = dafPersist.eupsVersions.EupsVersions().versions['hscPipe']
if StrictVersion(pipeVersion) >= StrictVersion('3.9.0'):
coaddData = "deepCoadd_calexp"
else:
coaddData = "deepCoadd"
# Get the SkyMap of the database
if butler is None:
try:
butler = dafPersist.Butler(root)
if verbose:
print SEP
print "### Load in the Butler"
except Exception:
print WAR
print '### Can not load the correct Butler!'
skyMap = butler.get("deepCoadd_skyMap", immediate=True)
# [Ra, Dec] list
raDec = afwCoord.Coord(ra*afwGeom.degrees, dec*afwGeom.degrees)
raList, decList = getCircleRaDec(ra, dec, size)
points = map(lambda x, y: afwGeom.Point2D(x, y), raList, decList)
raDecList = map(lambda x: afwCoord.IcrsCoord(x), points)
# Expected size and center position
dimExpect = (2 * size + 1)
# cenExpect = (dimExpect/2.0, dimExpect/2.0)
# Create a empty array
# For RGB image, the data type should be uint8
rgbEmpty = np.zeros((dimExpect, dimExpect, 3), dtype="uint8")
# Check the choice of filters
if len(filt) is not 3:
raise Exception("Have to be three filters!")
elif not (isHscFilter(filt[0]) & isHscFilter(filt[1]) &
isHscFilter(filt[2])):
raise Exception("Not all filters are valid !")
# Get the correct HSC filter name
filter1 = "HSC-%s" % filt[0].upper()
filter2 = "HSC-%s" % filt[1].upper()
filter3 = "HSC-%s" % filt[2].upper()
filtArr = [filter1, filter2, filter3]
# Cutout size
cutoutSize = int(size)
"""
Figure out the area we want, and read the data.
For coadds the WCS is the same in all bands,
but the code handles the general case
Start by finding the tract and patch
"""
matches = skyMap.findTractPatchList(raDecList)
tractList, patchList = getTractPatchList(matches)
nPatch = len(patchList)
# Output RGB image
if verbose:
print SEP
print "### WILL DEAL WITH %d (TRACT, PATCH)" % nPatch
print SEP
outRgb = prefix + '_' + filt + '_color.png'
newX = []
newY = []
boxX = []
boxY = []
boxSize = []
rgbArr = []
# Go through all these images
for j in range(nPatch):
# Tract, patch
tract, patch = tractList[j], patchList[j]
if verbose:
print "### Dealing with %d - %s" % (tract, patch)
# Check if the coordinate is available in all three bands.
# Change the method, try to generate something as long as it is
# covered by at least one band
images = {}
for i in range(3):
try:
# Find the coadd image
coadd = butler.get(coaddData, tract=tract, patch=patch,
filter=filtArr[i], immediate=True)
# Get the WCS information
wcs = coadd.getWcs()
# Convert the central coordinate from Ra,Dec to pixel unit
pixel = wcs.skyToPixel(raDec)
pixel = afwGeom.Point2I(pixel)
# Define the bounding box for the central pixel
bbox = afwGeom.Box2I(pixel, pixel)
# Grow the bounding box to the desired size
bbox.grow(int(cutoutSize))
xOri, yOri = bbox.getBegin()
# Compare to the coadd image, and clip
bbox.clip(coadd.getBBox(afwImage.PARENT))
subImage = afwImage.ExposureF(coadd, bbox,
afwImage.PARENT)
# Extract the image array
images[i] = subImage.getMaskedImage().getImage()
# Get the size and begginng coordinates of the BBox
bWidth = bbox.getWidth()
bHeight = bbox.getHeight()
bXbegin = bbox.getBeginX()
bYbegin = bbox.getBeginY()
except Exception:
print WAR
print "### Not available in %d - %s - %s" % (tract, patch,
filtArr[i])
images[i] = None
if not ((images[0] is None) and (images[1] is None) and (
images[2] is None)):
# Image from at least one band is available
# So bWidth, bHeight, bXbegin, bYbegin should be defined
boxX.append(bWidth)
boxY.append(bHeight)
boxSize.append(bWidth * bHeight)
newX.append(bXbegin - xOri)
newY.append(bYbegin - yOri)
for l in range(3):
if images[0] is not None:
bCut = images[0]
else:
# Replace the unavailable data with zero array XXX
bCut = np.zeros([bHeight, bWidth])
if images[1] is not None:
gCut = images[1]
else:
# Replace the unavailable data with zero array XXX
gCut = np.zeros([bHeight, bWidth])
if images[2] is not None:
rCut = images[2]
else:
# Replace the unavailable data with zero array XXX
rCut = np.zeros([bHeight, bWidth])
# Generate the RGB image
# 15/04/22: min ==> minimum
imgRgb = afwRgb.makeRGB(rCut, gCut, bCut, minimum=min,
range=(max - min), Q=Q,
saturatedPixelValue=None)
rgbArr.append(imgRgb)
else:
# Bypass the bad data
print WAR
print "### NO DATA IS AVAILABLE IN %d - %s" % (tract, patch)
print WAR
"""
try:
# Get the metadata
md1 = butler.get("deepCoadd_md", immediate=True,
tract=tract, patch=patch, filter=filter1)
md2 = butler.get("deepCoadd_md", immediate=True,
tract=tract, patch=patch, filter=filter2)
md3 = butler.get("deepCoadd_md", immediate=True,
tract=tract, patch=patch, filter=filter3)
except Exception, errMsg:
print "#########################################################"
print " The galaxy is not available in %d - %s" % (tract, patch)
print "#########################################################"
print errMsg
else:
#Then we can read the desired pixels
images = {}
# Go through the three bands
for i in range(3):
# Find the file of the coadd image
coadd = butler.get(coaddData, tract=tract, patch=patch,
filter=filtArr[i], immediate=True)
# Get the WCS information
wcs = coadd.getWcs()
# Convert the central coordinate from Ra,Dec to pixel unit
pixel = wcs.skyToPixel(raDec)
pixel = afwGeom.Point2I(pixel)
# Define the bounding box for the central pixel
bbox = afwGeom.Box2I(pixel, pixel)
# Grow the bounding box to the desired size
bbox.grow(int(cutoutSize))
xOri, yOri = bbox.getBegin()
# Compare to the coadd image, and clip
bbox.clip(coadd.getBBox(afwImage.PARENT))
# Get the masked image
try:
subImage = afwImage.ExposureF(coadd, bbox,
afwImage.PARENT)
# Extract the image array
images[i] = subImage.getMaskedImage().getImage()
bboxGood = True
if i == 1:
boxX.append(bbox.getWidth())
boxY.append(bbox.getHeight())
boxSize.append(bbox.getWidth() * bbox.getHeight())
newX.append(bbox.getBeginX() - xOri)
newY.append(bbox.getBeginY() - yOri)
except:
print '### SOMETHING IS WRONG WITH THIS BOUNDING BOX !!'
print " %d -- %s -- %s " % (tract, patch, filtArr[i])
print " Bounding Box Size: %d" % (bbox.getWidth() *
bbox.getHeight())
bboxGood = False
continue
if bboxGood:
# Define the Blue, Green, and Red channels
# These cutouts are still HSC ImageF object, not numpy array
bCut, gCut, rCut = images[0], images[1], images[2]
# Generate the RGB image
# 15/04/22: min ==> minimum
imgRgb = afwRgb.makeRGB(rCut, gCut, bCut, minimum=min,
range=(max - min), Q=Q,
saturatedPixelValue=None)
rgbArr.append(imgRgb)
else:
continue
"""
# Number of returned RGB image
nReturn = len(rgbArr)
if verbose:
print "### Return %d Useful Images" % nReturn
if nReturn > 0:
# XXX Test, should be removed later
if len(rgbArr) != len(boxSize):
raise Exception("### Something is weird here !")
indSize = np.argsort(boxSize)
# Go through the returned images, put them in the cutout region
for n in range(nReturn):
ind = indSize[n]
# This could lead to problem FIXME
rgbUse = rgbArr[ind]
for k in range(3):
rgbEmpty[newY[ind]:(newY[ind] + boxY[ind]),
newX[ind]:(newX[ind] + boxX[ind]),
k] = rgbUse[:, :, k]
imgRgb = rgbEmpty
# Add a scale bar
if scaleBar is not None:
sLength = ((scaleBar * 1.0) / 0.168) / (dimExpect * 1.0)
sString = "%d\"" % int(scaleBar)
else:
sLength = None
sString = None
# Better way to show the image
saveRgbPng(outRgb, imgRgb, name=name,
info1=info1, info2=info2, info3=info3,
sLength=sLength, sString=sString)
else:
print WAR
print "### NO COLOR IMAGE IS GENERATED FOR THIS OBJECT !!"
print WAR
# get a butler
butler = dp.Butler('output_data')
dataId = {'tract': 0, 'patch': '0,0'}
bandpass_color_map = {'green': 'r', 'red': 'i', 'blue': 'g'}
# get ref catalog
refs = {}
exposures = {}
for bandpass in bandpass_color_map.values():
dataId['filter'] = bandpass
refs[bandpass] = butler.get('deepCoadd_ref', dataId=dataId)
exposures[bandpass] = butler.get('deepCoadd', dataId=dataId)
rgb_im = rgb.makeRGB(
*(exposures[bandpass_color_map[color]].getMaskedImage().getImage()
for color in ('red', 'green', 'blue')))
item = exposures.popitem()
dims = item[1].getDimensions()
exposures.update((item, ))
fig = plt.figure(figsize=(10, 10))
plt.imshow(rgb_im, interpolation='nearest')
# Uncomment the following line to plot the detections
#plt.scatter(refs['g'].getX(), dims[1]-refs['g'].getY(), edgecolors='none', alpha=0.3)
plt.xlim(0, dims[0])
plt.ylim(dims[1], 0)
plt.show()
def testStars2(self):
"""Test creating an RGB image using makeRGB"""
rgbImage = rgb.makeRGB(self.images[R], self.images[G], self.images[B])
if display:
rgb.displayRGB(rgbImage)
def mpl_display_patch(self,
hsc_bands='IRG',
show=False,
subplots_kw={},
imshow_kw={},
subplots=None,
rgb_kw={}):
"""
Display RGB color image of patch using matplotlib.
Parameters
----------
hsc_bands : str, optional
HSC bands in RGB order.
show : bool, optional
If True, show figure.
subplots_kw : dict, optional
plt.subplots keywords.
imshow_kw : dict, optional
plt.imshow keywords.
subplots : tuple, optional
Matplotlib figure and axis (will be created if None).
rgb_kw : dict
afwRgb.makeRGB keywords.
Returns
-------
curret_axis : lsst.pipe.base.Struct
Matplotlib figure, axis, and the bbox.
"""
rgb_kw_default = {'Q': 8, 'dataRange': 1.25}
rgb_kw = utils.check_kwargs_defaults(rgb_kw, rgb_kw_default)
images = {'R': None, 'G': None, 'B': None}
for rgb_label, band in zip('RGB', hsc_bands):
images[rgb_label] = self.rgb_images[band]
img = afwRgb.makeRGB(images['R'], images['G'], images['B'], **rgb_kw)
if subplots is None:
fig, ax = plt.subplots(subplot_kw={
'xticks': [],
'yticks': []
},
**subplots_kw)
else:
fig, ax = subplots
ax.imshow(img, origin='lower', **imshow_kw)
if show:
try:
import RaiseWindow
except:
pass
plt.show()
self.current_axis = lsst.pipe.base.Struct(fig=fig,
ax=ax,
bbox=self.exp.getBBox())
return self.current_axis
def mpl_display_cutout(self,
coord_hsc,
size=100,
hsc_bands='IRG',
show=False,
subplots_kw={},
imshow_kw={},
subplots=None,
rgb_kw={}):
"""
Display RGB color image of cutout using matplotlib.
Parameters
----------
coord_hsc : tuple
Central coordinate in HSC tract system.
size : int, optional
Size to grow bbox in all directions.
hsc_bands : str, optional
HSC bands in RGB order.
show : bool, optional
If True, show figure.
subplots_kw : dict, optional
plt.subplots keywords.
imshow_kw : dict, optional
plt.imshow keywords.
subplots : tuple, optional
Matplotlib figure and axis (will be created if None).
rgb_kw : dict
afwRgb.makeRGB keywords.
Returns
-------
curret_axis : lsst.pipe.base.Struct
Matplotlib figure, axis, and the bbox.
"""
rgb_kw_default = {'Q': 8, 'dataRange': 1.25}
rgb_kw = utils.check_kwargs_defaults(rgb_kw, rgb_kw_default)
cutout = {'R': None, 'G': None, 'B': None}
for rgb_label, band in zip('RGB', hsc_bands):
rgb = self.rgb_images[band]
mi = imtools.get_cutout(coord_hsc, size, exp=rgb)
cutout[rgb_label] = mi
img = afwRgb.makeRGB(cutout['R'], cutout['G'], cutout['B'], **rgb_kw)
if subplots is None:
fig, ax = plt.subplots(subplot_kw={
'xticks': [],
'yticks': []
},
**subplots_kw)
else:
fig, ax = subplots
ax.imshow(img, origin='lower', **imshow_kw)
if show:
try:
import RaiseWindow
except:
pass
plt.show()
self.current_axis = lsst.pipe.base.Struct(fig=fig,
ax=ax,
bbox=mi.getBBox())
return self.current_axis
import lsst.afw.display.rgb as rgb
# get a butler
butler = dp.Butler('output_data')
dataId = {'tract':0, 'patch':'0,0'}
bandpass_color_map = {'green':'r', 'red':'i', 'blue':'g'}
# get ref catalog
refs = {}
exposures = {}
for bandpass in bandpass_color_map.itervalues():
dataId['filter'] = bandpass
refs[bandpass] = butler.get('deepCoadd_ref', dataId=dataId)
exposures[bandpass] = butler.get('deepCoadd', dataId=dataId)
rgb_im = rgb.makeRGB(*(exposures[bandpass_color_map[color]].getMaskedImage().getImage()
for color in ('red', 'green', 'blue')))
item = exposures.popitem()
dims = item[1].getDimensions()
exposures.update((item,))
fig = plt.figure(figsize=(10,10))
plt.imshow(rgb_im, interpolation='nearest')
# Uncomment the following line to plot the detections
#plt.scatter(refs['g'].getX(), dims[1]-refs['g'].getY(), edgecolors='none', alpha=0.3)
plt.xlim(0, dims[0])
plt.ylim(dims[1], 0)
plt.show()
def coaddColourImageFull(root,
ra,
dec,
size,
filt='gri',
prefix='hsc_coadd_cutout',
info1=None,
info2=None,
info3=None,
min=-0.0,
max=0.70,
Q=10,
name=None,
localMax=True,
scaleBar=10,
butler=None,
verbose=False):
"""General full colored picture of cutout."""
# No longer support hscPipe < 4
coaddData = "deepCoadd_calexp"
# See if we are using hscPipe > 5
try:
dafPersist.eupsVersions.EupsVersions().versions['hscPipe']
hscPipe5 = False
except AttributeError:
hscPipe5 = True
# Get the SkyMap of the database
if butler is None:
try:
butler = dafPersist.Butler(root)
except Exception:
print('\n### Can not load the correct Butler!')
skyMap = butler.get("deepCoadd_skyMap", immediate=True)
# [Ra, Dec] list
raDec = afwCoord.Coord(ra * afwGeom.degrees, dec * afwGeom.degrees)
raList, decList = getCircleRaDec(ra, dec, size)
points = map(lambda x, y: afwGeom.Point2D(x, y), raList, decList)
raDecList = map(lambda x: afwCoord.IcrsCoord(x), points)
# Expected size and center position
dimExpect = int(2 * size + 1)
# cenExpect = (dimExpect / 2.0, dimExpect / 2.0)
# Create a empty array
# For RGB image, the data type should be uint8
rgbEmpty = np.zeros((dimExpect, dimExpect, 3), dtype="uint8")
# Check the choice of filters
if len(filt) is not 3:
raise Exception("Have to be three filters!")
elif not (isHscFilter(filt[0]) & isHscFilter(filt[1])
& isHscFilter(filt[2])):
raise Exception("Not all filters are valid !")
# Get the correct HSC filter name
filter1 = "HSC-%s" % filt[0].upper()
filter2 = "HSC-%s" % filt[1].upper()
filter3 = "HSC-%s" % filt[2].upper()
filtArr = [filter1, filter2, filter3]
# Cutout size
cutoutSize = int(size)
"""
Figure out the area we want, and read the data.
For coadds the WCS is the same in all bands,
but the code handles the general case
Start by finding the tract and patch
"""
matches = skyMap.findTractPatchList(raDecList)
tractList, patchList = getTractPatchList(matches)
nPatch = len(patchList)
# Output RGB image
if verbose:
print("\n### WILL DEAL WITH %d (TRACT, PATCH)" % nPatch)
outRgb = prefix + '_' + filt + '_color.png'
newX = []
newY = []
boxX = []
boxY = []
boxSize = []
rgbArr = []
# Go through all these images
for j in range(nPatch):
# Tract, patch
tract, patch = tractList[j], patchList[j]
if verbose:
print("\n### Dealing with %d - %s" % (tract, patch))
# Check if the coordinate is available in all three bands.
# Change the method, try to generate something as long as it is
# covered by at least one band
images = {}
for i in range(3):
try:
# Find the coadd image
coadd = butler.get(coaddData,
tract=tract,
patch=patch,
filter=filtArr[i],
immediate=True)
# Get the WCS information
wcs = coadd.getWcs()
# Convert the central coordinate from Ra,Dec to pixel unit
pixel = wcs.skyToPixel(raDec)
pixel = afwGeom.Point2I(pixel)
# Define the bounding box for the central pixel
bbox = afwGeom.Box2I(pixel, pixel)
# Grow the bounding box to the desired size
bbox.grow(int(cutoutSize))
xOri, yOri = bbox.getBegin()
# Compare to the coadd image, and clip
bbox.clip(coadd.getBBox(afwImage.PARENT))
subImage = afwImage.ExposureF(coadd, bbox, afwImage.PARENT)
# Extract the image array
images[i] = subImage.getMaskedImage().getImage()
# Get the size and begginng coordinates of the BBox
bWidth = bbox.getWidth()
bHeight = bbox.getHeight()
bXbegin = bbox.getBeginX()
bYbegin = bbox.getBeginY()
except Exception:
print("\n### Not available in %d - %s - %s" %
(tract, patch, filtArr[i]))
images[i] = None
if not ((images[0] is None) and (images[1] is None) and
(images[2] is None)):
# Image from at least one band is available
# So bWidth, bHeight, bXbegin, bYbegin should be defined
boxX.append(bWidth)
boxY.append(bHeight)
boxSize.append(bWidth * bHeight)
newX.append(bXbegin - xOri)
newY.append(bYbegin - yOri)
for l in range(3):
if images[0] is not None:
bCut = images[0]
else:
# Replace the unavailable data with zero array XXX
bCut = np.zeros([bHeight, bWidth])
if images[1] is not None:
gCut = images[1]
else:
# Replace the unavailable data with zero array XXX
gCut = np.zeros([bHeight, bWidth])
if images[2] is not None:
rCut = images[2]
else:
# Replace the unavailable data with zero array XXX
rCut = np.zeros([bHeight, bWidth])
# Generate the RGB image
# 15/04/22: min ==> minimum
imgRgb = afwRgb.makeRGB(rCut,
gCut,
bCut,
minimum=min,
dataRange=(max - min),
Q=Q,
saturatedPixelValue=None)
rgbArr.append(imgRgb)
else:
# Bypass the bad data
print("\n### NO DATA IS AVAILABLE IN %d - %s" % (tract, patch))
# Number of returned RGB image
nReturn = len(rgbArr)
if verbose:
print("\n### Return %d Useful Images" % nReturn)
if nReturn > 0:
if len(rgbArr) != len(boxSize):
raise Exception("### Something is weird here !")
indSize = np.argsort(boxSize)
# Go through the returned images, put them in the cutout region
for n in range(nReturn):
ind = indSize[n]
# This could lead to problem FIXME
rgbUse = rgbArr[ind]
for k in range(3):
rgbEmpty[newY[ind]:(newY[ind] + boxY[ind]),
newX[ind]:(newX[ind] + boxX[ind]), k] = rgbUse[:, :,
k]
imgRgb = rgbEmpty
# Add a scale bar
if scaleBar is not None:
sLength = ((scaleBar * 1.0) / 0.168) / (dimExpect * 1.0)
sString = "%d\"" % int(scaleBar)
else:
sLength = None
sString = None
# Better way to show the image
saveRgbPng(outRgb,
imgRgb,
name=name,
info1=info1,
info2=info2,
info3=info3,
sLength=sLength,
sString=sString)
else:
print("\n### NO COLOR IMAGE IS GENERATED FOR THIS OBJECT !!")
maxShow = max
# To see if data are available for all the cut-out region
if (bCut.getHeight() < dimExpect) or (bCut.getWidth() < dimExpect):
print("\n### Only part of the desired cutout-region is returned !")
# Define the name of the output file
outRgb = prefix + '_' + filt + '_part_color.png'
partial = True
else:
outRgb = prefix + '_' + filt + '_color.png'
partial = False
# Generate the RGB image
# 15/04/22: min ==> minimum
imgRgb = afwRgb.makeRGB(rCut,
gCut,
bCut,
minimum=min,
dataRange=(maxShow - min),
Q=Q,
saturatedPixelValue=None)
if partial:
for k in range(3):
rgbEmpty[newY[k]:(newY[k] + images[k].getHeight()),
newX[k]:(newX[k] + images[k].getWidth()),
k] = imgRgb[:, :, k]
imgRgb = rgbEmpty
# Add a scale bar
if scaleBar is not None:
sLength = ((scaleBar * 1.0) / 0.168) / (dimExpect * 1.0)
sString = "%d\"" % int(scaleBar)
else:
sLength = None
class SubaruIsrTask(IsrTask):
ConfigClass = SubaruIsrConfig
def __init__(self, *args, **kwargs):
super(SubaruIsrTask, self).__init__(*args, **kwargs)
self.makeSubtask("crosstalk")
if self.config.doWriteVignettePolygon:
theta = numpy.linspace(0,
2 * numpy.pi,
num=self.config.numPolygonPoints,
endpoint=False)
x = self.config.vignette.radius * numpy.cos(
theta) + self.config.vignette.xCenter
y = self.config.vignette.radius * numpy.sin(
theta) + self.config.vignette.yCenter
points = numpy.array([x, y]).transpose()
self.vignettePolygon = Polygon(
[afwGeom.Point2D(x, y) for x, y in reversed(points)])
def runDataRef(self, sensorRef):
self.log.log(self.log.INFO,
"Performing ISR on sensor %s" % (sensorRef.dataId))
ccdExposure = sensorRef.get('raw')
if self.config.removePcCards: # Remove any PC00N00M cards in the header
raw_md = sensorRef.get("raw_md")
nPc = 0
for i in (
1,
2,
):
for j in (
1,
2,
):
k = "PC%03d%03d" % (i, j)
for md in (raw_md, ccdExposure.getMetadata()):
if md.exists(k):
md.remove(k)
nPc += 1
if nPc:
self.log.log(
self.log.INFO, "Recreating Wcs after stripping PC00n00m" %
(sensorRef.dataId))
ccdExposure.setWcs(afwImage.makeWcs(raw_md))
ccdExposure = self.convertIntToFloat(ccdExposure)
ccd = ccdExposure.getDetector()
for amp in ccd:
self.measureOverscan(ccdExposure, amp)
if self.config.doSaturation:
self.saturationDetection(ccdExposure, amp)
if self.config.doOverscan:
ampImage = afwImage.MaskedImageF(ccdExposure.getMaskedImage(),
amp.getRawDataBBox(),
afwImage.PARENT)
overscan = afwImage.MaskedImageF(
ccdExposure.getMaskedImage(),
amp.getRawHorizontalOverscanBBox(), afwImage.PARENT)
overscanArray = overscan.getImage().getArray()
median = numpy.ma.median(
numpy.ma.masked_where(overscan.getMask().getArray(),
overscanArray))
bad = numpy.where(
numpy.abs(overscanArray -
median) > self.config.overscanMaxDev)
overscan.getMask().getArray()[bad] = overscan.getMask(
).getPlaneBitMask("SAT")
statControl = afwMath.StatisticsControl()
statControl.setAndMask(
ccdExposure.getMaskedImage().getMask().getPlaneBitMask(
"SAT"))
lsstIsr.overscanCorrection(
ampMaskedImage=ampImage,
overscanImage=overscan,
fitType=self.config.overscanFitType,
order=self.config.overscanOrder,
collapseRej=self.config.overscanRej,
statControl=statControl,
)
if self.config.doVariance:
# Ideally, this should be done after bias subtraction,
# but CCD assembly demands a variance plane
ampExposure = ccdExposure.Factory(ccdExposure,
amp.getRawDataBBox(),
afwImage.PARENT)
self.updateVariance(ampExposure, amp)
ccdExposure = self.assembleCcd.assembleCcd(ccdExposure)
ccd = ccdExposure.getDetector()
doRotateCalib = False # Rotate calib images for bias/dark/flat correction?
nQuarter = ccd.getOrientation().getNQuarter()
if nQuarter != 0:
doRotateCalib = True
if self.config.doDefect:
defects = sensorRef.get('defects', immediate=True)
self.maskAndInterpDefect(ccdExposure, defects)
if self.config.qa.doWriteOss:
sensorRef.put(ccdExposure, "ossImage")
if self.config.qa.doThumbnailOss:
self.writeThumbnail(sensorRef, "ossThumb", ccdExposure)
if self.config.doBias:
biasExposure = self.getIsrExposure(sensorRef, "bias")
if not doRotateCalib:
self.biasCorrection(ccdExposure, biasExposure)
else:
with self.rotated(ccdExposure) as exp:
self.biasCorrection(exp, biasExposure)
if self.config.doLinearize:
self.linearize(ccdExposure)
if self.config.doCrosstalk:
self.crosstalk.run(ccdExposure)
if self.config.doDark:
darkExposure = self.getIsrExposure(sensorRef, "dark")
if not doRotateCalib:
self.darkCorrection(ccdExposure, darkExposure)
else:
with self.rotated(ccdExposure) as exp:
self.darkCorrection(exp, darkExposure)
if self.config.doFlat:
flatExposure = self.getIsrExposure(sensorRef, "flat")
if not doRotateCalib:
self.flatCorrection(ccdExposure, flatExposure)
else:
with self.rotated(ccdExposure) as exp:
self.flatCorrection(exp, flatExposure)
if self.config.doApplyGains:
self.applyGains(ccdExposure, self.config.normalizeGains)
if self.config.doWidenSaturationTrails:
self.widenSaturationTrails(ccdExposure.getMaskedImage().getMask())
if self.config.doSaturation:
self.saturationInterpolation(ccdExposure)
if self.config.doFringe:
self.fringe.runDataRef(ccdExposure, sensorRef)
if self.config.doSetBadRegions:
self.setBadRegions(ccdExposure)
self.maskAndInterpNan(ccdExposure)
if self.config.qa.doWriteFlattened:
sensorRef.put(ccdExposure, "flattenedImage")
if self.config.qa.doThumbnailFlattened:
self.writeThumbnail(sensorRef, "flattenedThumb", ccdExposure)
self.measureBackground(ccdExposure)
if self.config.doGuider:
self.guider(ccdExposure)
self.roughZeroPoint(ccdExposure)
if self.config.doWriteVignettePolygon:
self.setValidPolygonIntersect(ccdExposure, self.vignettePolygon)
if self.config.doWrite:
sensorRef.put(ccdExposure, "postISRCCD")
if self._display:
im = ccdExposure.getMaskedImage().getImage()
im_median = float(numpy.median(im.getArray()))
ds9.mtv(im)
ds9.scale(min=im_median * 0.95, max=im_median * 1.15)
return Struct(exposure=ccdExposure)
@contextmanager
def rotated(self, exp):
nQuarter = exp.getDetector().getOrientation().getNQuarter()
exp.setMaskedImage(
afwMath.rotateImageBy90(exp.getMaskedImage(), 4 - nQuarter))
try:
yield exp
finally:
exp.setMaskedImage(
afwMath.rotateImageBy90(exp.getMaskedImage(), nQuarter))
def applyGains(self, ccdExposure, normalizeGains):
ccd = ccdExposure.getDetector()
ccdImage = ccdExposure.getMaskedImage()
medians = []
for a in ccd:
sim = ccdImage.Factory(ccdImage, a.getDataSec())
sim *= a.getElectronicParams().getGain()
if normalizeGains:
medians.append(numpy.median(sim.getImage().getArray()))
if normalizeGains:
median = numpy.median(numpy.array(medians))
for i, a in enumerate(ccd):
sim = ccdImage.Factory(ccdImage, a.getDataSec())
sim *= median / medians[i]
def widenSaturationTrails(self, mask):
"""Grow the saturation trails by an amount dependent on the width of the trail"""
extraGrowDict = {}
for i in range(1, 6):
extraGrowDict[i] = 0
for i in range(6, 8):
extraGrowDict[i] = 1
for i in range(8, 10):
extraGrowDict[i] = 3
extraGrowMax = 4
if extraGrowMax <= 0:
return
saturatedBit = mask.getPlaneBitMask('SAT')
xmin, ymin = mask.getBBox().getMin()
width = mask.getWidth()
thresh = afwDetection.Threshold(saturatedBit,
afwDetection.Threshold.BITMASK)
fpList = afwDetection.FootprintSet(mask, thresh).getFootprints()
for fp in fpList:
for s in fp.getSpans():
x0, x1 = s.getX0(), s.getX1()
extraGrow = extraGrowDict.get(x1 - x0 + 1, extraGrowMax)
if extraGrow > 0:
y = s.getY() - ymin
x0 -= xmin + extraGrow
x1 -= xmin - extraGrow
if x0 < 0: x0 = 0
if x1 >= width - 1: x1 = width - 1
for x in range(x0, x1 + 1):
mask.set(x, y, mask.get(x, y) | saturatedBit)
def setBadRegions(self, exposure):
"""Set all BAD areas of the chip to the average of the rest of the exposure
@param[in,out] exposure exposure to process; must include both DataSec and BiasSec pixels
"""
if self.config.badStatistic == "MEDIAN":
statistic = afwMath.MEDIAN
elif self.config.badStatistic == "MEANCLIP":
statistic = afwMath.MEANCLIP
else:
raise RuntimeError(
"Impossible method %s of bad region correction" %
self.config.badStatistic)
mi = exposure.getMaskedImage()
mask = mi.getMask()
BAD = mask.getPlaneBitMask("BAD")
INTRP = mask.getPlaneBitMask("INTRP")
sctrl = afwMath.StatisticsControl()
sctrl.setAndMask(BAD)
value = afwMath.makeStatistics(mi, statistic, sctrl).getValue()
maskArray = mask.getArray()
imageArray = mi.getImage().getArray()
badPixels = numpy.logical_and((maskArray & BAD) > 0,
(maskArray & INTRP) == 0)
imageArray[:] = numpy.where(badPixels, value, imageArray)
self.log.info("Set %d BAD pixels to %.2f" % (badPixels.sum(), value))
def writeThumbnail(self, dataRef, dataset, exposure):
"""Write out exposure to a snapshot file named outfile in the given size.
"""
filename = dataRef.get(dataset + "_filename")[0]
directory = os.path.dirname(filename)
if not os.path.exists(directory):
try:
os.makedirs(directory)
except OSError, e:
# Don't fail if directory exists due to race
if e.errno != errno.EEXIST:
raise e
binning = self.config.thumbnailBinning
binnedImage = afwMath.binImage(exposure.getMaskedImage(), binning,
binning, afwMath.MEAN)
statsCtrl = afwMath.StatisticsControl()
statsCtrl.setAndMask(binnedImage.getMask().getPlaneBitMask(
["SAT", "BAD", "INTRP"]))
stats = afwMath.makeStatistics(
binnedImage, afwMath.MEDIAN | afwMath.STDEVCLIP | afwMath.MAX,
statsCtrl)
low = stats.getValue(
afwMath.MEDIAN) - self.config.thumbnailStdev * stats.getValue(
afwMath.STDEVCLIP)
makeRGB(binnedImage,
binnedImage,
binnedImage,
min=low,
range=self.config.thumbnailRange,
Q=self.config.thumbnailQ,
fileName=filename,
saturatedBorderWidth=self.config.thumbnailSatBorder,
saturatedPixelValue=stats.getValue(afwMath.MAX))
else:
maxShow = max
# To see if data are available for all the cut-out region
if (bCut.getHeight() < dimExpect) or (bCut.getWidth() < dimExpect):
print WAR
print " ### Only part of the desired cutout-region is returned !"
# Define the name of the output file
outRgb = prefix + '_' + filt + '_part_color.png'
partial = True
else:
outRgb = prefix + '_' + filt + '_color.png'
partial = False
# Generate the RGB image
# 15/04/22: min ==> minimum
imgRgb = afwRgb.makeRGB(rCut, gCut, bCut, minimum=min,
range=(maxShow - min), Q=Q,
saturatedPixelValue=None)
if partial:
for k in range(3):
rgbEmpty[newY[k]:(newY[k] + images[k].getHeight()),
newX[k]:(newX[k] + images[k].getWidth()),
k] = imgRgb[:, :, k]
imgRgb = rgbEmpty
# Add a scale bar
if scaleBar is not None:
sLength = ((scaleBar * 1.0) / 0.168) / (dimExpect * 1.0)
sString = "%d\"" % int(scaleBar)
else:
sLength = None
sString = None
# Better way to show the image
