def __rectify_spatial(order):
"""
"""
for frame in order.frames:
order.objImg[frame] = image_lib.rectify_spatial(
order.objImg[frame], order.flatOrder.smoothedSpatialTrace)
order.ffObjImg[frame] = image_lib.rectify_spatial(
order.ffObjImg[frame], order.flatOrder.smoothedSpatialTrace)
order.spatialRectified = True
logger.info('order has been rectified in the spatial dimension')
return
def reduce(self):
self.logger.info('reducing flat order {}'.format(self.orderNum))
# normalize flat
self.normFlatImg, self.mean = image_lib.normalize(
self.cutout, self.onOrderMask, self.offOrderMask)
self.normalized = True
self.logger.info('flat normalized, flat mean = ' +
str(round(self.mean, 1)))
# spatially rectify flat
self.rectFlatImg = image_lib.rectify_spatial(self.normFlatImg,
self.smoothedSpatialTrace)
self.spatialRectified = True
# compute top and bottom trim points
self.calcTrimPoints()
# trim rectified flat order images
self.rectFlatImg = self.rectFlatImg[self.botTrim:self.topTrim, :]
self.logger.debug('reduction of flat order {} complete'.format(
self.orderNum))
return
def reduce(self):
self.logger.info('reducing flat order {}'.format(self.orderNum))
# normalize flat
self.normFlatImg, self.mean = image_lib.normalize(
self.cutout, self.onOrderMask, self.offOrderMask)
self.normalized = True
self.logger.info('flat normalized, flat mean = ' + str(round(self.mean, 1)))
# spatially rectify flat
self.rectFlatImg = image_lib.rectify_spatial(self.normFlatImg, self.smoothedSpatialTrace)
self.spatialRectified = True
# compute top and bottom trim points
self.calcTrimPoints()
# trim rectified flat order images
self.rectFlatImg = self.rectFlatImg[self.botTrim:self.topTrim, :]
self.logger.debug('reduction of flat order {} complete'.format(self.orderNum))
return
def reduce_order(order, flat_order):
# flatten obj but keep original for noise calc
order.flattenedObjImg = np.array(order.objCutout / flat_order.normFlatImg)
if np.amin(order.flattenedObjImg) < 0:
order.flattenedObjImg -= np.amin(order.flattenedObjImg)
order.flattened = True
order.objImg = np.array(order.objCutout) # should probably use objImg instead of objCutout to begin with
logger.info('order has been flat fielded')
# rectify flat, normalized flat, obj and flattened obj in spatial dimension
order.objImg = image_lib.rectify_spatial(order.objImg, flat_order.smoothedSpatialTrace)
order.flattenedObjImg = image_lib.rectify_spatial(
order.flattenedObjImg, flat_order.smoothedSpatialTrace)
# trim rectified order
order.objImg = order.objImg[flat_order.botTrim:flat_order.topTrim, :]
order.flattenedObjImg = order.flattenedObjImg[flat_order.botTrim:flat_order.topTrim, :]
order.srNormFlatImg = flat_order.rectFlatImg
order.srFlatObjImg = order.flattenedObjImg
order.spatialRectified = True
# find spatial profile and peak
find_spatial_profile_and_peak(order)
# characterize spatial profile by fitting to Gaussian
characterize_spatial_profile(order)
# find and smooth spectral trace
try:
order.spectralTrace = nirspec_lib.smooth_spectral_trace(
nirspec_lib.find_spectral_trace(
order.flattenedObjImg), order.flattenedObjImg.shape[0])
except Exception as e:
logger.warning('not rectifying order {} in spectral dimension'.format(order.orderNum))
else:
flat_order.rectFlatImg = image_lib.rectify_spectral(flat_order.rectFlatImg, order.spectralTrace)
order.objImg = image_lib.rectify_spectral(order.objImg, order.spectralTrace)
order.objImgFlattened = image_lib.rectify_spectral(order.flattenedObjImg, order.spectralTrace)
order.spectralRectified = True
# compute noise image
order.noiseImg = nirspec_lib.calc_noise_img(
order.objImg, flat_order.rectFlatImg, order.integrationTime)
# extract spectra
extract_spectra(order, flat_order)
# calculate SNR
bg = 0.0
if order.topBgMean is not None:
bg += order.topBgMean
if order.botBgMean is not None:
bg += order.botBgMean
if order.topBgMean is not None and order.botBgMean is not None:
bg /= 2
order.snr = np.absolute(np.mean(order.flattenedObjImg[order.peakLocation:order.peakLocation + 1, :]) / bg)
logger.info('signal-to-noise ratio = {:.1f}'.format(order.snr))
# find and identify sky lines
line_pairs = None # line_pairs are (column number, accepted wavelength
try:
oh_wavelengths, oh_intensities = wavelength_utils.get_oh_lines()
except IOError as e:
logger.critical('cannot read OH line file: ' + str(e))
raise
try:
# synthesize sky spectrum and store in order object
order.synthesizedSkySpec = wavelength_utils.synthesize_sky(
oh_wavelengths, oh_intensities, order.gratingEqWaveScale)
# identify lines and return list of (column number, accepted wavelength) tuples
line_pairs = wavelength_utils.line_id(order, oh_wavelengths, oh_intensities)
except (IOError, ValueError) as e:
logger.warning('sky line matching failed: ' + str(e))
if line_pairs is not None:
logger.info(str(len(line_pairs)) + ' matched sky lines found in order')
# add line pairs to Order object as Line objects
for line_pair in line_pairs:
col, waveAccepted = line_pair
peak = order.skySpec[col]
cent = image_lib.centroid(order.skySpec, 1024, 5, col)
line = Line.Line(order.frame, order.orderNum, waveAccepted, col, cent, peak)
order.lines.append(line)
if len(order.lines) >= 3:
# do local wavelength fit
measured = []
accepted = []
for line in order.lines:
measured.append(order.gratingEqWaveScale[line.col])
accepted.append(line.waveAccepted)
(order.orderCalSlope, order.orderCalIncpt, order.orderCalCorrCoeff, p, e) = \
scipy.stats.linregress(np.array(measured), np.array(accepted))
order.orderCalNLines = len(order.lines)
logger.info('per order wavelength fit: n = {}, a = {:.6f}, b = {:.6f}, r = {:.6f}'.format(
len(order.lines), order.orderCalIncpt, order.orderCalSlope,
order.orderCalCorrCoeff))
for line in order.lines:
line.orderWaveFit = order.orderCalIncpt + \
(order.orderCalSlope * order.gratingEqWaveScale[line.col])
line.orderFitRes = abs(line.orderWaveFit - line.waveAccepted)
line.orderFitSlope = (order.orderCalSlope *
(order.gratingEqWaveScale[1023] - order.gratingEqWaveScale[0]))/1024.0
else:
logger.warning('no matched sky lines in order ' + str(order.orderNum))
order.orderCal = False
return
def reduce_order(order):
# normalize flat
order.normalizedFlatImg, order.flatMean = image_lib.normalize(
order.flatCutout, order.onOrderMask, order.offOrderMask)
order.flatNormalized = True
logger.info('flat normalized, flat mean = ' + str(round(order.flatMean, 1)))
# flatten obj but keep original for noise calc
order.flattenedObjImg = np.array(order.objCutout / order.normalizedFlatImg)
if np.amin(order.flattenedObjImg) < 0:
order.flattenedObjImg -= np.amin(order.flattenedObjImg)
order.flattened = True
order.objImg = np.array(order.objCutout) # should probably use objImg instead of objCutout to begin with
order.flatImg = np.array(order.flatCutout)
logger.info('order has been flat fielded')
# smooth spatial trace
# this should probably be done where the trace is first found
order.smoothedTrace, order.traceMask = nirspec_lib.smooth_spatial_trace(order.avgTrace)
logger.info('spatial trace smoothed, ' + str(order.objImg.shape[1] - np.count_nonzero(order.traceMask)) +
' points ignored')
# rectify flat, normalized flat, obj and flattened obj in spatial dimension
order.flatImg = image_lib.rectify_spatial(order.flatImg, order.smoothedTrace)
order.normalizedFlatImg = image_lib.rectify_spatial(order.normalizedFlatImg, order.smoothedTrace)
order.objImg = image_lib.rectify_spatial(order.objImg, order.smoothedTrace)
order.flattenedObjImg = image_lib.rectify_spatial(order.flattenedObjImg, order.smoothedTrace)
if order.lowestPoint > order.padding:
top = order.highestPoint - order.lowestPoint + order.padding - 3
else:
top = order.highestPoint - 3
h = np.amin(order.topTrace - order.botTrace)
bot = top - h + 3
bot = max(0, bot)
top = min(top, 1023)
order.flatImg = order.flatImg[bot:top, :]
order.normalizedFlatImg = order.normalizedFlatImg[bot:top, :]
order.objImg = order.objImg[bot:top, :]
order.flattenedObjImg = order.flattenedObjImg[bot:top, :]
order.srNormFlatImg = order.normalizedFlatImg
order.srFlatObjImg = order.flattenedObjImg
order.spatialRectified = True
# find spatial profile and peak
order.spatialProfile = order.flattenedObjImg.mean(axis=1)
# import pylab as pl
#
# pl.figure()
# pl.cla()
# pl.imshow(order.flattenedObjImg)
# pl.show()
#
# pl.figure(str(order.orderNum))
# pl.cla()
# pl.plot(order.spatialProfile)
# pl.show()
order.peakLocation = np.argmax(order.spatialProfile[5:-5]) + 5
logger.info('spatial profile peak intensity row {:d}'.format(order.peakLocation))
p0 = order.peakLocation - (config.params['obj_window'] / 2)
p1 = order.peakLocation + (config.params['obj_window'] / 2)
order.centroid = (scipy.ndimage.measurements.center_of_mass(
order.spatialProfile[p0:p1]))[0] + p0
logger.info('spatial profile peak centroid row {:.1f}'.format(float(order.centroid)))
# characterize spatial profile by fitting to Gaussian
try:
for w in range(10, 30, 10):
logger.debug('gaussian window width = {}'.format(2 * w))
x0 = max(0, order.peakLocation - w)
x1 = min(len(order.spatialProfile) - 1, order.peakLocation + w)
x = range(x1 - x0)
order.gaussianParams, pcov = scipy.optimize.curve_fit(
image_lib.gaussian, x, order.spatialProfile[x0:x1] - np.amin(order.spatialProfile[x0:x1]))
order.gaussianParams[1] += x0
if order.gaussianParams[2] > 1.0:
break
except Exception as e:
logger.warning('cannot fit spatial profile to Gaussian')
order.gaussianParams = None
else:
logger.info('spatial peak width = {:.1f} pixels'.format(abs(order.gaussianParams[2])))
# find and smooth spectral trace
try:
order.spectral_trace = nirspec_lib.smooth_spectral_trace(
nirspec_lib.find_spectral_trace(
order.flattenedObjImg, order.padding), order.flattenedObjImg.shape[0])
except Exception as e:
logger.warning('not rectifying order in spectral dimension')
else:
# rectify flat, normalized flat, obj and flattened obj in spectral dimension
order.flatImg = image_lib.rectify_spectral(order.flatImg, order.spectral_trace)
order.normalizedFlatImg = image_lib.rectify_spectral(order.normalizedFlatImg, order.spectral_trace)
order.objImg = image_lib.rectify_spectral(order.objImg, order.spectral_trace)
order.objImgFlattened = image_lib.rectify_spectral(order.flattenedObjImg, order.spectral_trace)
order.spectralRectified = True
# compute noise image
order.noiseImg = nirspec_lib.calc_noise_img(
order.objImg, order.normalizedFlatImg, order.integrationTime)
# extract spectra
order.objWindow, order.topSkyWindow, order.botSkyWindow = \
image_lib.get_extraction_ranges(order.objImg.shape[0], order.peakLocation,
config.params['obj_window'], config.params['sky_window'], config.params['sky_separation'])
logger.info('extraction window width = {}'.format(str(len(order.objWindow))))
logger.info('top background window width = {}'.format(str(len(order.topSkyWindow))))
if len(order.topSkyWindow) > 0:
logger.info('top background window separation = {}'.format(
str(order.topSkyWindow[0] - order.objWindow[-1])))
logger.info('bottom background window width = {}'.format(str(len(order.botSkyWindow))))
if len(order.botSkyWindow) > 0:
logger.info('bottom background window separation = {}'.format(
str(order.objWindow[0] - order.botSkyWindow[-1])))
order.objSpec, order.flatSpec, order.skySpec, order.noiseSpec, order.topBgMean, \
order.botBgMean = image_lib.extract_spectra(
order.flattenedObjImg, order.normalizedFlatImg, order.noiseImg,
order.objWindow, order.topSkyWindow, order.botSkyWindow)
# calculate SNR
bg = 0.0
if order.topBgMean is not None:
bg += order.topBgMean
if order.botBgMean is not None:
bg += order.botBgMean
if order.topBgMean is not None and order.botBgMean is not None:
bg /= 2
order.snr = np.absolute(np.mean(order.flattenedObjImg[order.peakLocation:order.peakLocation + 1, :]) / bg)
logger.info('signal-to-noise ratio = {:.1f}'.format(order.snr))
# find and identify sky lines
line_pairs = None # line_pairs are (column number, accepted wavelength
try:
oh_wavelengths, oh_intensities = wavelength_utils.get_oh_lines(config.params['oh_filename'])
except IOError as e:
logger.critical('cannot read OH line file: ' + str(e))
raise
try:
# synthesize sky spectrum and store in order object
order.synthesizedSkySpec = wavelength_utils.synthesize_sky(
oh_wavelengths, oh_intensities, order.wavelengthScaleCalc)
# identify lines and return list of (column number, accepted wavelength) tuples
line_pairs = wavelength_utils.line_id(order, oh_wavelengths, oh_intensities)
except (IOError, ValueError) as e:
logger.warning('sky line matching failed: ' + str(e))
if line_pairs is not None:
logger.info(str(len(line_pairs)) + ' matched sky lines found in order')
# add line pairs to Order object as Line objects
for line_pair in line_pairs:
line = Line.Line()
line.col, line.acceptedWavelength = line_pair
line.peak = order.skySpec[line.col]
# find centroid of peak
w = 5
p0 = max(0, line.col - (w / 2))
p1 = min(1023, line.col + (w / 2)) + 1
line.centroid = p0 + scipy.ndimage.center_of_mass(order.skySpec[p0:p1])[0]
if abs(line.centroid - line.col) > 1:
logger.warning('sky line centroid error, col = {}, centroid = {:.3f}'.format(
line.col, line.centroid))
line.centroid = line.col
order.lines.append(line)
if len(order.lines) >= 3:
# do local wavelength fit
measured = []
accepted = []
for line in order.lines:
measured.append(order.wavelengthScaleCalc[line.col])
accepted.append(line.acceptedWavelength)
(order.perOrderSlope, order.perOrderIntercept, order.perOrderCorrCoeff, p, e) = \
scipy.stats.linregress(np.array(measured), np.array(accepted))
logger.info('per order wavelength fit: n = {}, a = {:.6f}, b = {:.6f}, r = {:.6f}'.format(
len(order.lines), order.perOrderIntercept, order.perOrderSlope,
order.perOrderCorrCoeff))
for line in order.lines:
line.localFitWavelength = order.perOrderIntercept + \
(order.perOrderSlope * order.wavelengthScaleCalc[line.col])
line.localFitResidual = abs(line.localFitWavelength - line.acceptedWavelength)
line.localFitSlope = (order.perOrderSlope * (order.wavelengthScaleCalc[1023] - order.wavelengthScaleCalc[0]))/1024.0
else:
logger.warning('no matched sky lines in order ' + str(order.orderNum))
return
def reduce(self):
self.logger.info('reducing flat order {}'.format(self.orderNum))
# normalize flat
self.normFlatImg, self.median = image_lib.normalize(
self.cutout, self.onOrderMask, self.offOrderMask)
self.normalized = True
self.logger.info('flat normalized, flat median = ' +
str(round(self.median, 1)))
# spatially rectify flat
self.rectFlatImg = image_lib.rectify_spatial(self.normFlatImg,
self.smoothedSpatialTrace)
"""
self.rectFlatImgA = image_lib.rectify_spatial(self.normFlatImg, self.smoothedSpatialTraceA)
self.rectFlatImgB = image_lib.rectify_spatial(self.normFlatImg, self.smoothedSpatialTraceB)
"""
self.spatialRectified = True
# compute top and bottom trim points
self.calcTrimPoints()
### TESTING PLOT XXX
'''
from skimage import exposure
#norm = ImageNormalize(self.rectFlatImg, interval=ZScaleInterval())
fig = plt.figure(1985)
#plt.imshow(self.rectFlatImg, origin='lower', aspect='auto', norm=norm)
plt.imshow(self.cutout, origin='lower', aspect='auto')#, norm=norm)
plt.axhline(self.botTrim, c='r', ls=':')
plt.axhline(self.topTrim, c='b', ls=':')
#plt.axhline(self.lowestPoint, c='r', ls='--')
#plt.axhline(self.highestPoint, c='b', ls='--')
fig.suptitle('Order: %s'%self.orderNum)
fig = plt.figure(1986)
#plt.imshow(self.rectFlatImg, origin='lower', aspect='auto', norm=norm)
plt.imshow(self.normFlatImg, origin='lower', aspect='auto')#, norm=norm)
plt.axhline(self.botTrim, c='r', ls=':')
plt.axhline(self.topTrim, c='b', ls=':')
#plt.axhline(self.lowestPoint, c='r', ls='--')
#plt.axhline(self.highestPoint, c='b', ls='--')
fig.suptitle('Order: %s'%self.orderNum)
fig = plt.figure(1987)
#plt.imshow(self.rectFlatImg, origin='lower', aspect='auto', norm=norm)
plt.imshow(exposure.equalize_hist(self.rectFlatImg), origin='lower', aspect='auto')#, norm=norm)
plt.axhline(self.botTrim, c='r', ls=':')
plt.axhline(self.topTrim, c='b', ls=':')
#plt.axhline(self.botTrim+10, c='r', ls='--')
#plt.axhline(self.topTrim-10, c='b', ls='--')
#plt.axhline(self.lowestPoint, c='r', ls='--')
#plt.axhline(self.highestPoint, c='b', ls='--')
fig.suptitle('Order: %s'%self.orderNum)
print(self.cutoutPadding)
print(self.highestPoint, self.lowestPoint, self.topEdgeTrace - self.botEdgeTrace)
plt.show()
sys.exit()
'''
### TESTING PLOT XXX
# trim rectified flat order images
self.rectFlatImg = self.rectFlatImg[self.botTrim:self.topTrim, :]
#self.rectFlatImgA = self.rectFlatImgA[self.botTrimA:self.topTrimA, :]
#self.rectFlatImgB = self.rectFlatImgB[self.botTrimB:self.topTrimB, :]
self.logger.debug('reduction of flat order {} complete'.format(
self.orderNum))
return
def __rectify_spatial(order, eta=None, arc=None):
"""
"""
for frame in order.frames:
if frame == 'AB':
continue # Skip the AB frame, we will subtract them after rectification
logger.info(
'attempting spatial rectification of frame {} using object trace'.
format(frame))
try:
if frame in ['A', 'B']:
#print('FRAME', frame)
if config.params['onoff'] == True and frame == 'B':
order.objImg[frame] = image_lib.rectify_spatial(
order.objImg[frame], polyVals1)
order.ffObjImg[frame] = image_lib.rectify_spatial(
order.ffObjImg[frame], polyVals2)
logger.info(
'frame {}, order {} rectified using object trace'.
format(frame, order.flatOrder.orderNum))
elif config.params['spatial_rect_flat'] == True:
logger.info(
'order will be rectified in the spatial dimension using the median order trace'
)
order.objImg[frame] = image_lib.rectify_spatial(
order.objImg[frame],
order.flatOrder.smoothedSpatialTrace)
order.ffObjImg[frame] = image_lib.rectify_spatial(
order.ffObjImg[frame],
order.flatOrder.smoothedSpatialTrace)
logger.info(
'frame {}, order {} rectified in the spatial dimension using flat frame'
.format(frame, order.flatOrder.orderNum))
else:
#polyVals1 = cat.CreateSpatialMap(order.objImg[frame])
polyVals1 = cat.CreateSpatialMap(order.objCutout[frame])
order.objImg[frame] = image_lib.rectify_spatial(
order.objImg[frame], polyVals1)
logger.info(
'frame {}, order {} rectified using object trace'.
format(frame, order.flatOrder.orderNum))
#polyVals2 = cat.CreateSpatialMap(order.ffObjImg[frame])
polyVals2 = cat.CreateSpatialMap(order.ffObjCutout[frame])
order.ffObjImg[frame] = image_lib.rectify_spatial(
order.ffObjImg[frame], polyVals2)
logger.info(
'flat fielded frame {}, order {} rectified using object trace'
.format(frame, order.flatOrder.orderNum))
if eta is not None:
if frame == 'B':
if config.params['onoff'] == True:
order.etaImgB = order.etaImg
order.ffEtaImgB = order.ffEtaImg
else:
order.etaImgB = image_lib.rectify_spatial(
order.etaImgB, polyVals1)
order.ffEtaImgB = image_lib.rectify_spatial(
order.ffEtaImgB, polyVals2)
logger.info(
'etalon frame {}, order {} rectified using object trace'
.format(frame, order.flatOrder.orderNum))
else:
order.etaImg = image_lib.rectify_spatial(
order.etaImg, polyVals1)
order.ffEtaImg = image_lib.rectify_spatial(
order.ffEtaImg, polyVals2)
logger.info(
'etalon frame {}, order {} rectified using object trace'
.format(frame, order.flatOrder.orderNum))
if arc is not None:
if frame == 'B':
if config.params['onoff'] == True:
order.arcImgB = order.arcImg
order.ffArcImgB = order.ffArcImg
else:
order.arcImgB = image_lib.rectify_spatial(
order.arcImgB, polyVals1)
order.ffArcImgB = image_lib.rectify_spatial(
order.ffArcImgB, polyVals2)
logger.info(
'arc lamp frame {}, order {} rectified using object trace'
.format(frame, order.flatOrder.orderNum))
else:
order.arcImg = image_lib.rectify_spatial(
order.arcImg, polyVals1)
order.ffArcImg = image_lib.rectify_spatial(
order.ffArcImg, polyVals2)
logger.info(
'arc lamp frame {}, order {} rectified using object trace'
.format(frame, order.flatOrder.orderNum))
else:
order.objImg[frame] = image_lib.rectify_spatial(
order.objImg[frame], polyVals1)
order.ffObjImg[frame] = image_lib.rectify_spatial(
order.ffObjImg[frame], polyVals2)
except:
logger.warning(
'could not rectify using object trace, falling back to edge trace'
)
order.objImg[frame] = image_lib.rectify_spatial(
order.objImg[frame], order.flatOrder.smoothedSpatialTrace)
order.ffObjImg[frame] = image_lib.rectify_spatial(
order.ffObjImg[frame], order.flatOrder.smoothedSpatialTrace)
if eta is not None:
if frame == 'B':
order.etaImgB = image_lib.rectify_spatial(
order.etaImgB, order.flatOrder.smoothedSpatialTrace)
order.ffEtaImgB = image_lib.rectify_spatial(
order.ffEtaImgB, order.flatOrder.smoothedSpatialTrace)
else:
order.etaImg = image_lib.rectify_spatial(
order.etaImg, order.flatOrder.smoothedSpatialTrace)
order.ffEtaImg = image_lib.rectify_spatial(
order.ffEtaImg, order.flatOrder.smoothedSpatialTrace)
if arc is not None:
if frame == 'B':
order.arcImgB = image_lib.rectify_spatial(
order.arcImgB, order.flatOrder.smoothedSpatialTrace)
order.ffArcImgB = image_lib.rectify_spatial(
order.ffArcImgB, order.flatOrder.smoothedSpatialTrace)
else:
order.arcImg = image_lib.rectify_spatial(
order.arcImg, order.flatOrder.smoothedSpatialTrace)
order.ffArcImg = image_lib.rectify_spatial(
order.ffArcImg, order.flatOrder.smoothedSpatialTrace)
order.spatialRectified = True
logger.success('order has been rectified in the spatial dimension')
return
