def _perform(self):
"""
Returns an Argument() with the parameters that depends on this operation
"""
self.logger.info("Creating master illumination correction")
suffix = self.action.args.new_type.lower()
insuff = self.action.args.stack_type.lower()
stack_list = list(self.stack_list['filename'])
if len(stack_list) <= 0:
self.logger.warning("No flats found!")
return self.action.args
# get root for maps
tab = self.context.proctab.search_proctab(
frame=self.action.args.ccddata, target_type='ARCLAMP',
target_group=self.action.args.groupid)
if len(tab) <= 0:
self.logger.error("Geometry not solved!")
return self.action.args
mroot = strip_fname(tab['filename'][-1])
# Wavelength map image
wmf = mroot + '_wavemap.fits'
self.logger.info("Reading image: %s" % wmf)
wavemap = kcwi_fits_reader(
os.path.join(self.config.instrument.cwd, 'redux',
wmf))[0]
# Slice map image
slf = mroot + '_slicemap.fits'
self.logger.info("Reading image: %s" % slf)
slicemap = kcwi_fits_reader(os.path.join(
self.config.instrument.cwd, 'redux',
slf))[0]
# Position map image
pof = mroot + '_posmap.fits'
self.logger.info("Reading image: %s" % pof)
posmap = kcwi_fits_reader(os.path.join(
self.config.instrument.cwd, 'redux',
pof))[0]
# Read in stacked flat image
stname = strip_fname(stack_list[0]) + '_' + insuff + '.fits'
self.logger.info("Reading image: %s" % stname)
stacked = kcwi_fits_reader(os.path.join(
self.config.instrument.cwd, 'redux',
stname))[0]
# get type of flat
internal = ('SFLAT' in stacked.header['IMTYPE'])
twiflat = ('STWIF' in stacked.header['IMTYPE'])
domeflat = ('SDOME' in stacked.header['IMTYPE'])
if internal:
self.logger.info("Internal Flat")
elif twiflat:
self.logger.info("Twilight Flat")
elif domeflat:
self.logger.info("Dome Flat")
else:
self.logger.error("Flat of Unknown Type!")
return self.action.args
# knots per pixel
knotspp = self.config.instrument.KNOTSPP
# get image size
ny = stacked.header['NAXIS2']
# get binning
xbin = self.action.args.xbinsize
# Parameters for fitting
# vignetted slice position range
fitl = int(4/xbin)
fitr = int(24/xbin)
# un-vignetted slice position range
flatl = int(34/xbin)
flatr = int(72/xbin)
# flat fitting slice position range
ffleft = int(10/xbin)
ffright = int(70/xbin)
nrefx = int(ffright - ffleft)
buffer = 6.0/float(xbin)
# reference slice
refslice = 9
allidx = np.arange(int(140/xbin))
newflat = stacked.data.copy()
# dichroic fraction
try:
dichroic_fraction = wavemap.header['DICHFRAC']
except KeyError:
dichroic_fraction = 1.
# get reference slice data
q = [i for i, v in enumerate(slicemap.data.flat) if v == refslice]
# get wavelength limits
waves = wavemap.data.compress((wavemap.data > 0.).flat)
waves = [waves.min(), waves.max()]
self.logger.info("Wavelength limits: %.1f - %1.f" % (waves[0],
waves[1]))
# correct vignetting if we are using internal flats
if internal:
self.logger.info("Internal flats require vignetting correction")
# get good region for fitting
if self.action.args.camera == 0: # Blue
wmin = waves[0]
wmax = min([waves[1], 5620.])
elif self.action.args.camera == 1: # Red
wmin = max([waves[0], 5580.])
wmax = waves[1]
else:
self.logger.warning("Camera keyword not defined")
wmin = waves[0]
wmax = waves[1]
dw = (wmax - wmin) / 30.0
wavemin = (wmin+wmax) / 2.0 - dw
wavemax = (wmin+wmax) / 2.0 + dw
self.logger.info("Using %.1f - %.1f A of slice %d" % (wavemin,
wavemax,
refslice))
xflat = []
yflat = []
wflat = []
qq = []
for i in q:
if wavemin < wavemap.data.flat[i] < wavemax:
xflat.append(posmap.data.flat[i])
yflat.append(stacked.data.flat[i])
wflat.append(wavemap.data.flat[i])
qq.append(i)
# get un-vignetted portion
qflat = [i for i, v in enumerate(xflat) if flatl <= v <= flatr]
xflat = [xflat[i] for i in qflat]
yflat = [yflat[i] for i in qflat]
wflat = [wflat[i] for i in qflat]
# sort on wavelength
sw = np.argsort(wflat)
ywflat = [yflat[i] for i in sw]
wwflat = [wflat[i] for i in sw]
ww0 = np.min(wwflat)
# fit wavelength slope
wavelinfit = np.polyfit(wwflat-ww0, ywflat, 2)
wslfit = np.polyval(wavelinfit, wflat-ww0)
# plot slope fit
if self.config.instrument.plot_level >= 1:
p = figure(title=self.action.args.plotlabel + ' WAVE SLOPE FIT',
x_axis_label='wave px',
y_axis_label='counts',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(wwflat, ywflat, legend_label="Data")
p.line(wflat, wslfit, line_color='red', line_width=3,
legend_label="Fit")
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# take out slope
yflat = yflat / wslfit
# now sort on slice position
ss = np.argsort(xflat)
xflat = [xflat[i] for i in ss]
yflat = [yflat[i] for i in ss]
# fit un-vignetted slope
resflat = np.polyfit(xflat, yflat, 1)
# select the points we will fit for the vignetting
# get reference region
xfit = [posmap.data.flat[i] for i in qq]
yfit = [stacked.data.flat[i] for i in qq]
wflat = [wavemap.data.flat[i] for i in qq]
# take out wavelength slope
yfit = yfit / np.polyval(wavelinfit, wflat-ww0)
# select the vignetted region
qfit = [i for i, v in enumerate(xfit) if fitl <= v <= fitr]
xfit = [xfit[i] for i in qfit]
yfit = [yfit[i] for i in qfit]
# sort on slice position
s = np.argsort(xfit)
xfit = [xfit[i] for i in s]
yfit = [yfit[i] for i in s]
# fit vignetted slope
resfit = np.polyfit(xfit, yfit, 1)
# corrected data
ycdata = stacked.data.flat[qq] / \
np.polyval(wavelinfit, wavemap.data.flat[qq]-ww0)
ycmin = 0.5 # np.min(ycdata)
ycmax = 1.25 # np.max(ycdata)
# compute the intersection
xinter = -(resflat[1] - resfit[1]) / (resflat[0] - resfit[0])
# plot slice profile and fits
if self.config.instrument.plot_level >= 1:
p = figure(title=self.action.args.plotlabel + ' Vignetting',
x_axis_label='Slice Pos (px)',
y_axis_label='Ratio',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(posmap.data.flat[qq], ycdata, legend_label='Data')
p.line(allidx, resfit[1] + resfit[0]*allidx,
line_color='purple', legend_label='Vign.')
p.line(allidx, resflat[1] + resflat[0]*allidx, line_color='red',
legend_label='UnVign.')
p.line([fitl, fitl], [ycmin, ycmax], line_color='blue')
p.line([fitr, fitr], [ycmin, ycmax], line_color='blue')
p.line([flatl, flatl], [ycmin, ycmax], line_color='green')
p.line([flatr, flatr], [ycmin, ycmax], line_color='green')
p.line([xinter-buffer, xinter-buffer], [ycmin, ycmax],
line_color='black')
p.line([xinter + buffer, xinter + buffer], [ycmin, ycmax],
line_color='black')
p.line([xinter, xinter], [ycmin, ycmax], line_color='red')
p.y_range = Range1d(ycmin, ycmax)
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# figure out where the correction applies
qcor = [i for i, v in enumerate(posmap.data.flat)
if 0 <= v <= (xinter-buffer)]
# apply the correction!
self.logger.info("Applying vignetting correction...")
for i in qcor:
newflat.flat[i] = (resflat[1]+resflat[0]*posmap.data.flat[i]) \
/ (resfit[1]+resfit[0]*posmap.data.flat[i]) * \
stacked.data.flat[i]
# now deal with the intermediate (buffer) region
self.logger.info("Done, now handling buffer region")
# get buffer points to fit in reference region
qbff = [i for i in qq if (xinter-buffer) <=
posmap.data.flat[i] <= (xinter+buffer)]
# get slice pos and data for buffer fitting
xbuff = [posmap.data.flat[i] for i in qbff]
ybuff = [stacked.data.flat[i] / np.polyval(wavelinfit,
wavemap.data.flat[i]-ww0)
for i in qbff]
# sort on slice position
ssp = np.argsort(xbuff)
xbuff = [xbuff[i] for i in ssp]
ybuff = [ybuff[i] for i in ssp]
# fit buffer with low-order poly
buffit = np.polyfit(xbuff, ybuff, 3)
# plot buffer fit
if self.config.instrument.plot_level >= 1:
p = figure(title=self.action.args.plotlabel + ' Buffer Region',
x_axis_label='Slice Pos (px)',
y_axis_label='Ratio',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(xbuff, ybuff)
p.line(xbuff, np.polyval(buffit, xbuff), line_color='red')
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# get all buffer points in image
qbuf = [i for i, v in enumerate(posmap.data.flat)
if (xinter-buffer) <= v <= (xinter+buffer)]
# apply buffer correction to all buffer points in newflat
for i in qbuf:
newflat.flat[i] = \
(resflat[1] + resflat[0] * posmap.data.flat[i]) / \
np.polyval(buffit, posmap.data.flat[i]) * newflat.flat[i]
self.logger.info("Vignetting correction complete.")
self.logger.info("Fitting master illumination")
# now fit master flat
# get reference slice points
qref = [i for i in q if ffleft <= posmap.data.flat[i] <= ffright]
xfr = wavemap.data.flat[qref]
yfr = newflat.flat[qref]
# sort on wavelength
s = np.argsort(xfr)
xfr = xfr[s]
yfr = yfr[s]
wavegood0 = wavemap.header['WAVGOOD0']
wavegood1 = wavemap.header['WAVGOOD1']
# correction for BM where we see a ledge
if 'BM' in self.action.args.grating:
ledge_wave = bm_ledge_position(self.action.args.cwave)
self.logger.info("BM ledge calculated wavelength "
"for ref slice = %.2f (A)" % ledge_wave)
if wavegood0 <= ledge_wave <= wavegood1:
self.logger.info("BM grating requires correction")
qledge = [i for i, v in enumerate(xfr)
if ledge_wave-25 <= v <= ledge_wave+25]
xledge = [xfr[i] for i in qledge]
yledge = [yfr[i] for i in qledge]
s = np.argsort(xledge)
xledge = [xledge[i] for i in s]
yledge = [yledge[i] for i in s]
win = boxcar(250)
smyledge = sp.signal.convolve(yledge,
win, mode='same') / sum(win)
ylmax = np.max(yledge)
ylmin = np.min(yledge)
fpoints = np.arange(0, 100) / 100. * 50 + (ledge_wave-25)
ledgefit, ledgemsk = Bspline.iterfit(np.asarray(xledge),
smyledge, fullbkpt=fpoints,
upper=1, lower=1)
ylfit, _ = ledgefit.value(np.asarray(fpoints))
deriv = -(np.roll(ylfit, 1) - np.roll(ylfit, -1)) / 2.0
# trim edges
trm = int(len(deriv)/5)
deriv = deriv[trm:-trm]
xvals = fpoints[trm:-trm]
peaks, _ = find_peaks(deriv, height=100)
if len(peaks) != 1:
self.logger.warning("Extra peak found!")
p = figure(title=self.action.args.plotlabel +
' Ledge', x_axis_label='Wavelength (A)',
y_axis_label='Value',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(xledge, smyledge, fill_color='green')
p.line(fpoints, ylfit)
bokeh_plot(p, self.context.bokeh_session)
input("Next? <cr>: ")
p = figure(title=self.action.args.plotlabel +
' Deriv', x_axis_label='px',
y_axis_label='Value',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
xx = list(range(len(deriv)))
ylim = get_plot_lims(deriv)
p.circle(xx, deriv)
for pk in peaks:
p.line([pk, pk], ylim)
bokeh_plot(p, self.context.bokeh_session)
print("Please indicate the integer pixel value of the peak")
ipk = int(input("Peak? <int>: "))
else:
ipk = peaks[0]
apk = xvals[ipk]
if self.config.instrument.plot_level >= 3:
p = figure(
title=self.action.args.plotlabel + ' Peak of ledge',
x_axis_label='Wave (A)',
y_axis_label='Value',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(xvals, deriv, legend_label='Data')
p.line([apk, apk], [-50, 200], line_color='red',
legend_label='Pk')
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
xlow = apk - 3 - 5
xhi = apk - 3
zlow = apk + 3
zhi = apk + 3 + 5
qlow = [i for i, v in enumerate(fpoints) if xlow <= v <= xhi]
xlf = np.asarray([fpoints[i] for i in qlow])
ylf = np.asarray([ylfit[i] for i in qlow])
lowfit = np.polyfit(xlf, ylf, 1)
qhi = [i for i, v in enumerate(fpoints) if zlow <= v <= zhi]
xlf = np.asarray([fpoints[i] for i in qhi])
ylf = np.asarray([ylfit[i] for i in qhi])
hifit = np.polyfit(xlf, ylf, 1)
ratio = (hifit[1] + hifit[0] * apk) / \
(lowfit[1] + lowfit[0] * apk)
self.logger.info("BM ledge ratio: %.3f" % ratio)
# correct flat data
qcorr = [i for i, v in enumerate(xfr) if v >= apk]
for i in qcorr:
yfr[i] /= ratio
# plot BM ledge
if self.config.instrument.plot_level >= 1:
p = figure(
title=self.action.args.plotlabel + ' BM Ledge Region',
x_axis_label='Wave (A)',
y_axis_label='Value',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
# Input data
p.circle(xledge, yledge, fill_color='blue',
legend_label='Data')
# correct input data
qcorrect = [i for i, v in enumerate(xledge) if v >= apk]
xplt = []
yplt = []
for i in qcorrect:
xplt.append(xledge[i])
yplt.append(yledge[i] / ratio)
p.circle(xplt, yplt, fill_color='orange',
legend_label='Corrected')
p.line(fpoints, ylfit, line_color='red', legend_label='Fit')
p.line([xlow, xlow], [ylmin, ylmax], line_color='blue')
p.line([xhi, xhi], [ylmin, ylmax], line_color='blue')
p.line([zlow, zlow], [ylmin, ylmax], line_color='black')
p.line([zhi, zhi], [ylmin, ylmax], line_color='black')
p.line(fpoints, lowfit[1] + lowfit[0] * fpoints,
line_color='purple')
p.line(fpoints, hifit[1] + hifit[0] * fpoints,
line_color='green')
p.line([apk, apk], [ylmin, ylmax], line_color='green',
legend_label='Pk')
p.y_range = Range1d(ylmin, ylmax)
p.legend.location = 'top_left'
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# END: handling BM grating ledge
# if we are fitting a twilight flat, treat it like a sky image with a
# larger number of knots
if twiflat:
knots = int(ny * knotspp)
else:
knots = 100
self.logger.info("Using %d knots for bspline fit" % knots)
# generate a fit from ref slice points
bkpt = np.min(xfr) + np.arange(knots+1) * \
(np.max(xfr) - np.min(xfr)) / knots
sftr, _ = Bspline.iterfit(xfr[nrefx:-nrefx], yfr[nrefx:-nrefx],
fullbkpt=bkpt)
yfitr, _ = sftr.value(xfr)
# generate a blue slice spectrum bspline fit
blueslice = 12
blueleft = 60 / xbin
blueright = 80 / xbin
qb = [i for i, v in enumerate(slicemap.data.flat) if v == blueslice]
qblue = [i for i in qb if blueleft <= posmap.data.flat[i] <= blueright]
xfb = wavemap.data.flat[qblue]
yfb = newflat.flat[qblue]
s = np.argsort(xfb)
xfb = xfb[s]
yfb = yfb[s]
bkpt = np.min(xfb) + np.arange(knots+1) * \
(np.max(xfb) - np.min(xfb)) / knots
sftb, _ = Bspline.iterfit(xfb[nrefx:-nrefx], yfb[nrefx:-nrefx],
fullbkpt=bkpt)
yfitb, _ = sftb.value(xfb)
# generate a red slice spectrum bspline fit
redslice = 23
redleft = 60 / xbin
redright = 80 / xbin
qr = [i for i, v in enumerate(slicemap.data.flat) if v == redslice]
qred = [i for i in qr if redleft <= posmap.data.flat[i] <= redright]
xfd = wavemap.data.flat[qred]
yfd = newflat.flat[qred]
s = np.argsort(xfd)
xfd = xfd[s]
yfd = yfd[s]
bkpt = np.min(xfd) + np.arange(knots + 1) * \
(np.max(xfd) - np.min(xfd)) / knots
sftd, _ = Bspline.iterfit(xfd[nrefx:-nrefx], yfd[nrefx:-nrefx],
fullbkpt=bkpt)
yfitd, _ = sftd.value(xfd)
# waves
minwave = np.min(xfb)
maxwave = np.max(xfd)
# are we a twilight flat?
if twiflat:
nwaves = int(ny * knotspp)
else:
nwaves = 1000
waves = minwave + (maxwave - minwave) * np.arange(nwaves+1) / nwaves
if self.config.instrument.plot_level >= 1:
# output filename stub
rbfnam = "redblue_%05d_%s_%s_%s" % \
(self.action.args.ccddata.header['FRAMENO'],
self.action.args.illum, self.action.args.grating,
self.action.args.ifuname)
if xbin == 1:
stride = int(len(xfr) / 8000.)
if stride <= 0:
stride = 1
else:
stride = 1
xrplt = xfr[::stride]
yrplt = yfitr[::stride]
yrplt_d = yfr[::stride]
xbplt = xfb[::stride]
ybplt = yfitb[::stride]
ybplt_d = yfb[::stride]
xdplt = xfd[::stride]
ydplt = yfitd[::stride]
ydplt_d = yfd[::stride]
p = figure(
title=self.action.args.plotlabel + ' Blue/Red fits',
x_axis_label='Wave (A)',
y_axis_label='Flux (e-)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.line(xrplt, yrplt, line_color='black', legend_label='Ref')
p.circle(xrplt, yrplt_d, size=1, line_alpha=0., fill_color='black')
p.line(xbplt, ybplt, line_color='blue', legend_label='Blue')
p.circle(xbplt, ybplt_d, size=1, line_alpha=0., fill_color='blue')
p.line(xdplt, ydplt, line_color='red', legend_label='Red')
p.circle(xdplt, ydplt_d, size=1, line_alpha=0., fill_color='red')
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
save_plot(p, filename=rbfnam+'.png')
wavebuffer = 0.1
minrwave = np.min(xfr)
maxrwave = np.max(xfr)
wavebuffer2 = 0.05
wlb0 = minrwave+(maxrwave-minrwave)*wavebuffer2
wlb1 = minrwave+(maxrwave-minrwave)*wavebuffer
wlr0 = minrwave+(maxrwave-minrwave)*(1.-wavebuffer)
wlr1 = minrwave+(maxrwave-minrwave)*(1.-wavebuffer2)
qbluefit = [i for i, v in enumerate(waves) if wlb0 < v < wlb1]
qredfit = [i for i, v in enumerate(waves) if wlr0 < v < wlr1]
nqb = len(qbluefit)
nqr = len(qredfit)
self.logger.info("Wavelength regions: blue = %.1f - %.1f, "
"red = %.1f - %.1f" % (wlb0, wlb1, wlr0, wlr1))
self.logger.info("Fit points: blue = %d, red = %d" % (nqb, nqr))
if nqb > 0:
bluefit, _ = sftb.value(waves[qbluefit])
refbluefit, _ = sftr.value(waves[qbluefit])
blue_zero_cross = np.nanmin(bluefit) <= 0. or np.nanmin(
refbluefit) <= 0.
bluelinfit = np.polyfit(waves[qbluefit], refbluefit/bluefit, 1)
bluelinfity = bluelinfit[1] + bluelinfit[0] * waves[qbluefit]
else:
bluefit = None
blue_zero_cross = False
refbluefit = None
bluelinfit = None
bluelinfity = None
if nqr > 0:
redfit, _ = sftd.value(waves[qredfit])
refredfit, _ = sftr.value(waves[qredfit])
red_zero_cross = np.nanmin(redfit) <= 0. or np.nanmin(
refredfit) <= 0.
redlinfit = np.polyfit(waves[qredfit], refredfit/redfit, 1)
redlinfity = redlinfit[1] + redlinfit[0] * waves[qredfit]
else:
redfit = None
red_zero_cross = False
refredfit = None
redlinfit = None
redlinfity = None
if blue_zero_cross:
self.logger.info("Blue extension zero crossing detected")
if red_zero_cross:
self.logger.info("Red extension zero crossing detected")
if self.config.instrument.plot_level >= 1:
if nqb > 1:
# plot blue fits
p = figure(
title=self.action.args.plotlabel + ' Blue fits',
x_axis_label='Wave (A)',
y_axis_label='Flux (e-)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.line(waves[qbluefit], refbluefit, line_color='black',
legend_label='Ref')
p.circle(waves[qbluefit], refbluefit, fill_color='black')
p.line(waves[qbluefit], bluefit, line_color='blue',
legend_label='Blue')
p.circle(waves[qbluefit], bluefit, fill_color='blue')
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# plot blue ratios
p = figure(
title=self.action.args.plotlabel + ' Blue ratios',
x_axis_label='Wave (A)',
y_axis_label='Ratio',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.line(waves[qbluefit], refbluefit/bluefit, line_color='black',
legend_label='Ref')
p.circle(waves[qbluefit], refbluefit/bluefit,
fill_color='black')
p.line(waves[qbluefit], bluelinfity,
line_color='blue', legend_label='Blue')
p.circle(waves[qbluefit], bluelinfity,
fill_color='blue')
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
if nqr > 1:
# plot red fits
p = figure(
title=self.action.args.plotlabel + ' Red fits',
x_axis_label='Wave (A)',
y_axis_label='Flux (e-)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.line(waves[qredfit], refredfit, line_color='black',
legend_label='Ref')
p.circle(waves[qredfit], refredfit, fill_color='black')
p.line(waves[qredfit], redfit, line_color='red',
legend_label='Red')
p.circle(waves[qredfit], redfit, fill_color='red')
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# plot red ratios
p = figure(
title=self.action.args.plotlabel + ' Red ratios',
x_axis_label='Wave (A)',
y_axis_label='Ratio',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.line(waves[qredfit], refredfit/redfit, line_color='black',
legend_label='Ref')
p.circle(waves[qredfit], refredfit/redfit, fill_color='black')
p.line(waves[qredfit], redlinfity,
line_color='red', legend_label='Red')
p.circle(waves[qredfit], redlinfity, fill_color='red')
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# at this point we are going to try to merge the points
self.logger.info("Correcting points outside %.1f - %.1f A"
% (minrwave, maxrwave))
qselblue = [i for i, v in enumerate(xfb) if v <= minrwave]
qselred = [i for i, v in enumerate(xfd) if v >= maxrwave]
nqsb = len(qselblue)
nqsr = len(qselred)
blue_all_tie = yfitr[0]
red_all_tie = yfitr[-1]
self.logger.info("Blue/Red ref tie values: %.3f, %.3f"
% (blue_all_tie, red_all_tie))
if nqsb > 0:
self.logger.info("Blue ext tie value: %.3f" % yfb[qselblue[-1]])
if blue_zero_cross:
blue_offset = yfb[qselblue[-1]] - blue_all_tie
bluefluxes = [yfb[i] - blue_offset for i in qselblue]
self.logger.info("Blue zero crossing, only applying offset")
else:
blue_offset = yfb[qselblue[-1]] * \
(bluelinfit[1]+bluelinfit[0]*xfb[qselblue[-1]]) \
- blue_all_tie
bluefluxes = [yfb[i] * (bluelinfit[1]+bluelinfit[0]*xfb[i])
- blue_offset for i in qselblue]
self.logger.info("Blue linear ratio fit scaling applied")
self.logger.info("Blue offset of %.2f applied" % blue_offset)
else:
bluefluxes = None
if nqsr > 0:
self.logger.info("Red ext tie value: %.3f" % yfd[qselred[0]])
if red_zero_cross:
red_offset = yfd[qselred[0]] - red_all_tie
redfluxes = [yfd[i] - red_offset for i in qselred]
self.logger.info("Red zero crossing, only applying offset")
else:
red_offset = yfd[qselred[0]] * \
(redlinfit[1]+redlinfit[0]*xfd[qselred[0]]) \
- red_all_tie
redfluxes = [yfd[i] * (redlinfit[1]+redlinfit[0]*xfd[i])
- red_offset for i in qselred]
self.logger.info("Red linear ratio fit scaling applied")
self.logger.info("Red offset of %.2f applied" % red_offset)
else:
redfluxes = None
allx = xfr
allfx = xfr[nrefx:-nrefx]
ally = yfr[nrefx:-nrefx]
if nqsb > 0:
bluex = xfb[qselblue]
allx = np.append(bluex, allx)
allfx = np.append(bluex[nrefx:], allfx)
ally = np.append(bluefluxes[nrefx:], ally)
if nqsr > 0:
redx = xfd[qselred]
allx = np.append(allx, redx)
allfx = np.append(allfx, redx[:-nrefx])
ally = np.append(ally, redfluxes[:-nrefx])
s = np.argsort(allx)
allx = allx[s]
s = np.argsort(allfx)
allfx = allfx[s]
ally = ally[s]
bkpt = np.min(allx) + np.arange(knots+1) * \
(np.max(allx) - np.min(allx)) / knots
sftall, _ = Bspline.iterfit(allfx, ally, fullbkpt=bkpt)
yfitall, _ = sftall.value(allx)
if self.config.instrument.plot_level >= 1:
# output filename stub
fltfnam = "flat_%05d_%s_%s_%s" % \
(self.action.args.ccddata.header['FRAMENO'],
self.action.args.illum, self.action.args.grating,
self.action.args.ifuname)
if xbin == 1:
stride = int(len(allx) / 8000.)
else:
stride = 1
xplt = allfx[::stride]
yplt = ally[::stride]
fxplt = allx[::stride]
fplt = yfitall[::stride]
yran = [np.nanmin(ally), np.nanmax(ally)]
p = figure(
title=self.action.args.plotlabel + ' Master Illumination',
x_axis_label='Wave (A)',
y_axis_label='Flux (e-)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(xplt, yplt, size=1, line_alpha=0., fill_color='black',
legend_label='Data')
p.line(fxplt, fplt, line_color='red', legend_label='Fit',
line_width=2)
p.line([minrwave, minrwave], yran, line_color='orange',
legend_label='Cor lim')
p.line([maxrwave, maxrwave], yran, line_color='orange')
p.legend.location = "top_left"
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
save_plot(p, filename=fltfnam+".png")
# OK, Now we have extended to the full range... so... we are going to
# make a ratio flat!
comflat = np.zeros(newflat.shape, dtype=float)
qz = [i for i, v in enumerate(wavemap.data.flat) if v >= 0]
comvals = sftall.value(wavemap.data.flat[qz])
comflat.flat[qz] = comvals
ratio = np.zeros(newflat.shape, dtype=float)
qzer = [i for i, v in enumerate(newflat.flat) if v != 0]
ratio.flat[qzer] = comflat.flat[qzer] / newflat.flat[qzer]
# trim negative points
qq = [i for i, v in enumerate(ratio.flat) if v < 0]
if len(qq) > 0:
ratio.flat[qq] = 0.0
# trim the high points near edges of slice
qq = [i for i, v in enumerate(ratio.flat) if v >= 3. and
(posmap.data.flat[i] <= 4/xbin or
posmap.data.flat[i] >= 136/xbin)]
if len(qq) > 0:
ratio.flat[qq] = 0.0
# don't correct low signal points
qq = [i for i, v in enumerate(newflat.flat) if v < 30.]
if len(qq) > 0:
ratio.flat[qq] = 1.0
# get master flat output name
mfname = stack_list[0].split('.fits')[0] + '_' + suffix + '.fits'
log_string = MakeMasterFlat.__module__
stacked.header['IMTYPE'] = self.action.args.new_type
stacked.header['HISTORY'] = log_string
stacked.header['MASTFLAT'] = (True, 'master flat image?')
stacked.header['WAVMAPF'] = wmf
stacked.header['SLIMAPF'] = slf
stacked.header['POSMAPF'] = pof
# store flat in output frame
stacked.data = ratio
# output master flat
kcwi_fits_writer(stacked, output_file=mfname,
output_dir=self.config.instrument.output_directory)
self.context.proctab.update_proctab(frame=stacked, suffix=suffix,
newtype=self.action.args.new_type,
filename=self.action.args.name)
self.context.proctab.write_proctab()
self.logger.info(log_string)
return self.action.args
def _perform(self):
"""
Returns an Argument() with the parameters that depends on this operation
"""
method = 'average'
suffix = self.action.args.new_type.lower()
combine_list = list(self.combine_list['filename'])
# get master bias output name
# mbname = combine_list[-1].split('.fits')[0] + '_' + suffix + '.fits'
mbname = master_bias_name(self.action.args.ccddata)
stack = []
stackf = []
for bias in combine_list:
stackf.append(bias)
# using [0] drops the table
stack.append(kcwi_fits_reader(bias)[0])
stacked = ccdproc.combine(stack,
method=method,
sigma_clip=True,
sigma_clip_low_thresh=None,
sigma_clip_high_thresh=2.0)
stacked.header['IMTYPE'] = self.action.args.new_type
stacked.header['NSTACK'] = (len(combine_list),
'number of images stacked')
stacked.header['STCKMETH'] = (method, 'method used for stacking')
for ii, fname in enumerate(stackf):
fname_base = os.path.basename(fname)
stacked.header['STACKF%d' % (ii + 1)] = (fname_base,
"stack input file")
# for readnoise stats use 2nd and 3rd bias
diff = stack[1].data.astype(np.float32) - \
stack[2].data.astype(np.float32)
namps = stack[1].header['NVIDINP']
for ia in range(namps):
# get gain
gain = stacked.header['GAIN%d' % (ia + 1)]
# get amp section
sec, rfor = parse_imsec(stacked.header['DSEC%d' % (ia + 1)])
noise = diff[sec[0]:(sec[1] + 1), sec[2]:(sec[3] + 1)]
noise = np.reshape(noise, noise.shape[0]*noise.shape[1]) * \
gain / 1.414
# get stats on noise
c, low, upp = sigmaclip(noise, low=3.5, high=3.5)
bias_rn = c.std()
self.logger.info("Amp%d read noise from bias in e-: %.3f" %
((ia + 1), bias_rn))
stacked.header['BIASRN%d' % (ia + 1)] = \
(float("%.3f" % bias_rn), "RN in e- from bias")
if self.config.instrument.plot_level >= 1:
# output filename stub
biasfnam = "bias_%05d_amp%d_rdnoise" % \
(self.action.args.ccddata.header['FRAMENO'], ia+1)
plabel = '[ Img # %d' % self.action.args.ccddata.header[
'FRAMENO']
plabel += ' (Bias)'
plabel += ' %s' % self.action.args.ccddata.header['BINNING']
plabel += ' %s' % self.action.args.ccddata.header['AMPMODE']
plabel += ' %d' % self.action.args.ccddata.header['GAINMUL']
plabel += ' %s' % ('fast' if self.action.args.ccddata.
header['CCDMODE'] else 'slow')
plabel += ' ] '
hist, edges = np.histogram(noise,
range=(low, upp),
density=False,
bins=50)
x = np.linspace(low, upp, 500)
pdf = np.max(hist) * np.exp(-x**2 / (2. * bias_rn**2))
p = figure(title=plabel + 'BIAS NOISE amp %d = %.3f' %
(ia + 1, bias_rn),
x_axis_label='e-',
y_axis_label='N',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.quad(top=hist,
bottom=0,
left=edges[:-1],
right=edges[1:],
fill_color="navy",
line_color="white",
alpha=0.5)
p.line(x,
pdf,
line_color="#ff8888",
line_width=4,
alpha=0.7,
legend_label="PDF")
p.line([-bias_rn, -bias_rn], [0, np.max(hist)],
color='red',
legend_label="Sigma")
p.line([bias_rn, bias_rn], [0, np.max(hist)], color='red')
p.y_range.start = 0
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
save_plot(p, filename=biasfnam + ".png")
log_string = MakeMasterBias.__module__
stacked.header['HISTORY'] = log_string
self.logger.info(log_string)
kcwi_fits_writer(stacked,
output_file=mbname,
output_dir=self.config.instrument.output_directory)
self.context.proctab.update_proctab(frame=stacked,
suffix=suffix,
newtype=self.action.args.new_type,
filename=self.action.args.name)
self.context.proctab.write_proctab()
return Arguments(name=mbname)
def _perform(self):
self.logger.info("Tracing continuum bars")
if self.config.instrument.plot_level >= 1:
do_plot = True
else:
do_plot = False
if len(self.action.args.middle_centers) < 1:
self.logger.error("No bars found")
elif not self.action.args.bar_avg:
self.logger.error("No threshold for tracing")
else:
# initialize
samp = int(80 / self.action.args.ybinsize)
win = self.action.args.window
bar_thresh = self.action.args.bar_avg
self.logger.info("Tracing bars with threshold of %.1f" % bar_thresh)
xi = [] # x input
xo = [] # x output
yi = [] # y input (and output)
barid = [] # bar id number
slid = [] # slice id number
# loop over bars
for barn, barx in enumerate(self.action.args.middle_centers):
# nearest pixel to bar center
barxi = int(barx + 0.5)
self.logger.info("bar number %d is at %.3f" % (barn, barx))
# middle row data
xi.append(barx)
xo.append(barx)
yi.append(self.action.args.middle_row)
barid.append(barn)
slid.append(int(barn/5))
# trace up
samy = self.action.args.middle_row + samp
done = False
while samy < (self.action.args.ccddata.data.shape[0] - win) \
and not done:
ys = np.median(
self.action.args.ccddata.data[(samy - win):
(samy + win + 1),
(barxi - win):
(barxi + win + 1)],
axis=0)
ys = ys - np.nanmin(ys)
if np.nanmax(ys) > bar_thresh and np.nansum(ys) > 0:
xs = list(range(barxi - win, barxi + win + 1))
xc = np.nansum(xs * ys) / np.nansum(ys)
xi.append(xc)
xo.append(barx)
yi.append(samy)
barid.append(barn)
slid.append(int(barn/5))
barxi = int(xc)
else:
done = True
samy += samp
# trace down
# nearest pixel to bar center
barxi = int(barx + 0.5)
samy = self.action.args.middle_row - samp
done = False
while samy >= win and not done:
ys = np.median(
self.action.args.ccddata.data[(samy - win):
(samy + win + 1),
(barxi - win):
(barxi + win + 1)],
axis=0)
ys = ys - np.nanmin(ys)
if np.nanmax(ys) > bar_thresh and np.nansum(ys) > 0:
xs = list(range(barxi - win, barxi + win + 1))
xc = np.sum(xs * ys) / np.sum(ys)
xi.append(xc)
xo.append(barx)
yi.append(samy)
barid.append(barn)
slid.append(int(barn / 5))
barxi = int(xc)
else:
done = True
samy -= samp
# end loop over bars
# create source and destination coords
yo = yi
dst = np.column_stack((xi, yi))
src = np.column_stack((xo, yo))
if do_plot:
# output filename stub
trcfnam = "bars_%05d_%s_%s_%s" % \
(self.action.args.ccddata.header['FRAMENO'],
self.action.args.illum, self.action.args.grating,
self.action.args.ifuname)
# plot them
p = figure(title=self.action.args.plotlabel +
'SPATIAL CONTROL POINTS',
x_axis_label="CCD X (px)", y_axis_label="CCD Y (px)",
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.scatter(xi, yi, marker='x', size=2, color='blue')
p.scatter(self.action.args.middle_centers,
[self.action.args.middle_row]*120, color='red')
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
save_plot(p, filename=trcfnam+".png")
trace = {
'src': src,
'dst': dst,
'barid': barid,
'slid': slid,
'MIDROW': self.action.args.middle_row,
'WINDOW': self.action.args.window,
'REFDELX': self.action.args.reference_delta_x,
'CBARSNO': self.action.args.contbar_image_number,
'CBARSFL': self.action.args.contbar_image}
# in this line we pass the trace information to an argument
# instead of writing it to a table
self.context.trace = trace
ofname = strip_fname(self.action.args.contbar_image) + \
"_trace.fits"
write_table(table=[src, dst, barid, slid],
names=('src', 'dst', 'barid', 'slid'),
output_dir=os.path.join(
self.config.instrument.cwd,
self.config.instrument.output_directory),
output_name=ofname,
clobber=self.config.instrument.clobber,
comment=['Source and destination fiducial points',
'Derived from KCWI continuum bars images',
'For defining spatial transformation'],
keywords={'MIDROW': (self.action.args.middle_row,
"Middle Row of image"),
'WINDOW': (self.action.args.window,
"Window for bar"),
'REFDELX': (
self.action.args.reference_delta_x,
"Reference bar sep in px"),
'CBARSNO': (
self.action.args.contbar_image_number,
"Cont. bars image number"),
'CBARSFL': (self.action.args.contbar_image,
"Cont. bars image")})
if self.config.instrument.saveintims:
from kcwidrp.primitives.kcwi_file_primitives import \
kcwi_fits_writer
from skimage import transform as tf
# fit transform
self.logger.info("Fitting spatial control points")
tform = tf.estimate_transform('polynomial', src, dst, order=3)
self.logger.info("Transforming bars image")
warped = tf.warp(self.action.args.ccddata.data, tform)
# write out warped image
self.action.args.ccddata.data = warped
kcwi_fits_writer(
self.action.args.ccddata, output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix='warped')
self.logger.info("Transformed bars produced")
log_string = TraceBars.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
self.logger.info(log_string)
return self.action.args
def _perform(self):
"""Solve individual arc bar spectra for wavelength"""
self.logger.info("Solving individual arc spectra")
# plot control booleans
master_inter = (self.config.instrument.plot_level >= 2)
do_inter = (self.config.instrument.plot_level >= 3)
# output control
verbose = (self.config.instrument.verbose > 1)
# Bar statistics
bar_sig = []
bar_nls = []
# set thresh for finding lines
hgt = 50.
self.logger.info("line thresh = %.2f" % hgt)
# get relevant part of atlas spectrum
atwave = self.action.args.refwave[self.action.args.atminrow:self.
action.args.atmaxrow]
atspec = self.action.args.reflux[self.action.args.atminrow:self.action.
args.atmaxrow]
# convert list into ndarray
at_wave = np.asarray(self.action.args.at_wave)
at_flux = np.asarray(self.action.args.at_flux)
# get x values starting at zero pixels
self.action.args.xsvals = np.arange(
0, len(self.context.arcs[self.config.instrument.REFBAR]))
# loop over arcs and generate a wavelength solution for each
next_bar_to_plot = 0
poly_order = 4
for ib, b in enumerate(self.context.arcs):
# Starting with pascal shifted coeffs from fit_center()
coeff = self.action.args.twkcoeff[ib]
# get bar wavelengths
bw = np.polyval(coeff, self.action.args.xsvals)
# smooth spectrum according to slicer
if 'Small' in self.action.args.ifuname:
# no smoothing for Small slicer
bspec = b
else:
if 'Large' in self.action.args.ifuname:
# max smoothing for Large slicer
win = boxcar(5)
else:
# intermediate smoothing for Medium slicer
win = boxcar(3)
# do the smoothing
bspec = sp.signal.convolve(b, win, mode='same') / sum(win)
# store values to fit
at_wave_dat = [] # atlas line wavelengths
at_flux_dat = [] # atlas line peak fluxes
arc_pix_dat = [] # arc line pixel positions
arc_int_dat = [] # arc line pixel intensities
rej_wave = [] # rejected line wavelengths
rej_flux = [] # rejected line fluxes
gaus_sig = []
nrej = 0
# loop over lines
for iw, aw in enumerate(self.action.args.at_wave):
# get window for this line
try:
# get arc line initial pixel position
line_x = [i for i, v in enumerate(bw) if v >= aw][0]
# get window for arc line
minow, maxow, count = get_line_window(
bspec,
line_x,
thresh=hgt,
logger=(self.logger if verbose else None))
# do we have enough points to fit?
if count < 5 or not minow or not maxow:
rej_wave.append(aw)
rej_flux.append(self.action.args.at_flux[iw])
nrej += 1
if verbose:
self.logger.info(
"Arc window rejected for line %.3f" % aw)
continue
# check if window no longer contains initial value
if minow > line_x > maxow:
rej_wave.append(aw)
rej_flux.append(self.action.args.at_flux[iw])
nrej += 1
if verbose:
self.logger.info(
"Arc window wandered off for line %.3f" % aw)
continue
# get data to fit
yvec = bspec[minow:maxow + 1]
xvec = self.action.args.xsvals[minow:maxow + 1]
wvec = bw[minow:maxow + 1]
f0 = max(yvec)
par_start = [f0, np.nanmean(xvec), 1.0]
par_bounds = ([f0 * 0.9, np.min(xvec),
0.5], [f0 * 1.1,
np.max(xvec), 2.5])
# Gaussian fit
try:
fit, _ = curve_fit(gaus, xvec, yvec, p0=par_start)
# bounds=par_bounds, method='trf')
sp_pk_x = fit[1]
gaus_sig.append(fit[2])
except (RuntimeError, ValueError):
rej_wave.append(aw)
rej_flux.append(self.action.args.at_flux[iw])
nrej += 1
if verbose:
self.logger.info(
"Arc Gaussian fit rejected for line %.3f" % aw)
# sp_pk_x = line_x
continue
# get interpolation of arc line
int_line = interpolate.interp1d(xvec,
yvec,
kind='cubic',
bounds_error=False,
fill_value='extrapolate')
# use very dense sampling
xplot = np.linspace(min(xvec), max(xvec), num=1000)
# re-sample line with dense sampling
plt_line = int_line(xplot)
# get peak position
max_index = plt_line.argmax()
peak = xplot[max_index]
# calculate centroid
cent = np.sum(xvec * yvec) / np.sum(yvec)
# how different is the centroid from the peak?
if abs(cent - peak) > 0.7:
# keep track of rejected line
rej_wave.append(aw)
rej_flux.append(self.action.args.at_flux[iw])
nrej += 1
if verbose:
self.logger.info("Arc peak - cent offset = %.2f "
"rejected for line %.3f" %
(abs(cent - peak), aw))
continue
if plt_line[max_index] < 100:
# keep track of rejected line
rej_wave.append(aw)
rej_flux.append(self.action.args.at_flux[iw])
nrej += 1
if verbose:
self.logger.info("Arc peak too low = %.2f "
"rejected for line %.3f" %
(plt_line[max_index], aw))
continue
# store surviving line data
arc_pix_dat.append(peak)
arc_int_dat.append(plt_line[max_index])
at_wave_dat.append(aw)
at_flux_dat.append(self.action.args.at_flux[iw])
# plot, if requested
if do_inter and ib == next_bar_to_plot:
ptitle = " Bar# %d - line %3d/%3d: xc = %.1f, " \
"Wave = %9.2f" % \
(ib, (iw + 1), len(self.action.args.at_wave),
peak, aw)
atx0 = [
i for i, v in enumerate(atwave) if v >= min(wvec)
][0]
atx1 = [
i for i, v in enumerate(atwave) if v >= max(wvec)
][0]
atnorm = np.nanmax(yvec) / np.nanmax(atspec[atx0:atx1])
p = figure(
title=self.action.args.plotlabel +
"ATLAS/ARC LINE FITS" + ptitle,
x_axis_label="Wavelength (A)",
y_axis_label="Relative Flux",
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
ylim = [0, np.nanmax(yvec)]
p.line(atwave[atx0:atx1],
atspec[atx0:atx1] * atnorm,
color='blue',
legend_label='Atlas')
p.circle(atwave[atx0:atx1],
atspec[atx0:atx1] * atnorm,
color='green',
legend_label='Atlas')
p.line([aw, aw],
ylim,
color='red',
legend_label='AtCntr')
p.x_range = Range1d(start=min(wvec), end=max(wvec))
p.extra_x_ranges = {
"pix": Range1d(start=min(xvec), end=max(xvec))
}
p.add_layout(
LinearAxis(x_range_name="pix",
axis_label="CCD Y pix"), 'above')
p.line(xplot,
plt_line,
color='black',
legend_label='Arc',
x_range_name="pix")
p.circle(xvec,
yvec,
legend_label='Arc',
color='red',
x_range_name="pix")
ylim = [0, np.nanmax(plt_line)]
p.line([cent, cent],
ylim,
color='green',
legend_label='Cntr',
line_dash='dashed',
x_range_name="pix")
p.line([sp_pk_x, sp_pk_x],
ylim,
color='magenta',
legend_label='Gpeak',
line_dash='dashdot',
x_range_name="pix")
p.line([peak, peak],
ylim,
color='black',
legend_label='Peak',
line_dash='dashdot',
x_range_name="pix")
p.y_range.start = 0
bokeh_plot(p, self.context.bokeh_session)
q = input(ptitle + " - Next? <cr>, q to quit: ")
if 'Q' in q.upper():
do_inter = False
except IndexError:
if verbose:
self.logger.info(
"Atlas line not in observation: %.2f" % aw)
rej_wave.append(aw)
rej_flux.append(self.action.args.at_flux[iw])
nrej += 1
continue
except ValueError:
if verbose:
self.logger.info(
"Interpolation error for line at %.2f" % aw)
rej_wave.append(aw)
rej_flux.append(self.action.args.at_flux[iw])
nrej += 1
self.logger.info("")
self.logger.info("Fitting wavelength solution starting with %d "
"lines after rejecting %d lines" %
(len(arc_pix_dat), nrej))
# Fit wavelengths
# Get poly order
if self.action.args.dichroic_fraction <= 0.6:
poly_order = 2
elif 0.6 < self.action.args.dichroic_fraction < 0.75:
poly_order = 3
else:
poly_order = 4
self.logger.info("Fitting with polynomial order %d" % poly_order)
# Initial fit
wfit = np.polyfit(arc_pix_dat, at_wave_dat, poly_order)
pwfit = np.poly1d(wfit)
arc_wave_fit = pwfit(arc_pix_dat)
# fit residuals
resid = arc_wave_fit - at_wave_dat
resid_c, low, upp = sigmaclip(resid, low=3., high=3.)
wsig = resid_c.std()
# maximum outlier
max_resid = np.max(abs(resid))
self.logger.info("wsig: %.3f, max_resid: %.3f" % (wsig, max_resid))
# keep track of rejected lines
rej_rsd = [] # rejected line residuals
rej_rsd_wave = [] # rejected line wavelengths
rej_rsd_flux = [] # rejected line fluxes
# iteratively remove outliers
it = 0
while max_resid > 2.5 * wsig and it < 25:
arc_dat = [] # arc line pixel values
arc_fdat = [] # arc line flux data
at_dat = [] # atlas line wavelength values
at_fdat = [] # atlas line flux data
# trim largest outlier
for il, rsd in enumerate(resid):
if abs(rsd) < max_resid:
# append data for line that passed cut
arc_dat.append(arc_pix_dat[il])
arc_fdat.append(arc_int_dat[il])
at_dat.append(at_wave_dat[il])
at_fdat.append(at_flux_dat[il])
else:
if verbose:
self.logger.info("It%d REJ: %d, %.2f, %.3f, %.3f" %
(it, il, arc_pix_dat[il],
at_wave_dat[il], rsd))
# keep track of rejected lines
rej_rsd_wave.append(at_wave_dat[il])
rej_rsd_flux.append(at_flux_dat[il])
rej_rsd.append(rsd)
# copy cleaned data back into input arrays
arc_pix_dat = arc_dat.copy()
arc_int_dat = arc_fdat.copy()
at_wave_dat = at_dat.copy()
at_flux_dat = at_fdat.copy()
# refit cleaned data
wfit = np.polyfit(arc_pix_dat, at_wave_dat, poly_order)
# new wavelength function
pwfit = np.poly1d(wfit)
# new wavelengths for arc lines
arc_wave_fit = pwfit(arc_pix_dat)
# calculate residuals of arc lines
resid = arc_wave_fit - at_wave_dat
# get statistics
resid_c, low, upp = sigmaclip(resid, low=3., high=3.)
wsig = resid_c.std()
# maximum outlier
max_resid = np.max(abs(resid))
# wsig = np.nanstd(resid)
it += 1
# END while max_resid > 3.5 * wsig and it < 5:
# log arc bar results
self.logger.info("")
self.logger.info("BAR %03d, Slice = %02d, RMS = %.3f, N = %d" %
(ib, int(ib / 5), wsig, len(arc_pix_dat)))
self.logger.info("Nits: %d, wsig: %.3f, max_resid: %.3f" %
(it, wsig, max_resid))
self.logger.info("NRejRsd: %d, NRejFit: %d" %
(len(rej_rsd_wave), len(rej_wave)))
self.logger.info("Line width median sigma: %.2f px" %
np.nanmedian(gaus_sig))
self.logger.info("Coefs: " +
' '.join(['%.6g' % (c, )
for c in reversed(wfit)]))
# store final fit coefficients
self.action.args.fincoeff.append(wfit)
# store statistics
bar_sig.append(wsig)
bar_nls.append(len(arc_pix_dat))
# do plotting?
if master_inter and ib == next_bar_to_plot:
# plot bar fit residuals
ptitle = " for Bar %03d, Slice %02d, RMS = %.3f, N = %d" % \
(ib, int(ib / 5), wsig, len(arc_pix_dat))
p = figure(title=self.action.args.plotlabel + "RESIDUALS" +
ptitle,
x_axis_label="Wavelength (A)",
y_axis_label="Fit - Inp (A)",
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.diamond(at_wave_dat, resid, legend_label='Rsd', size=8)
if rej_rsd_wave:
p.diamond(rej_rsd_wave,
rej_rsd,
color='orange',
legend_label='Rej',
size=8)
xlim = [self.action.args.atminwave, self.action.args.atmaxwave]
ylim = [
np.nanmin(list(resid) + list(rej_rsd)),
np.nanmax(list(resid) + list(rej_rsd))
]
p.line(xlim, [0., 0.], color='black', line_dash='dotted')
p.line(xlim, [wsig, wsig], color='gray', line_dash='dashdot')
p.line(xlim, [-wsig, -wsig], color='gray', line_dash='dashdot')
p.line([self.action.args.cwave, self.action.args.cwave],
ylim,
legend_label='CWAV',
color='magenta',
line_dash='dashdot')
bokeh_plot(p, self.context.bokeh_session)
input("Next? <cr>: ")
# overplot atlas and bar using fit wavelengths
p = figure(title=self.action.args.plotlabel + "ATLAS/ARC FIT" +
ptitle,
x_axis_label="Wavelength (A)",
y_axis_label="Flux",
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
bwav = pwfit(self.action.args.xsvals)
p.line(bwav, b, color='darkgrey', legend_label='Arc')
p.diamond(arc_wave_fit, arc_int_dat, color='darkgrey', size=8)
ylim = [np.nanmin(b), np.nanmax(b)]
atnorm = np.nanmax(b) / np.nanmax(atspec)
p.line(atwave,
atspec * atnorm,
color='blue',
legend_label='Atlas')
p.line([self.action.args.cwave, self.action.args.cwave],
ylim,
color='magenta',
line_dash='dashdot',
legend_label='CWAV')
p.diamond(at_wave,
at_flux * atnorm,
legend_label='Kept',
color='green',
size=8)
if rej_rsd_wave:
p.diamond(rej_rsd_wave,
[rj * atnorm for rj in rej_rsd_flux],
color='orange',
legend_label='RejRsd',
size=6)
p.diamond(rej_wave, [rj * atnorm for rj in rej_flux],
color='red',
legend_label='RejFit',
size=6)
bokeh_plot(p, self.context.bokeh_session)
q = input("Next? <int> or <cr>, q - quit: ")
if 'Q' in q.upper():
master_inter = False
else:
try:
next_bar_to_plot = int(q)
except ValueError:
next_bar_to_plot = ib + 1
# Plot final results
# plot output name stub
pfname = "arc_%05d_%s_%s_%s_tf%02d" % (
self.action.args.ccddata.header['FRAMENO'], self.action.args.illum,
self.action.args.grating, self.action.args.ifuname,
int(100 * self.config.instrument.TAPERFRAC))
# Plot coefs
if self.config.instrument.plot_level >= 1:
ylabs = ['Ang/px^4', 'Ang/px^3', 'Ang/px^2', 'Ang/px', 'Ang']
ylabs = ylabs[-(poly_order + 1):]
for ic in reversed(range(len(self.action.args.fincoeff[0]))):
cn = poly_order - ic
ptitle = self.action.args.plotlabel + "COEF %d VALUES" % cn
p = figure(title=ptitle,
x_axis_label="Bar #",
y_axis_label="Coef %d (%s)" % (cn, ylabs[ic]),
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
coef = []
for c in self.action.args.fincoeff:
coef.append(c[ic])
p.diamond(list(range(120)), coef, size=8)
xlim = [-1, 120]
ylim = get_plot_lims(coef)
p.xgrid.grid_line_color = None
oplot_slices(p, ylim)
set_plot_lims(p, xlim=xlim, ylim=ylim)
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# save coefficients plot
save_plot(p, filename=pfname + '_coef%d.png' % cn)
# Plot number of lines fit
self.action.args.av_bar_nls = float(np.nanmean(bar_nls))
self.action.args.st_bar_nls = float(np.nanstd(bar_nls))
ptitle = self.action.args.plotlabel + \
"FIT STATS <Nlns> = %.1f +- %.1f" % (self.action.args.av_bar_nls,
self.action.args.st_bar_nls)
p = figure(title=ptitle,
x_axis_label="Bar #",
y_axis_label="N Lines",
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.diamond(list(range(120)), bar_nls, size=8)
xlim = [-1, 120]
ylim = get_plot_lims(bar_nls)
self.logger.info(
"<N Lines> = %.1f +- %.1f" %
(self.action.args.av_bar_nls, self.action.args.st_bar_nls))
p.line(xlim,
[self.action.args.av_bar_nls, self.action.args.av_bar_nls],
color='red')
p.line(xlim,
[(self.action.args.av_bar_nls - self.action.args.st_bar_nls),
(self.action.args.av_bar_nls - self.action.args.st_bar_nls)],
color='green',
line_dash='dashed')
p.line(xlim,
[(self.action.args.av_bar_nls + self.action.args.st_bar_nls),
(self.action.args.av_bar_nls + self.action.args.st_bar_nls)],
color='green',
line_dash='dashed')
p.xgrid.grid_line_color = None
oplot_slices(p, ylim)
set_plot_lims(p, xlim=xlim, ylim=ylim)
if self.config.instrument.plot_level >= 1:
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# save N lines plot
save_plot(p, filename=pfname + '_nlines.png')
# Plot fit sigmas
self.action.args.av_bar_sig = float(np.nanmean(bar_sig))
self.action.args.st_bar_sig = float(np.nanstd(bar_sig))
self.logger.info(
"<STD> = %.3f +- %.3f (A)" %
(self.action.args.av_bar_sig, self.action.args.st_bar_sig))
ptitle = self.action.args.plotlabel + \
"FIT STATS <RMS> = %.3f +- %.3f" % (self.action.args.av_bar_sig,
self.action.args.st_bar_sig)
p = figure(title=ptitle,
x_axis_label="Bar #",
y_axis_label="RMS (A)",
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.diamond(list(range(120)), bar_sig, size=8)
xlim = [-1, 120]
ylim = get_plot_lims(bar_sig)
p.line(xlim,
[self.action.args.av_bar_sig, self.action.args.av_bar_sig],
color='red')
p.line(xlim,
[(self.action.args.av_bar_sig - self.action.args.st_bar_sig),
(self.action.args.av_bar_sig - self.action.args.st_bar_sig)],
color='green',
line_dash='dashed')
p.line(xlim,
[(self.action.args.av_bar_sig + self.action.args.st_bar_sig),
(self.action.args.av_bar_sig + self.action.args.st_bar_sig)],
color='green',
line_dash='dashed')
p.xgrid.grid_line_color = None
oplot_slices(p, ylim)
set_plot_lims(p, xlim=xlim, ylim=ylim)
if self.config.instrument.plot_level >= 1:
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
# save residual plot
save_plot(p, filename=pfname + '_resid.png')
log_string = SolveArcs.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
self.logger.info(log_string)
return self.action.args
def _perform(self):
# Header keyword to update
key = 'SCATSUB'
keycom = "was scattered light subtracted?"
# Skip if nod-and-shuffle
if self.action.args.nasmask:
self.logger.info("NAS Mask: skipping scattered light subtraction")
self.action.args.ccddata.header[key] = (False, keycom)
elif self.config.instrument.skipscat:
self.logger.info("Skipping scattered light subtraction by request")
self.action.args.ccddata.header[key] = (False, keycom)
else:
# Binning
ybin = self.action.args.ybinsize
# Get size of image
siz = self.action.args.ccddata.data.shape
# Get x range for scattered light
x0 = int(siz[1] / 2 - 180 / self.action.args.xbinsize)
x1 = int(siz[1] / 2 + 180 / self.action.args.xbinsize)
# Get y limits
y0 = 0
# y1 = int(siz[0] / 2 - 1)
# y2 = y1 + 1
y3 = siz[0]
# print("x limits: %d, %d, y limits: %d, %d" % (x0, x1, y0, y3))
# Y data values
yvals = np.nanmedian(self.action.args.ccddata.data[y0:y3, x0:x1],
axis=1)
# Fix extreme values
yvals[0] = np.nanmedian(yvals[1:10])
yvals[-1] = np.nanmedian(yvals[-11:-2])
# X data values
xvals = np.arange(len(yvals), dtype=np.float)
# filter window
fwin = 151
scat = savgol_filter(yvals, fwin, 3)
signal_to_noise = np.mean(scat) / np.nanstd(yvals - scat)
if signal_to_noise < 25.:
if signal_to_noise < 5:
fwin = 501
else:
fwin = 303
scat = savgol_filter(yvals, fwin, 3)
signal_to_noise = np.mean(scat) / np.nanstd(yvals - scat)
self.logger.info("Smoothing scattered light with window of %d px" %
fwin)
self.logger.info("Mean signal to noise = %.2f" % signal_to_noise)
if self.config.instrument.plot_level >= 1:
# output filename stub
scfnam = "scat_%05d_%s_%s_%s" % \
(self.action.args.ccddata.header['FRAMENO'],
self.action.args.illum, self.action.args.grating,
self.action.args.ifuname)
# plot
p = figure(title=self.action.args.plotlabel +
"SCATTERED LIGHT FIT",
x_axis_label='y pixel',
y_axis_label='e-',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(xvals, yvals, legend_label="Scat")
p.line(xvals,
scat,
color='red',
line_width=3,
legend_label="Smooth")
p.legend.location = "top_left"
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
save_plot(p, filename=scfnam + ".png")
# Subtract scattered light
self.logger.info("Starting scattered light subtraction")
for ix in range(0, siz[1]):
self.action.args.ccddata.data[y0:y3, ix] = \
self.action.args.ccddata.data[y0:y3, ix] - scat
self.action.args.ccddata.header[key] = (True, keycom)
log_string = SubtractScatteredLight.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
self.logger.info(log_string)
# write out int image
kcwi_fits_writer(self.action.args.ccddata,
table=self.action.args.table,
output_file=self.action.args.name,
output_dir=self.config.instrument.output_directory,
suffix="intd")
self.context.proctab.update_proctab(frame=self.action.args.ccddata,
suffix="intd")
self.context.proctab.write_proctab()
return self.action.args
def _perform(self):
"""
Returns an Argument() with the parameters that depends on this operation
"""
self.logger.info("Creating master sky")
suffix = self.action.args.new_type.lower()
# get root for maps
tab = self.context.proctab.n_proctab(
frame=self.action.args.ccddata, target_type='ARCLAMP',
target_group=self.action.args.groupid)
if len(tab) <= 0:
self.logger.error("Geometry not solved!")
return self.action.args
groot = tab['OFNAME'][0].split('.fits')[0]
# Wavelength map image
wmf = groot + '_wavemap.fits'
self.logger.info("Reading image: %s" % wmf)
wavemap = kcwi_fits_reader(
os.path.join(os.path.dirname(self.action.args.name), 'redux',
wmf))[0]
# Slice map image
slf = groot + '_slicemap.fits'
self.logger.info("Reading image: %s" % slf)
slicemap = kcwi_fits_reader(
os.path.join(os.path.dirname(self.action.args.name), 'redux',
slf))[0]
# Position map image
pof = groot + '_posmap.fits'
self.logger.info("Reading image: %s" % pof)
posmap = kcwi_fits_reader(
os.path.join(os.path.dirname(self.action.args.name), 'redux',
pof))[0]
posmax = np.nanmax(posmap.data)
posbuf = int(10. / self.action.args.xbinsize)
# wavelength region
wavegood0 = wavemap.header['WAVGOOD0']
wavegood1 = wavemap.header['WAVGOOD1']
waveall0 = wavemap.header['WAVALL0']
waveall1 = wavemap.header['WAVALL1']
# get image size
sm_sz = self.action.args.ccddata.data.shape
# sky masking
# default is no masking (True = mask, False = don't mask)
binary_mask = np.zeros(sm_sz, dtype=bool)
# was sky masking requested?
if self.action.args.skymask:
if os.path.exists(self.action.args.skymask):
self.logger.info("Reading sky mask file: %s"
% self.action.args.skymask)
hdul = fits.open(self.action.args.skymask)
binary_mask = hdul[0].data
# verify size match
bm_sz = binary_mask.shape
if bm_sz[0] != sm_sz[0] or bm_sz[1] != sm_sz[1]:
self.logger.warning("Sky mask size mis-match: "
"masking disabled")
binary_mask = np.zeros(sm_sz, dtype=bool)
else:
self.logger.warning("Sky mask image not found: %s"
% self.action.args.skymask)
# count masked pixels
tmsk = len(np.nonzero(np.where(binary_mask.flat, True, False))[0])
self.logger.info("Number of pixels masked = %d" % tmsk)
finiteflux = np.isfinite(self.action.args.ccddata.data.flat)
# get un-masked points mapped to exposed regions on CCD
# handle dichroic bad region
if self.action.args.dich:
if self.action.args.camera == 0: # Blue
q = [i for i, v in enumerate(slicemap.data.flat)
if 0 <= v <= 23 and
posbuf < posmap.data.flat[i] < (posmax - posbuf) and
waveall0 <= wavemap.data.flat[i] <= waveall1 and
not (v > 20 and wavemap.data.flat[i] > 5600.) and
finiteflux[i] and not binary_mask.flat[i]]
else: # Red
q = [i for i, v in enumerate(slicemap.data.flat)
if 0 <= v <= 23 and
posbuf < posmap.data.flat[i] < (posmax - posbuf) and
waveall0 <= wavemap.data.flat[i] <= waveall1 and
not (v > 20 and wavemap.data.flat[i] < 5600.) and
finiteflux[i] and not binary_mask.flat[i]]
else:
q = [i for i, v in enumerate(slicemap.data.flat)
if 0 <= v <= 23 and
posbuf < posmap.data.flat[i] < (posmax - posbuf) and
waveall0 <= wavemap.data.flat[i] <= waveall1 and
finiteflux[i] and not binary_mask.flat[i]]
# get all points mapped to exposed regions on the CCD (for output)
qo = [i for i, v in enumerate(slicemap.data.flat)
if 0 <= v <= 23 and posmap.data.flat[i] >= 0 and
waveall0 <= wavemap.data.flat[i] <= waveall1 and
finiteflux[i]]
# extract relevant image values
fluxes = self.action.args.ccddata.data.flat[q]
# relevant wavelengths
waves = wavemap.data.flat[q]
self.logger.info("Number of fit waves = %d" % len(waves))
# keep output wavelengths
owaves = wavemap.data.flat[qo]
self.logger.info("Number of output waves = %d" % len(owaves))
# sort on wavelength
s = np.argsort(waves)
waves = waves[s]
fluxes = fluxes[s]
# knots per pixel
knotspp = self.config.instrument.KNOTSPP
n = int(sm_sz[0] * knotspp)
# calculate break points for b splines
bkpt = np.min(waves) + np.arange(n+1) * \
(np.max(waves) - np.min(waves)) / n
# log
self.logger.info("Nknots = %d, min = %.2f, max = %.2f (A)" %
(n, np.min(bkpt), np.max(bkpt)))
# do bspline fit
sft0, gmask = Bspline.iterfit(waves, fluxes, fullbkpt=bkpt,
upper=1, lower=1)
gp = [i for i, v in enumerate(gmask) if v]
yfit1, _ = sft0.value(waves)
self.logger.info("Number of good points = %d" % len(gp))
# check result
if np.max(yfit1) < 0:
self.logger.warning("B-spline failure")
if n > 2000:
if n == 5000:
n = 2000
if n == 8000:
n = 5000
# calculate breakpoints
bkpt = np.min(waves) + np.arange(n + 1) * \
(np.max(waves) - np.min(waves)) / n
# log
self.logger.info("Nknots = %d, min = %.2f, max = %.2f (A)" %
(n, np.min(bkpt), np.max(bkpt)))
# do bspline fit
sft0, gmask = Bspline.iterfit(waves, fluxes, fullbkpt=bkpt,
upper=1, lower=1)
yfit1, _ = sft0.value(waves)
if np.max(yfit1) <= 0:
self.logger.warning("B-spline final failure, sky is zero")
# get values at original wavelengths
yfit, _ = sft0.value(owaves)
# for plotting
gwaves = waves[gp]
gfluxes = fluxes[gp]
npts = len(gwaves)
stride = int(npts / 8000.)
xplt = gwaves[::stride]
yplt = gfluxes[::stride]
fplt, _ = sft0.value(xplt)
yrng = [np.min(yplt), np.max(yplt)]
self.logger.info("Stride = %d" % stride)
# plot, if requested
if self.config.instrument.plot_level >= 1:
# output filename stub
skyfnam = "sky_%05d_%s_%s_%s" % \
(self.action.args.ccddata.header['FRAMENO'],
self.action.args.illum, self.action.args.grating,
self.action.args.ifuname)
p = figure(
title=self.action.args.plotlabel + ' Master Sky',
x_axis_label='Wave (A)',
y_axis_label='Flux (e-)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.circle(xplt, yplt, size=1, line_alpha=0., fill_color='purple',
legend_label='Data')
p.line(xplt, fplt, line_color='red', legend_label='Fit')
p.line([wavegood0, wavegood0], yrng, line_color='green')
p.line([wavegood1, wavegood1], yrng, line_color='green')
p.y_range.start = yrng[0]
p.y_range.end = yrng[1]
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
save_plot(p, filename=skyfnam+".png")
# create sky image
sky = np.zeros(self.action.args.ccddata.data.shape, dtype=float)
sky.flat[qo] = yfit
# store original data, header
img = self.action.args.ccddata.data
hdr = self.action.args.ccddata.header.copy()
self.action.args.ccddata.data = sky
# get master sky output name
ofn = self.action.args.ccddata.header['OFNAME']
msname = ofn.split('.fits')[0] + '_' + suffix + '.fits'
log_string = MakeMasterSky.__module__
self.action.args.ccddata.header['IMTYPE'] = self.action.args.new_type
self.action.args.ccddata.header['HISTORY'] = log_string
self.action.args.ccddata.header['SKYMODEL'] = (True, 'sky model image?')
self.action.args.ccddata.header['SKYIMAGE'] = \
(ofn, 'image used for sky model')
if tmsk > 0:
self.action.args.ccddata.header['SKYMSK'] = (True,
'was sky masked?')
# self.action.args.ccddata.header['SKYMSKF'] = (skymf,
# 'sky mask file')
else:
self.action.args.ccddata.header['SKYMSK'] = (False,
'was sky masked?')
self.action.args.ccddata.header['WAVMAPF'] = wmf
self.action.args.ccddata.header['SLIMAPF'] = slf
self.action.args.ccddata.header['POSMAPF'] = pof
# output master sky
kcwi_fits_writer(self.action.args.ccddata, output_file=msname,
output_dir=self.config.instrument.output_directory)
self.context.proctab.update_proctab(frame=self.action.args.ccddata,
suffix=suffix,
newtype=self.action.args.new_type)
self.context.proctab.write_proctab()
# restore original image
self.action.args.ccddata.data = img
self.action.args.ccddata.header = hdr
self.logger.info(log_string)
return self.action.args
def _perform(self):
self.logger.info("Making inverse sensitivity curve")
suffix = 'invsens'
stdname = self.action.args.stdname
# get size
sz = self.action.args.ccddata.data.shape
# default pixel ranges
z = np.arange(sz[0])
z0 = 175
z1 = sz[0] - 175
# get exposure time
expt = self.action.args.ccddata.header['XPOSURE']
if expt == 0.:
self.logger.warning("No exposure time found, setting to 1s")
expt = 1.
else:
self.logger.info("Using exposure time of %.1f" % expt)
# get wavelength scale
w0 = self.action.args.ccddata.header['CRVAL3']
dw = self.action.args.ccddata.header['CD3_3']
crpixw = self.action.args.ccddata.header['CRPIX3']
# get all good wavelength range
wgoo0 = self.action.args.ccddata.header['WAVGOOD0']
if wgoo0 < 3650:
wgoo0 = 3650.
wgoo1 = self.action.args.ccddata.header['WAVGOOD1']
# get all inclusive wavelength range
wall0 = self.action.args.ccddata.header['WAVALL0']
wall1 = self.action.args.ccddata.header['WAVALL1']
# get DAR padding in y
pad_y = self.action.args.ccddata.header['DARPADY']
# get sky subtraction status
skycor = self.action.args.ccddata.header['SKYCOR']
# get telescope and atm. correction
if 'TELESCOP' in self.action.args.ccddata.header:
tel = self.action.args.ccddata.header['TELESCOP']
else:
tel = 'KeckI'
if 'Keck' in tel:
area = 760000.0
else:
area = -1.0
# compute good y pixel ranges
if w0 > 0. and dw > 0. and wgoo0 > 0. and wgoo1 > 0.:
z0 = int((wgoo0 - w0) / dw) + 10
z1 = int((wgoo1 - w0) / dw) - 10
# wavelength scale
w = w0 + z * dw
# good spatial range
gy0 = pad_y if pad_y > 1 else 1
gy1 = sz[1] - (pad_y if pad_y > 2 else 2)
# log results
self.logger.info("Invsen. Pars: Y0, Y1, Z0, Z1, Wav0, Wav1: "
"%d, %d, %d, %d, %.3f, %.3f" %
(gy0, gy1, z0, z1, w[z0], w[z1]))
# central wavelength
cwv = self.action.args.cwave
# find standard in slices
# sum over wavelength
tot = np.sum(self.action.args.ccddata.data[z0:z1, gy0:gy1, :], 0)
# yy = np.arange(gy1-gy0) + gy0
mxsl = -1.
mxsg = 0.
# for each slice
for i in range(sz[2]):
tstd = float(np.nanstd(tot[:, i]))
if tstd > mxsg:
mxsg = tstd
mxsl = i
# relevant slices
sl0 = (mxsl - 3) if mxsl >= 3 else 0
sl1 = (mxsl + 3) if (mxsl + 3) <= sz[2] - 1 else sz[2] - 1
# get y position of std
cy, _ = find_peaks(tot[:, mxsl], height=np.nanmean(tot[:, mxsl]))
cy = int(cy[0]) + gy0
# log results
self.logger.info("Std lices: max, sl0, sl1, spatial cntrd: "
"%d, %d, %d, %.2f" % (mxsl, sl0, sl1, cy))
# get dwave spectrum
ofn = self.action.args.ccddata.header['OFNAME']
delfn = ofn.split('.')[0] + '_dcubed.fits'
full_path = os.path.join(os.path.dirname(self.action.args.name),
self.config.instrument.output_directory,
delfn)
if os.path.exists(full_path):
dew = kcwi_fits_reader(full_path)[0]
dwspec = dew.data[:, cy, mxsl]
zeros = np.where(dwspec == 0)
if len(zeros) > 0:
dwspec[zeros] = dw
else:
dwspec = np.zeros(sz[0]) + dw
# copy of input cube
scub = self.action.args.ccddata.data.copy()
# sky window width in pixels
skywin = int(self.config.instrument.psfwid / self.action.args.xbinsize)
# do sky subtraction, if needed
if not skycor:
self.logger.warning("Sky should have been subtraced already")
# apply extinction correction
ucub = scub.copy() # uncorrected cube
kcwi_correct_extin(scub,
self.action.args.ccddata.header,
logger=self.logger)
# get slice spectra y limits
sy0 = (cy - skywin) if (cy - skywin) > 0 else 0
sy1 = (cy + skywin) if (cy + skywin) < (sz[1] - 1) else (sz[1] - 1)
# sum over y range
slspec = np.sum(scub[:, sy0:sy1, :], 1)
ulspec = np.sum(ucub[:, sy0:sy1, :], 1)
# sum over slices
obsspec = np.sum(slspec[:, sl0:sl1], 1)
ubsspec = np.sum(ulspec[:, sl0:sl1], 1)
# convert to e-/second
obsspec /= expt
ubsspec /= expt
# check for zeros
zeros = np.where(obsspec == 0.)
if len(zeros) > 0:
obsmean = np.nanmean(obsspec)
obsspec[zeros] = obsmean
# read in standard star spectrum
hdul = pf.open(self.action.args.stdfile)
swl = hdul[1].data['WAVELENGTH']
sflx = hdul[1].data['FLUX']
sfw = hdul[1].data['FWHM']
hdul.close()
# get region of interest
sroi = [i for i, v in enumerate(swl) if w[0] <= v <= w[-1]]
nsroi = len(sroi)
if nsroi <= 0:
self.logger.error("No standard wavelengths in common")
return self.action.args
# expand range after checking for edges
if sroi[0] > 0:
sroi.insert(0, sroi[0] - 1)
nsroi += 1
if sroi[-1] < (len(swl) - 1):
sroi.append(sroi[-1] + 1)
nsroi += 1
# very sparsely sampled w.r.t. object
if nsroi <= 1:
self.logger.error("Not enough standard points")
return self.action.args
self.logger.info("Number of standard point = %d" % nsroi)
# how many points?
# do_line = nsroi > 20
swl = swl[sroi]
sflx = sflx[sroi]
sfw = sfw[sroi]
fwhm = np.max(sfw)
self.logger.info("Reference spectrum FWHM used = %.1f (A)" % fwhm)
# resample standard onto our wavelength grid
rsint = interp1d(swl, sflx, kind='cubic', fill_value='extrapolate')
rsflx = rsint(w)
# get effective inverse sensitivity
invsen = rsflx / obsspec
# convert to photons/s/cm^2/(wl bin = dw)
rspho = 5.03411250e7 * rsflx * w * dw
# get effective area
earea = ubsspec / rspho
# correct to native bins
earea *= dw / dwspec
# Balmer lines
blines = [6563., 4861., 4341., 4102., 3970., 3889., 3835.]
# default values (for BM)
bwid = 0.008 # fractional width to mask
ford = 9 # fit order
if 'BL' in self.action.args.grating:
bwid = 0.004
ford = 7
elif 'BH' in self.action.args.grating:
bwid = 0.012
ford = 9
# Adjust for dichroic fraction
try:
dichroic_fraction = self.action.args.ccddata.header['DICHFRAC']
except KeyError:
dichroic_fraction = 1.
ford = int(ford * dichroic_fraction)
if ford < 3:
ford = 3
self.logger.info("Fitting Invsens with polynomial order %d" % ford)
bwids = [bl * bwid for bl in blines]
# fit inverse sensitivity and effective area
# get initial region of interest
wl_good = [i for i, v in enumerate(w) if wgoo0 <= v <= wgoo1]
nwl_good = len(wl_good)
if nwl_good <= 0:
self.logger.error("No good wavelengths to fit")
return self.action.args
wlm0 = wgoo0
wlm1 = wgoo1
# interactively set wavelength limits
if self.config.instrument.plot_level >= 1:
yran = [np.min(obsspec), np.max(obsspec)]
source = ColumnDataSource(data=dict(x=w, y=obsspec))
done = False
while not done:
p = figure(tooltips=[("x", "@x{0,0.0}"), ("y", "@y{0,0.0}")],
title=self.action.args.plotlabel + ' Obs Spec',
x_axis_label='Wave (A)',
y_axis_label='Intensity (e-)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.line('x', 'y', line_color='black', source=source)
p.line([wgoo0, wgoo0],
yran,
line_color='green',
legend_label='WAVGOOD')
p.line([wgoo1, wgoo1], yran, line_color='green')
p.line([wlm0, wlm0],
yran,
line_color='blue',
legend_label='LIMITS')
p.line([wlm1, wlm1], yran, line_color='blue')
p.line([cwv, cwv], yran, line_color='red', legend_label='CWAV')
set_plot_lims(p, xlim=[wall0, wall1], ylim=yran)
bokeh_plot(p, self.context.bokeh_session)
print("WL limits: %.1f - %.1f" % (wlm0, wlm1))
qstr = input("New? <float> <float>, <cr> - done: ")
if len(qstr) <= 0:
done = True
else:
try:
wlm0 = float(qstr.split()[0])
wlm1 = float(qstr.split()[1])
if wlm1 < wlm0 or wlm0 < wall0 or wlm1 > wall1:
wlm0 = wgoo0
wlm1 = wgoo1
print("range/order error, try again")
except (IndexError, ValueError):
wlm0 = wgoo0
wlm1 = wgoo1
print("format error, try again")
# update region of interest
wl_good = [i for i, v in enumerate(w) if wlm0 <= v <= wlm1]
nwl_good = len(wl_good)
# END: interactively set wavelength limits
# Now interactively identify lines
if self.config.instrument.plot_level >= 1:
yran = [np.min(obsspec), np.max(obsspec)]
# source = ColumnDataSource(data=dict(x=w, y=obsspec))
done = False
while not done:
p = figure(tooltips=[("x", "@x{0.0}"), ("y", "@y{0.0}")],
title=self.action.args.plotlabel + ' Obs Spec',
x_axis_label='Wave (A)',
y_axis_label='Intensity (e-)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.line(w, obsspec, line_color='black')
p.line([wgoo0, wgoo0],
yran,
line_color='green',
legend_label='WAVGOOD')
p.line([wgoo1, wgoo1], yran, line_color='green')
p.line([wlm0, wlm0],
yran,
line_color='blue',
legend_label='LIMITS')
p.line([wlm1, wlm1], yran, line_color='blue')
p.line([cwv, cwv], yran, line_color='red', legend_label='CWAV')
for il, bl in enumerate(blines):
if wall0 < bl < wall1:
p.line([bl, bl], yran, line_color='orange')
p.line([bl - bwids[il], bl - bwids[il]],
yran,
line_color='orange',
line_dash='dashed')
p.line([bl + bwids[il], bl + bwids[il]],
yran,
line_color='orange',
line_dash='dashed')
set_plot_lims(p, xlim=[wall0, wall1], ylim=yran)
bokeh_plot(p, self.context.bokeh_session)
qstr = input("New lines? <float> [<float>] ... (A), "
"<cr> - done: ")
if len(qstr) <= 0:
done = True
else:
for lstr in qstr.split():
try:
new_line = float(lstr)
except ValueError:
print("bad line: %s" % lstr)
continue
if wlm0 < new_line < wlm1:
blines.append(new_line)
bwids.append(bwid * new_line)
else:
print("line outside range: %s" % lstr)
# END: interactively identify lines
# set up fitting vectors, flux, waves, measure errors
sf = invsen[wl_good] # dependent variable
af = earea[wl_good] # effective area
wf = w[wl_good] # independent variable
bf = obsspec[wl_good] # input electrons
rf = rsflx[wl_good] # reference flux
mw = np.ones(nwl_good, dtype=float) # weights
use = np.ones(nwl_good, dtype=int) # toggles for usage
# loop over Balmer lines
for il, bl in enumerate(blines):
roi = [
i for i, v in enumerate(wf)
if (bl - bwids[il]) <= v <= (bl + bwids[il])
]
nroi = len(roi)
if nroi > 0:
use[roi] = 0
self.logger.info("Masking line at %.1f +- %.4f (A)" %
(bl, bwids[il]))
# ignore bad points by setting large errors
mf = []
ef = []
ww = []
used = []
not_used = []
for i in range(len(use)):
if use[i] == 1:
used.append(i)
else:
mw[i] = 1.e-9
mf.append(sf[i])
ef.append(100. * af[i] / area)
ww.append(wf[i])
not_used.append(i)
# initial polynomial fit of inverse sensitivity
wf0 = np.min(wf)
res = np.polyfit(wf - wf0, sf, deg=ford, w=mw)
finvsen = np.polyval(res, w - wf0)
sinvsen = np.polyval(res, wf - wf0)
calspec = obsspec * finvsen
scalspec = bf * sinvsen
# initial polynomial fit of effective area
res = np.polyfit(wf - wf0, af, ford, w=mw)
fearea = np.polyval(res, w - wf0)
# calculate residuals
resid = 100.0 * (scalspec - rf) / rf
if len(not_used) > 0:
rbad = resid[not_used]
else:
rbad = None
rsd_mean = float(np.nanmean(resid[used]))
rsd_stdv = float(np.nanstd(resid[used]))
self.logger.info("Calibration residuals = %f +- %f %%" %
(rsd_mean, rsd_stdv))
# plots
peff = None
pivs = None
pcal = None
prsd = None
# interactively adjust fit
if self.config.instrument.plot_level >= 1:
done = False
while not done:
yran = [np.min(100. * af / area), np.max(100. * af / area)]
effmax = np.nanmax(100. * fearea / area)
effmean = np.nanmean(100. * fearea / area)
peff = figure(title=self.action.args.plotlabel + ' Efficiency',
x_axis_label='Wave (A)',
y_axis_label='Effective Efficiency (%)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
peff.line(wf,
100. * af / area,
line_color='black',
legend_label='Data')
peff.line(w,
100. * fearea / area,
line_color='red',
legend_label='Fit')
peff.scatter(ww, ef, marker='x', legend_label='Rej')
peff.line([wlm0, wlm0],
yran,
line_color='green',
legend_label='WL lims')
peff.line([wlm1, wlm1], yran, line_color='green')
peff.line([wall0, wall1], [effmax, effmax],
line_color='black',
line_dash='dashed')
peff.line([wall0, wall1], [effmean, effmean],
line_color='black',
line_dash='dashdot')
set_plot_lims(peff, xlim=[wall0, wall1], ylim=yran)
bokeh_plot(peff, self.context.bokeh_session)
if self.config.instrument.plot_level >= 1:
input("Next? <cr>: ")
else:
time.sleep(2. * self.config.instrument.plot_pause)
yran = [np.min(sf), np.max(sf)]
pivs = figure(title=self.action.args.plotlabel +
' Inverse sensitivity',
x_axis_label='Wave (A)',
y_axis_label='Invserse Sensitivity (Flux/e-/s)',
y_axis_type='log',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
pivs.line(wf, sf, line_color='black', legend_label='Data')
pivs.line(w, finvsen, line_color='red', legend_label='Fit')
pivs.scatter(ww, mf, marker='x', legend_label='Rej')
pivs.line([wlm0, wlm0],
yran,
line_color='green',
legend_label='WL lims')
pivs.line([wlm1, wlm1], yran, line_color='green')
set_plot_lims(pivs, xlim=[wall0, wall1], ylim=yran)
bokeh_plot(pivs, self.context.bokeh_session)
if self.config.instrument.plot_level >= 1:
input("Next? <cr>: ")
else:
time.sleep(2. * self.config.instrument.plot_pause)
yran = [np.min(calspec[wl_good]), np.max(calspec[wl_good])]
pcal = figure(title=self.action.args.plotlabel + ' Calibrated',
x_axis_label='Wave (A)',
y_axis_label='Flux (ergs/s/cm^2/A)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
pcal.line(w, calspec, line_color='black', legend_label='Obs')
pcal.line(w, rsflx, line_color='red', legend_label='Ref')
pcal.line([wlm0, wlm0],
yran,
line_color='green',
legend_label='WL lims')
pcal.line([wlm1, wlm1], yran, line_color='green')
set_plot_lims(pcal, xlim=[wall0, wall1], ylim=yran)
bokeh_plot(pcal, self.context.bokeh_session)
if self.config.instrument.plot_level >= 1:
input("Next? <cr>: ")
else:
time.sleep(2. * self.config.instrument.plot_pause)
yran = [np.min(resid), np.max(resid)]
prsd = figure(title=self.action.args.plotlabel +
' Residuals = %.1f +- %.1f (%%)' %
(rsd_mean, rsd_stdv),
x_axis_label='Wave (A)',
y_axis_label='Obs - Ref / Ref (%)',
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
prsd.line(wf,
resid,
line_color='black',
legend_label='Obs - Ref (%)')
if len(not_used) > 0:
prsd.scatter(ww, rbad, marker='x', legend_label='Rej')
prsd.line([wlm0, wlm0],
yran,
line_color='green',
legend_label='WL lims')
prsd.line([wlm1, wlm1], yran, line_color='green')
prsd.line([wall0, wall1], [rsd_mean, rsd_mean],
line_color='red')
prsd.line([wall0, wall1],
[rsd_mean + rsd_stdv, rsd_mean + rsd_stdv],
line_color='black',
line_dash='dashed')
prsd.line([wall0, wall1],
[rsd_mean - rsd_stdv, rsd_mean - rsd_stdv],
line_color='black',
line_dash='dashed')
set_plot_lims(prsd, xlim=[wall0, wall1], ylim=yran)
bokeh_plot(prsd, self.context.bokeh_session)
if self.config.instrument.plot_level >= 1:
qstr = input("Current fit order = %d, "
"New fit order? <int>, <cr> - done: " % ford)
if len(qstr) <= 0:
done = True
else:
try:
ford = int(qstr)
# update fit of inverse sensitivity
res = np.polyfit(wf - wf0, sf, deg=ford, w=mw)
finvsen = np.polyval(res, w - wf0)
sinvsen = np.polyval(res, wf - wf0)
calspec = obsspec * finvsen
scalspec = bf * sinvsen
# update polynomial fit of effective area
res = np.polyfit(wf - wf0, af, ford, w=mw)
fearea = np.polyval(res, w - wf0)
# re-calculate residuals
resid = 100.0 * (scalspec - rf) / rf
if len(not_used) > 0:
rbad = resid[not_used]
else:
rbad = None
rsd_mean = float(np.nanmean(resid[used]))
rsd_stdv = float(np.nanstd(resid[used]))
self.logger.info(
"Calibration residuals = %f +- %f %%" %
(rsd_mean, rsd_stdv))
except ValueError:
print("Bad fit order, try again")
else:
done = True
time.sleep(2. * self.config.instrument.plot_pause)
# log results
effmax = float(np.nanmax(100. * fearea / area))
effmean = float(np.nanmean(100. * fearea / area))
self.logger.info("Peak, mean efficiency (%%): %.1f, %.1f" %
(effmax, effmean))
self.logger.info("Fit order = %d" % ford)
# output plots
pfname = "std_%05d_%s_%s_%s_%d" % (
self.action.args.ccddata.header['FRAMENO'], stdname,
self.action.args.grating.strip(),
self.action.args.ifuname.strip(), int(self.action.args.cwave))
# Save plots
save_plot(peff, filename=pfname + '_eff.png') # Efficiency
save_plot(pivs, filename=pfname + '_invsens.png') # Inv. Sens.
save_plot(prsd, filename=pfname + '_resid.png') # Residuals
save_plot(pcal, filename=pfname + '_cal.png') # Calibrated
log_string = MakeInvsens.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
self.logger.info(log_string)
# write out effective inverse sensitivity
# update inverse sensitivity header
hdr = self.action.args.ccddata.header.copy()
hdr['HISTORY'] = log_string
hdr['INVSENS'] = (True, 'effective inv. sens. spectrum?')
hdr['INVSW0'] = (w[z0], 'low wavelength for eff. inv. sens.')
hdr['INVSW1'] = (w[z1], 'high wavelength for eff. inv. sens.')
hdr['INVSZ0'] = (z0, 'low wave pixel for eff. inv. sens.')
hdr['INVSZ1'] = (z1, 'high wave pixel for eff. inv. sens.')
hdr['INVSY0'] = (gy0, 'low spatial pixel for eff. inv. sens.')
hdr['INVSY1'] = (gy1, 'high spatial pixel for eff. inv. sens.')
hdr['INVSLMX'] = (mxsl, 'brightest std star slice')
hdr['INVSL0'] = (sl0, 'lowest std star slice summed')
hdr['INVSL1'] = (sl1, 'highest std star slice summed')
hdr['INVSLY'] = (cy, 'spatial pixel position of std within slice')
hdr['INVFW0'] = (wlm0, 'low wavelength for fits')
hdr['INVFW1'] = (wlm1, 'high wavelength for fits')
hdr['INVFORD'] = (ford, 'fit order')
hdr['EXPTIME'] = (1., 'effective exposure time (seconds)')
hdr['XPOSURE'] = (1., 'effective exposure time (seconds)')
# remove old WCS
del hdr['RADESYS']
del hdr['EQUINOX']
del hdr['LONPOLE']
del hdr['LATPOLE']
del hdr['NAXIS2']
if 'NAXIS3' in hdr:
del hdr['NAXIS3']
del hdr['CTYPE1']
del hdr['CTYPE2']
del hdr['CTYPE3']
del hdr['CUNIT1']
del hdr['CUNIT2']
del hdr['CUNIT3']
del hdr['CNAME1']
del hdr['CNAME2']
del hdr['CNAME3']
del hdr['CRVAL1']
del hdr['CRVAL2']
del hdr['CRVAL3']
del hdr['CRPIX1']
del hdr['CRPIX2']
del hdr['CRPIX3']
del hdr['CD1_1']
del hdr['CD1_2']
del hdr['CD2_1']
del hdr['CD2_2']
del hdr['CD3_3']
# set wavelength axis WCS values
hdr['WCSDIM'] = 1
hdr['CTYPE1'] = ('AWAV', 'Air Wavelengths')
hdr['CUNIT1'] = ('Angstrom', 'Wavelength units')
hdr['CNAME1'] = ('KCWI INVSENS Wavelength', 'Wavelength name')
hdr['CRVAL1'] = (w0, 'Wavelength zeropoint')
hdr['CRPIX1'] = (crpixw, 'Wavelength reference pixel')
hdr['CDELT1'] = (dw, 'Wavelength Angstroms per pixel')
ofn = self.action.args.ccddata.header['OFNAME']
invsname = ofn.split('.fits')[0] + '_' + suffix + '.fits'
eaname = ofn.split('.fits')[0] + '_ea.fits'
# set units
invsens_u = u.erg / (u.angstrom * u.cm**2 * u.s * u.electron)
# output inverse sensitivity
out_invsens = CCDData(np.asarray([invsen, finvsen, obsspec]),
meta=hdr,
unit=invsens_u)
kcwi_fits_writer(out_invsens,
output_file=invsname,
output_dir=self.config.instrument.output_directory)
self.context.proctab.update_proctab(frame=out_invsens,
suffix=suffix,
newtype='INVSENS')
# output effective area
ea_u = u.cm**2 / u.angstrom
out_ea = CCDData(np.asarray([earea, fearea]), meta=hdr, unit=ea_u)
kcwi_fits_writer(out_ea,
output_file=eaname,
output_dir=self.config.instrument.output_directory)
self.context.proctab.update_proctab(frame=out_ea,
suffix='ea',
newtype='EAREA')
return self.action.args
def _perform(self):
"""Get atlas line positions for wavelength fitting"""
self.logger.info("Finding isolated atlas lines")
# get atlas wavelength range
# get pixel values (no longer centered in the middle)
specsz = len(self.context.arcs[self.config.instrument.REFBAR])
xvals = np.arange(0, specsz)
# min, max rows, trimming the ends
minrow = 50
maxrow = specsz - 50
# wavelength range
mnwvs = []
mxwvs = []
refbar_disp = 1.
# Get wavelengths for each bar
for b in range(self.config.instrument.NBARS):
waves = np.polyval(self.action.args.twkcoeff[b], xvals)
mnwvs.append(np.min(waves))
mxwvs.append(np.max(waves))
if b == self.config.instrument.REFBAR:
refbar_disp = self.action.args.twkcoeff[b][-2]
self.logger.info("Ref bar (%d) dispersion = %.3f Ang/px" %
(self.config.instrument.REFBAR, refbar_disp))
# Get extrema (trim ends a bit)
minwav = min(mnwvs) + 10.
maxwav = max(mxwvs) - 10.
wave_range = maxwav - minwav
# Do we have a dichroic?
if self.action.args.dich:
if self.action.args.camera == 0: # Blue
maxwav = min([maxwav, 5620.])
elif self.action.args.camera == 1: # Red
minwav = max([minwav, 5580.])
else:
self.logger.error("Camera keyword not defined!")
dichroic_fraction = (maxwav - minwav) / wave_range
# Get corresponding atlas range
minrw = [
i for i, v in enumerate(self.action.args.refwave) if v >= minwav
][0]
maxrw = [
i for i, v in enumerate(self.action.args.refwave) if v <= maxwav
][-1]
self.logger.info("Min, Max wave (A): %.2f, %.2f" % (minwav, maxwav))
if self.action.args.dich:
self.logger.info("Dichroic fraction: %.3f" % dichroic_fraction)
# store atlas ranges
self.action.args.atminrow = minrw
self.action.args.atmaxrow = maxrw
self.action.args.atminwave = minwav
self.action.args.atmaxwave = maxwav
self.action.args.dichroic_fraction = dichroic_fraction
# get atlas sub spectrum
atspec = self.action.args.reflux[minrw:maxrw]
atwave = self.action.args.refwave[minrw:maxrw]
# get reference bar arc spectrum, pixel values, and prelim wavelengths
subxvals = xvals[minrow:maxrow]
subyvals = self.context.arcs[
self.config.instrument.REFBAR][minrow:maxrow].copy()
subwvals = np.polyval(
self.action.args.twkcoeff[self.config.instrument.REFBAR], subxvals)
# smooth subyvals
win = boxcar(3)
subyvals = sp.signal.convolve(subyvals, win, mode='same') / sum(win)
# find good peaks in arc spectrum
smooth_width = 4 # in pixels
# peak width
peak_width = int(self.action.args.atsig / abs(refbar_disp))
if peak_width < 4:
peak_width = 4
# slope threshold
slope_thresh = 0.7 * smooth_width / 2. / 100.
# slope_thresh = 0.7 * smooth_width / 1000. # more severe for arc
# slope_thresh = 0.016 / peak_width
# get amplitude threshold
ampl_thresh = 0.
self.logger.info("Using a peak_width of %d px, a slope_thresh of %.5f "
"a smooth_width of %d and an ampl_thresh of %.3f" %
(peak_width, slope_thresh, smooth_width, ampl_thresh))
arc_cent, avwsg, arc_hgt = findpeaks(subwvals, subyvals, smooth_width,
slope_thresh, ampl_thresh,
peak_width)
avwfwhm = avwsg * 2.354
self.logger.info("Found %d lines with <sig> = %.3f (A),"
" <FWHM> = %.3f (A)" %
(len(arc_cent), avwsg, avwfwhm))
# fitting window based on grating type
if 'H' in self.action.args.grating or 'M' in self.action.args.grating:
fwid = avwfwhm
else:
fwid = avwsg
# clean near neighbors
spec_cent = arc_cent
spec_hgt = arc_hgt
#
# generate an atlas line list
refws = [] # atlas line wavelength
refas = [] # atlas line amplitude
rej_fit_w = [] # fit rejected atlas line wavelength
rej_fit_y = [] # fit rejected atlas line amplitude
rej_par_w = [] # par rejected atlas line wavelength
rej_par_a = [] # par rejected atlas line amplitude
nrej = 0
# look at each arc spectrum line
for i, pk in enumerate(spec_cent):
if pk <= minwav or pk >= maxwav:
continue
# get atlas pixel position corresponding to arc line
try:
line_x = [ii for ii, v in enumerate(atwave) if v >= pk][0]
# get window around atlas line to fit
minow, maxow, count = get_line_window(atspec, line_x)
except IndexError:
count = 0
minow = None
maxow = None
self.logger.warning("line at edge: %d, %.2f, %.f2f" %
(i, pk, max(atwave)))
# is resulting window large enough for fitting?
if count < 5 or not minow or not maxow:
# keep track of fit rejected lines
rej_fit_w.append(pk)
rej_fit_y.append(spec_hgt[i])
nrej += 1
self.logger.info("Atlas window rejected for line %.3f" % pk)
continue
# get data to fit
yvec = atspec[minow:maxow + 1]
xvec = atwave[minow:maxow + 1]
# attempt Gaussian fit
try:
fit, _ = curve_fit(gaus, xvec, yvec, p0=[spec_hgt[i], pk, 1.])
except RuntimeError:
# keep track of Gaussian fit rejected lines
rej_fit_w.append(pk)
rej_fit_y.append(spec_hgt[i])
nrej += 1
self.logger.info("Atlas Gaussian fit rejected for line %.3f" %
pk)
continue
# get interpolation function of atlas line
int_line = interpolate.interp1d(xvec,
yvec,
kind='cubic',
bounds_error=False,
fill_value='extrapolate')
# use very dense pixel sampling
x_dense = np.linspace(min(xvec), max(xvec), num=1000)
# resample line with dense sampling
y_dense = int_line(x_dense)
# get peak amplitude and wavelength
pki = y_dense.argmax()
pkw = x_dense[pki]
# calculate some diagnostic parameters for the line
# how many atlas pixels have we moved?
xoff = abs(pkw - fit[1]) / self.action.args.refdisp
# what is the wavelength offset in Angstroms?
woff = abs(pkw - pk)
# what fraction of the canonical fit width is the line?
wrat = abs(fit[2]) / fwid # can be neg or pos
# current criteria for these diagnostic parameters
if woff > 5. or xoff > 1.5 or wrat > 1.1:
# keep track of par rejected atlas lines
rej_par_w.append(pkw)
rej_par_a.append(y_dense[pki])
nrej += 1
self.logger.info(
"Atlas line parameters rejected for line %.3f" % pk)
self.logger.info("woff = %.3f, xoff = %.2f, wrat = %.3f" %
(woff, xoff, wrat))
continue
refws.append(pkw)
refas.append(y_dense[pki])
# eliminate faintest lines if we have a large number
self.logger.info("number of remaining lines: %d" % len(refas))
if len(refas) > 400:
# sort on flux
sf = np.argsort(refas)
refws = np.asarray(refws)[sf]
refas = np.asarray(refas)[sf]
# remove faintest two-thirds
hlim = int(len(refas) * 0.67)
refws = refws[hlim:]
refas = refas[hlim:]
# sort back onto wavelength
sw = np.argsort(refws)
refws = refws[sw].tolist()
refas = refas[sw].tolist()
# check if line list was given on command line
if self.config.instrument.LINELIST:
with open(self.config.instrument.LINELIST) as llfn:
atlines = llfn.readlines()
refws = []
refas = []
for line in atlines:
if '#' in line:
continue
refws.append(float(line.split()[0]))
refas.append(float(line.split()[1]))
self.logger.info("Read %d lines from %s" %
(len(refws), self.config.instrument.LINELIST))
else:
self.logger.info("Using %d generated lines" % len(refws))
# store wavelengths, fluxes
self.action.args.at_wave = refws
self.action.args.at_flux = refas
# output filename stub
atfnam = "arc_%05d_%s_%s_%s_atlines" % \
(self.action.args.ccddata.header['FRAMENO'],
self.action.args.illum, self.action.args.grating,
self.action.args.ifuname)
# output directory
output_dir = os.path.join(self.config.instrument.cwd,
self.config.instrument.output_directory)
# write out final atlas line list
atlines = np.array([refws, refas])
atlines = atlines.T
with open(os.path.join(output_dir, atfnam + '.txt'), 'w') as atlfn:
np.savetxt(atlfn, atlines, fmt=['%12.3f', '%12.3f'])
# plot final list of Atlas lines and show rejections
norm_fac = np.nanmax(atspec)
if self.config.instrument.plot_level >= 1:
p = figure(title=self.action.args.plotlabel +
"ATLAS LINES Ngood = %d, Nrej = %d" %
(len(refws), nrej),
x_axis_label="Wavelength (A)",
y_axis_label="Normalized Flux",
plot_width=self.config.instrument.plot_width,
plot_height=self.config.instrument.plot_height)
p.line(subwvals,
subyvals / np.nanmax(subyvals),
legend_label='RefArc',
color='lightgray')
p.line(atwave,
atspec / norm_fac,
legend_label='Atlas',
color='blue')
# Rejected: nearby neighbor
# p.diamond(rej_neigh_w, rej_neigh_y / norm_fac,
# legend_label='NeighRej', color='cyan', size=8)
# Rejected: fit failure
p.diamond(rej_fit_w,
rej_fit_y / norm_fac,
legend_label='FitRej',
color='red',
size=8)
# Rejected: line parameter outside range
p.diamond(rej_par_w,
rej_par_a / norm_fac,
legend_label='ParRej',
color='orange',
size=8)
p.diamond(refws,
refas / norm_fac,
legend_label='Kept',
color='green',
size=10)
p.line([minwav, minwav], [-0.1, 1.1],
legend_label='WavLim',
color='brown')
p.line([maxwav, maxwav], [-0.1, 1.1], color='brown')
p.x_range = Range1d(min([min(subwvals), minwav - 10.]),
max(subwvals))
p.y_range = Range1d(-0.04, 1.04)
bokeh_plot(p, self.context.bokeh_session)
if self.config.instrument.plot_level >= 2:
input("Next? <cr>: ")
else:
time.sleep(self.config.instrument.plot_pause)
save_plot(p, filename=atfnam + ".png")
self.logger.info("Final atlas list has %d lines" % len(refws))
log_string = GetAtlasLines.__module__
self.action.args.ccddata.header['HISTORY'] = log_string
self.logger.info(log_string)
return self.action.args