def locate_object_file(self, suffix): ofn = self.action.args.name objfn = strip_fname(ofn) + f'_{suffix}.fits' full_path = os.path.join(self.config.instrument.cwd, self.config.instrument.output_directory, objfn) if os.path.exists(full_path): return kcwi_fits_reader(full_path)[0] else: self.logger.error(f'Unable to read file {objfn}') return None
def _pre_condition(self): contbars_in_proctable = self.context.proctab.search_proctab( frame=self.action.args.ccddata, target_type='CONTBARS', nearest=True) self.logger.info("%d continuum bars frames found" % len(contbars_in_proctable)) if len(contbars_in_proctable) > 0: self.action.args.original_filename = strip_fname( contbars_in_proctable['filename'][0])+ "_trace.fits" return True else: self.action.args.original_filename = None return False
def _pre_condition(self): """Checks if object is a standard star""" self.logger.info("Checking precondition for MakeInvsens") stdfile = None stdname = None if 'object' in self.action.args.imtype.lower(): self.logger.info("Checking OBJECT keyword") stdfile, stdname = kcwi_get_std( self.action.args.ccddata.header['OBJECT'], self.logger) if not stdfile: self.logger.info("Checking TARGNAME keyword") stdfile, stdname = kcwi_get_std( self.action.args.ccddata.header['TARGNAME'], self.logger) else: self.logger.warning("Not object type: %s" % self.action.args.imtype) self.action.args.stdfile = stdfile self.action.args.stdname = stdname # have we been processed correctly? if 'DARCOR' in self.action.args.ccddata.header: if self.action.args.ccddata.header['DARCOR']: # check pre condition if self.action.args.stdfile is not None: # does file already exist? ofn = self.action.args.name msname = strip_fname(ofn) + '_invsens.fits' rdir = self.config.instrument.output_directory invsensf = os.path.join(self.config.instrument.cwd, rdir, msname) if os.path.exists(invsensf): self.logger.warning("Master cal already exists: %s" % invsensf) return False else: self.logger.info("Master cal will be generated.") return True else: self.logger.warning("Not a KCWI standard observation.") return False else: self.logger.warning("DAR not corrected, cannot generate " "inverse sensitivity") else: self.logger.warning("Not processed enough to generate " "inverse sensitivity") return False
def _perform(self): """ Returns an Argument() with the parameters that depends on this operation """ args = self.action.args method = 'average' suffix = args.new_type.lower() combine_list = list(self.combine_list['filename']) # get master dark output name mdname = strip_fname(combine_list[0]) + '_' + suffix + '.fits' stack = [] stackf = [] for dark in combine_list: # get dark intensity (int) image file name in redux directory stackf.append(dark.split('.fits')[0] + '_int.fits') darkfn = os.path.join(args.in_directory, stackf[-1]) # using [0] gets just the image data stack.append(kcwi_fits_reader(darkfn)[0]) stacked = ccdproc.combine(stack, method=method, sigma_clip=True, sigma_clip_low_thresh=None, sigma_clip_high_thresh=2.0) stacked.unit = stack[0].unit stacked.header.IMTYPE = args.new_type stacked.header['NSTACK'] = (len(combine_list), 'number of images stacked') stacked.header['STCKMETH'] = (method, 'method used for stacking') for ii, fname in enumerate(stackf): stacked.header['STACKF%d' % (ii + 1)] = (fname, "stack input file") log_string = MakeMasterDark.__module__ stacked.header['HISTORY'] = log_string self.logger.info(log_string) kcwi_fits_writer(stacked, output_file=mdname, output_dir=self.config.instrument.output_directory) self.context.proctab.update_proctab(frame=stacked, suffix=suffix, newtype=args.new_type, filename=self.action.args.name) self.context.proctab.write_proctab() return self.action.args
def _perform(self): """ Returns an Argument() with the parameters that depends on this operation """ self.logger.info("Creating master illumination correction") suffix = self.action.args.new_type.lower() insuff = self.action.args.stack_type.lower() stack_list = list(self.stack_list['filename']) if len(stack_list) <= 0: self.logger.warning("No flats found!") return self.action.args # get root for maps tab = self.context.proctab.search_proctab( frame=self.action.args.ccddata, target_type='ARCLAMP', target_group=self.action.args.groupid) if len(tab) <= 0: self.logger.error("Geometry not solved!") return self.action.args mroot = strip_fname(tab['filename'][-1]) # Wavelength map image wmf = mroot + '_wavemap.fits' self.logger.info("Reading image: %s" % wmf) wavemap = kcwi_fits_reader( os.path.join(self.config.instrument.cwd, 'redux', wmf))[0] # Slice map image slf = mroot + '_slicemap.fits' self.logger.info("Reading image: %s" % slf) slicemap = kcwi_fits_reader(os.path.join( self.config.instrument.cwd, 'redux', slf))[0] # Position map image pof = mroot + '_posmap.fits' self.logger.info("Reading image: %s" % pof) posmap = kcwi_fits_reader(os.path.join( self.config.instrument.cwd, 'redux', pof))[0] # Read in stacked flat image stname = strip_fname(stack_list[0]) + '_' + insuff + '.fits' self.logger.info("Reading image: %s" % stname) stacked = kcwi_fits_reader(os.path.join( self.config.instrument.cwd, 'redux', stname))[0] # get type of flat internal = ('SFLAT' in stacked.header['IMTYPE']) twiflat = ('STWIF' in stacked.header['IMTYPE']) domeflat = ('SDOME' in stacked.header['IMTYPE']) if internal: self.logger.info("Internal Flat") elif twiflat: self.logger.info("Twilight Flat") elif domeflat: self.logger.info("Dome Flat") else: self.logger.error("Flat of Unknown Type!") return self.action.args # knots per pixel knotspp = self.config.instrument.KNOTSPP # get image size ny = stacked.header['NAXIS2'] # get binning xbin = self.action.args.xbinsize # Parameters for fitting # vignetted slice position range fitl = int(4/xbin) fitr = int(24/xbin) # un-vignetted slice position range flatl = int(34/xbin) flatr = int(72/xbin) # flat fitting slice position range ffleft = int(10/xbin) ffright = int(70/xbin) nrefx = int(ffright - ffleft) buffer = 6.0/float(xbin) # reference slice refslice = 9 allidx = np.arange(int(140/xbin)) newflat = stacked.data.copy() # dichroic fraction try: dichroic_fraction = wavemap.header['DICHFRAC'] except KeyError: dichroic_fraction = 1. # get reference slice data q = [i for i, v in enumerate(slicemap.data.flat) if v == refslice] # get wavelength limits waves = wavemap.data.compress((wavemap.data > 0.).flat) waves = [waves.min(), waves.max()] self.logger.info("Wavelength limits: %.1f - %1.f" % (waves[0], waves[1])) # correct vignetting if we are using internal flats if internal: self.logger.info("Internal flats require vignetting correction") # get good region for fitting if self.action.args.camera == 0: # Blue wmin = waves[0] wmax = min([waves[1], 5620.]) elif self.action.args.camera == 1: # Red wmin = max([waves[0], 5580.]) wmax = waves[1] else: self.logger.warning("Camera keyword not defined") wmin = waves[0] wmax = waves[1] dw = (wmax - wmin) / 30.0 wavemin = (wmin+wmax) / 2.0 - dw wavemax = (wmin+wmax) / 2.0 + dw self.logger.info("Using %.1f - %.1f A of slice %d" % (wavemin, wavemax, refslice)) xflat = [] yflat = [] wflat = [] qq = [] for i in q: if wavemin < wavemap.data.flat[i] < wavemax: xflat.append(posmap.data.flat[i]) yflat.append(stacked.data.flat[i]) wflat.append(wavemap.data.flat[i]) qq.append(i) # get un-vignetted portion qflat = [i for i, v in enumerate(xflat) if flatl <= v <= flatr] xflat = [xflat[i] for i in qflat] yflat = [yflat[i] for i in qflat] wflat = [wflat[i] for i in qflat] # sort on wavelength sw = np.argsort(wflat) ywflat = [yflat[i] for i in sw] wwflat = [wflat[i] for i in sw] ww0 = np.min(wwflat) # fit wavelength slope wavelinfit = np.polyfit(wwflat-ww0, ywflat, 2) wslfit = np.polyval(wavelinfit, wflat-ww0) # plot slope fit if self.config.instrument.plot_level >= 1: p = figure(title=self.action.args.plotlabel + ' WAVE SLOPE FIT', x_axis_label='wave px', y_axis_label='counts', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.circle(wwflat, ywflat, legend_label="Data") p.line(wflat, wslfit, line_color='red', line_width=3, legend_label="Fit") bokeh_plot(p, self.context.bokeh_session) if self.config.instrument.plot_level >= 2: input("Next? <cr>: ") else: time.sleep(self.config.instrument.plot_pause) # take out slope yflat = yflat / wslfit # now sort on slice position ss = np.argsort(xflat) xflat = [xflat[i] for i in ss] yflat = [yflat[i] for i in ss] # fit un-vignetted slope resflat = np.polyfit(xflat, yflat, 1) # select the points we will fit for the vignetting # get reference region xfit = [posmap.data.flat[i] for i in qq] yfit = [stacked.data.flat[i] for i in qq] wflat = [wavemap.data.flat[i] for i in qq] # take out wavelength slope yfit = yfit / np.polyval(wavelinfit, wflat-ww0) # select the vignetted region qfit = [i for i, v in enumerate(xfit) if fitl <= v <= fitr] xfit = [xfit[i] for i in qfit] yfit = [yfit[i] for i in qfit] # sort on slice position s = np.argsort(xfit) xfit = [xfit[i] for i in s] yfit = [yfit[i] for i in s] # fit vignetted slope resfit = np.polyfit(xfit, yfit, 1) # corrected data ycdata = stacked.data.flat[qq] / \ np.polyval(wavelinfit, wavemap.data.flat[qq]-ww0) ycmin = 0.5 # np.min(ycdata) ycmax = 1.25 # np.max(ycdata) # compute the intersection xinter = -(resflat[1] - resfit[1]) / (resflat[0] - resfit[0]) # plot slice profile and fits if self.config.instrument.plot_level >= 1: p = figure(title=self.action.args.plotlabel + ' Vignetting', x_axis_label='Slice Pos (px)', y_axis_label='Ratio', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.circle(posmap.data.flat[qq], ycdata, legend_label='Data') p.line(allidx, resfit[1] + resfit[0]*allidx, line_color='purple', legend_label='Vign.') p.line(allidx, resflat[1] + resflat[0]*allidx, line_color='red', legend_label='UnVign.') p.line([fitl, fitl], [ycmin, ycmax], line_color='blue') p.line([fitr, fitr], [ycmin, ycmax], line_color='blue') p.line([flatl, flatl], [ycmin, ycmax], line_color='green') p.line([flatr, flatr], [ycmin, ycmax], line_color='green') p.line([xinter-buffer, xinter-buffer], [ycmin, ycmax], line_color='black') p.line([xinter + buffer, xinter + buffer], [ycmin, ycmax], line_color='black') p.line([xinter, xinter], [ycmin, ycmax], line_color='red') p.y_range = Range1d(ycmin, ycmax) bokeh_plot(p, self.context.bokeh_session) if self.config.instrument.plot_level >= 2: input("Next? <cr>: ") else: time.sleep(self.config.instrument.plot_pause) # figure out where the correction applies qcor = [i for i, v in enumerate(posmap.data.flat) if 0 <= v <= (xinter-buffer)] # apply the correction! self.logger.info("Applying vignetting correction...") for i in qcor: newflat.flat[i] = (resflat[1]+resflat[0]*posmap.data.flat[i]) \ / (resfit[1]+resfit[0]*posmap.data.flat[i]) * \ stacked.data.flat[i] # now deal with the intermediate (buffer) region self.logger.info("Done, now handling buffer region") # get buffer points to fit in reference region qbff = [i for i in qq if (xinter-buffer) <= posmap.data.flat[i] <= (xinter+buffer)] # get slice pos and data for buffer fitting xbuff = [posmap.data.flat[i] for i in qbff] ybuff = [stacked.data.flat[i] / np.polyval(wavelinfit, wavemap.data.flat[i]-ww0) for i in qbff] # sort on slice position ssp = np.argsort(xbuff) xbuff = [xbuff[i] for i in ssp] ybuff = [ybuff[i] for i in ssp] # fit buffer with low-order poly buffit = np.polyfit(xbuff, ybuff, 3) # plot buffer fit if self.config.instrument.plot_level >= 1: p = figure(title=self.action.args.plotlabel + ' Buffer Region', x_axis_label='Slice Pos (px)', y_axis_label='Ratio', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.circle(xbuff, ybuff) p.line(xbuff, np.polyval(buffit, xbuff), line_color='red') bokeh_plot(p, self.context.bokeh_session) if self.config.instrument.plot_level >= 2: input("Next? <cr>: ") else: time.sleep(self.config.instrument.plot_pause) # get all buffer points in image qbuf = [i for i, v in enumerate(posmap.data.flat) if (xinter-buffer) <= v <= (xinter+buffer)] # apply buffer correction to all buffer points in newflat for i in qbuf: newflat.flat[i] = \ (resflat[1] + resflat[0] * posmap.data.flat[i]) / \ np.polyval(buffit, posmap.data.flat[i]) * newflat.flat[i] self.logger.info("Vignetting correction complete.") self.logger.info("Fitting master illumination") # now fit master flat # get reference slice points qref = [i for i in q if ffleft <= posmap.data.flat[i] <= ffright] xfr = wavemap.data.flat[qref] yfr = newflat.flat[qref] # sort on wavelength s = np.argsort(xfr) xfr = xfr[s] yfr = yfr[s] wavegood0 = wavemap.header['WAVGOOD0'] wavegood1 = wavemap.header['WAVGOOD1'] # correction for BM where we see a ledge if 'BM' in self.action.args.grating: ledge_wave = bm_ledge_position(self.action.args.cwave) self.logger.info("BM ledge calculated wavelength " "for ref slice = %.2f (A)" % ledge_wave) if wavegood0 <= ledge_wave <= wavegood1: self.logger.info("BM grating requires correction") qledge = [i for i, v in enumerate(xfr) if ledge_wave-25 <= v <= ledge_wave+25] xledge = [xfr[i] for i in qledge] yledge = [yfr[i] for i in qledge] s = np.argsort(xledge) xledge = [xledge[i] for i in s] yledge = [yledge[i] for i in s] win = boxcar(250) smyledge = sp.signal.convolve(yledge, win, mode='same') / sum(win) ylmax = np.max(yledge) ylmin = np.min(yledge) fpoints = np.arange(0, 100) / 100. * 50 + (ledge_wave-25) ledgefit, ledgemsk = Bspline.iterfit(np.asarray(xledge), smyledge, fullbkpt=fpoints, upper=1, lower=1) ylfit, _ = ledgefit.value(np.asarray(fpoints)) deriv = -(np.roll(ylfit, 1) - np.roll(ylfit, -1)) / 2.0 # trim edges trm = int(len(deriv)/5) deriv = deriv[trm:-trm] xvals = fpoints[trm:-trm] peaks, _ = find_peaks(deriv, height=100) if len(peaks) != 1: self.logger.warning("Extra peak found!") p = figure(title=self.action.args.plotlabel + ' Ledge', x_axis_label='Wavelength (A)', y_axis_label='Value', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.circle(xledge, smyledge, fill_color='green') p.line(fpoints, ylfit) bokeh_plot(p, self.context.bokeh_session) input("Next? <cr>: ") p = figure(title=self.action.args.plotlabel + ' Deriv', x_axis_label='px', y_axis_label='Value', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) xx = list(range(len(deriv))) ylim = get_plot_lims(deriv) p.circle(xx, deriv) for pk in peaks: p.line([pk, pk], ylim) bokeh_plot(p, self.context.bokeh_session) print("Please indicate the integer pixel value of the peak") ipk = int(input("Peak? <int>: ")) else: ipk = peaks[0] apk = xvals[ipk] if self.config.instrument.plot_level >= 3: p = figure( title=self.action.args.plotlabel + ' Peak of ledge', x_axis_label='Wave (A)', y_axis_label='Value', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.circle(xvals, deriv, legend_label='Data') p.line([apk, apk], [-50, 200], line_color='red', legend_label='Pk') bokeh_plot(p, self.context.bokeh_session) if self.config.instrument.plot_level >= 2: input("Next? <cr>: ") else: time.sleep(self.config.instrument.plot_pause) xlow = apk - 3 - 5 xhi = apk - 3 zlow = apk + 3 zhi = apk + 3 + 5 qlow = [i for i, v in enumerate(fpoints) if xlow <= v <= xhi] xlf = np.asarray([fpoints[i] for i in qlow]) ylf = np.asarray([ylfit[i] for i in qlow]) lowfit = np.polyfit(xlf, ylf, 1) qhi = [i for i, v in enumerate(fpoints) if zlow <= v <= zhi] xlf = np.asarray([fpoints[i] for i in qhi]) ylf = np.asarray([ylfit[i] for i in qhi]) hifit = np.polyfit(xlf, ylf, 1) ratio = (hifit[1] + hifit[0] * apk) / \ (lowfit[1] + lowfit[0] * apk) self.logger.info("BM ledge ratio: %.3f" % ratio) # correct flat data qcorr = [i for i, v in enumerate(xfr) if v >= apk] for i in qcorr: yfr[i] /= ratio # plot BM ledge if self.config.instrument.plot_level >= 1: p = figure( title=self.action.args.plotlabel + ' BM Ledge Region', x_axis_label='Wave (A)', y_axis_label='Value', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) # Input data p.circle(xledge, yledge, fill_color='blue', legend_label='Data') # correct input data qcorrect = [i for i, v in enumerate(xledge) if v >= apk] xplt = [] yplt = [] for i in qcorrect: xplt.append(xledge[i]) yplt.append(yledge[i] / ratio) p.circle(xplt, yplt, fill_color='orange', legend_label='Corrected') p.line(fpoints, ylfit, line_color='red', legend_label='Fit') p.line([xlow, xlow], [ylmin, ylmax], line_color='blue') p.line([xhi, xhi], [ylmin, ylmax], line_color='blue') p.line([zlow, zlow], [ylmin, ylmax], line_color='black') p.line([zhi, zhi], [ylmin, ylmax], line_color='black') p.line(fpoints, lowfit[1] + lowfit[0] * fpoints, line_color='purple') p.line(fpoints, hifit[1] + hifit[0] * fpoints, line_color='green') p.line([apk, apk], [ylmin, ylmax], line_color='green', legend_label='Pk') p.y_range = Range1d(ylmin, ylmax) p.legend.location = 'top_left' bokeh_plot(p, self.context.bokeh_session) if self.config.instrument.plot_level >= 2: input("Next? <cr>: ") else: time.sleep(self.config.instrument.plot_pause) # END: handling BM grating ledge # if we are fitting a twilight flat, treat it like a sky image with a # larger number of knots if twiflat: knots = int(ny * knotspp) else: knots = 100 self.logger.info("Using %d knots for bspline fit" % knots) # generate a fit from ref slice points bkpt = np.min(xfr) + np.arange(knots+1) * \ (np.max(xfr) - np.min(xfr)) / knots sftr, _ = Bspline.iterfit(xfr[nrefx:-nrefx], yfr[nrefx:-nrefx], fullbkpt=bkpt) yfitr, _ = sftr.value(xfr) # generate a blue slice spectrum bspline fit blueslice = 12 blueleft = 60 / xbin blueright = 80 / xbin qb = [i for i, v in enumerate(slicemap.data.flat) if v == blueslice] qblue = [i for i in qb if blueleft <= posmap.data.flat[i] <= blueright] xfb = wavemap.data.flat[qblue] yfb = newflat.flat[qblue] s = np.argsort(xfb) xfb = xfb[s] yfb = yfb[s] bkpt = np.min(xfb) + np.arange(knots+1) * \ (np.max(xfb) - np.min(xfb)) / knots sftb, _ = Bspline.iterfit(xfb[nrefx:-nrefx], yfb[nrefx:-nrefx], fullbkpt=bkpt) yfitb, _ = sftb.value(xfb) # generate a red slice spectrum bspline fit redslice = 23 redleft = 60 / xbin redright = 80 / xbin qr = [i for i, v in enumerate(slicemap.data.flat) if v == redslice] qred = [i for i in qr if redleft <= posmap.data.flat[i] <= redright] xfd = wavemap.data.flat[qred] yfd = newflat.flat[qred] s = np.argsort(xfd) xfd = xfd[s] yfd = yfd[s] bkpt = np.min(xfd) + np.arange(knots + 1) * \ (np.max(xfd) - np.min(xfd)) / knots sftd, _ = Bspline.iterfit(xfd[nrefx:-nrefx], yfd[nrefx:-nrefx], fullbkpt=bkpt) yfitd, _ = sftd.value(xfd) # waves minwave = np.min(xfb) maxwave = np.max(xfd) # are we a twilight flat? if twiflat: nwaves = int(ny * knotspp) else: nwaves = 1000 waves = minwave + (maxwave - minwave) * np.arange(nwaves+1) / nwaves if self.config.instrument.plot_level >= 1: # output filename stub rbfnam = "redblue_%05d_%s_%s_%s" % \ (self.action.args.ccddata.header['FRAMENO'], self.action.args.illum, self.action.args.grating, self.action.args.ifuname) if xbin == 1: stride = int(len(xfr) / 8000.) if stride <= 0: stride = 1 else: stride = 1 xrplt = xfr[::stride] yrplt = yfitr[::stride] yrplt_d = yfr[::stride] xbplt = xfb[::stride] ybplt = yfitb[::stride] ybplt_d = yfb[::stride] xdplt = xfd[::stride] ydplt = yfitd[::stride] ydplt_d = yfd[::stride] p = figure( title=self.action.args.plotlabel + ' Blue/Red fits', x_axis_label='Wave (A)', y_axis_label='Flux (e-)', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.line(xrplt, yrplt, line_color='black', legend_label='Ref') p.circle(xrplt, yrplt_d, size=1, line_alpha=0., fill_color='black') p.line(xbplt, ybplt, line_color='blue', legend_label='Blue') p.circle(xbplt, ybplt_d, size=1, line_alpha=0., fill_color='blue') p.line(xdplt, ydplt, line_color='red', legend_label='Red') p.circle(xdplt, ydplt_d, size=1, line_alpha=0., fill_color='red') bokeh_plot(p, self.context.bokeh_session) if self.config.instrument.plot_level >= 2: input("Next? <cr>: ") else: time.sleep(self.config.instrument.plot_pause) save_plot(p, filename=rbfnam+'.png') wavebuffer = 0.1 minrwave = np.min(xfr) maxrwave = np.max(xfr) wavebuffer2 = 0.05 wlb0 = minrwave+(maxrwave-minrwave)*wavebuffer2 wlb1 = minrwave+(maxrwave-minrwave)*wavebuffer wlr0 = minrwave+(maxrwave-minrwave)*(1.-wavebuffer) wlr1 = minrwave+(maxrwave-minrwave)*(1.-wavebuffer2) qbluefit = [i for i, v in enumerate(waves) if wlb0 < v < wlb1] qredfit = [i for i, v in enumerate(waves) if wlr0 < v < wlr1] nqb = len(qbluefit) nqr = len(qredfit) self.logger.info("Wavelength regions: blue = %.1f - %.1f, " "red = %.1f - %.1f" % (wlb0, wlb1, wlr0, wlr1)) self.logger.info("Fit points: blue = %d, red = %d" % (nqb, nqr)) if nqb > 0: bluefit, _ = sftb.value(waves[qbluefit]) refbluefit, _ = sftr.value(waves[qbluefit]) blue_zero_cross = np.nanmin(bluefit) <= 0. or np.nanmin( refbluefit) <= 0. bluelinfit = np.polyfit(waves[qbluefit], refbluefit/bluefit, 1) bluelinfity = bluelinfit[1] + bluelinfit[0] * waves[qbluefit] else: bluefit = None blue_zero_cross = False refbluefit = None bluelinfit = None bluelinfity = None if nqr > 0: redfit, _ = sftd.value(waves[qredfit]) refredfit, _ = sftr.value(waves[qredfit]) red_zero_cross = np.nanmin(redfit) <= 0. or np.nanmin( refredfit) <= 0. redlinfit = np.polyfit(waves[qredfit], refredfit/redfit, 1) redlinfity = redlinfit[1] + redlinfit[0] * waves[qredfit] else: redfit = None red_zero_cross = False refredfit = None redlinfit = None redlinfity = None if blue_zero_cross: self.logger.info("Blue extension zero crossing detected") if red_zero_cross: self.logger.info("Red extension zero crossing detected") if self.config.instrument.plot_level >= 1: if nqb > 1: # plot blue fits p = figure( title=self.action.args.plotlabel + ' Blue fits', x_axis_label='Wave (A)', y_axis_label='Flux (e-)', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.line(waves[qbluefit], refbluefit, line_color='black', legend_label='Ref') p.circle(waves[qbluefit], refbluefit, fill_color='black') p.line(waves[qbluefit], bluefit, line_color='blue', legend_label='Blue') p.circle(waves[qbluefit], bluefit, fill_color='blue') bokeh_plot(p, self.context.bokeh_session) if self.config.instrument.plot_level >= 2: input("Next? <cr>: ") else: time.sleep(self.config.instrument.plot_pause) # plot blue ratios p = figure( title=self.action.args.plotlabel + ' Blue ratios', x_axis_label='Wave (A)', y_axis_label='Ratio', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.line(waves[qbluefit], refbluefit/bluefit, line_color='black', legend_label='Ref') p.circle(waves[qbluefit], refbluefit/bluefit, fill_color='black') p.line(waves[qbluefit], bluelinfity, line_color='blue', legend_label='Blue') p.circle(waves[qbluefit], bluelinfity, fill_color='blue') bokeh_plot(p, self.context.bokeh_session) if self.config.instrument.plot_level >= 2: input("Next? <cr>: ") else: time.sleep(self.config.instrument.plot_pause) if nqr > 1: # plot red fits p = figure( title=self.action.args.plotlabel + ' Red fits', x_axis_label='Wave (A)', y_axis_label='Flux (e-)', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.line(waves[qredfit], refredfit, line_color='black', legend_label='Ref') p.circle(waves[qredfit], refredfit, fill_color='black') p.line(waves[qredfit], redfit, line_color='red', legend_label='Red') p.circle(waves[qredfit], redfit, fill_color='red') bokeh_plot(p, self.context.bokeh_session) if self.config.instrument.plot_level >= 2: input("Next? <cr>: ") else: time.sleep(self.config.instrument.plot_pause) # plot red ratios p = figure( title=self.action.args.plotlabel + ' Red ratios', x_axis_label='Wave (A)', y_axis_label='Ratio', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.line(waves[qredfit], refredfit/redfit, line_color='black', legend_label='Ref') p.circle(waves[qredfit], refredfit/redfit, fill_color='black') p.line(waves[qredfit], redlinfity, line_color='red', legend_label='Red') p.circle(waves[qredfit], redlinfity, fill_color='red') bokeh_plot(p, self.context.bokeh_session) if self.config.instrument.plot_level >= 2: input("Next? <cr>: ") else: time.sleep(self.config.instrument.plot_pause) # at this point we are going to try to merge the points self.logger.info("Correcting points outside %.1f - %.1f A" % (minrwave, maxrwave)) qselblue = [i for i, v in enumerate(xfb) if v <= minrwave] qselred = [i for i, v in enumerate(xfd) if v >= maxrwave] nqsb = len(qselblue) nqsr = len(qselred) blue_all_tie = yfitr[0] red_all_tie = yfitr[-1] self.logger.info("Blue/Red ref tie values: %.3f, %.3f" % (blue_all_tie, red_all_tie)) if nqsb > 0: self.logger.info("Blue ext tie value: %.3f" % yfb[qselblue[-1]]) if blue_zero_cross: blue_offset = yfb[qselblue[-1]] - blue_all_tie bluefluxes = [yfb[i] - blue_offset for i in qselblue] self.logger.info("Blue zero crossing, only applying offset") else: blue_offset = yfb[qselblue[-1]] * \ (bluelinfit[1]+bluelinfit[0]*xfb[qselblue[-1]]) \ - blue_all_tie bluefluxes = [yfb[i] * (bluelinfit[1]+bluelinfit[0]*xfb[i]) - blue_offset for i in qselblue] self.logger.info("Blue linear ratio fit scaling applied") self.logger.info("Blue offset of %.2f applied" % blue_offset) else: bluefluxes = None if nqsr > 0: self.logger.info("Red ext tie value: %.3f" % yfd[qselred[0]]) if red_zero_cross: red_offset = yfd[qselred[0]] - red_all_tie redfluxes = [yfd[i] - red_offset for i in qselred] self.logger.info("Red zero crossing, only applying offset") else: red_offset = yfd[qselred[0]] * \ (redlinfit[1]+redlinfit[0]*xfd[qselred[0]]) \ - red_all_tie redfluxes = [yfd[i] * (redlinfit[1]+redlinfit[0]*xfd[i]) - red_offset for i in qselred] self.logger.info("Red linear ratio fit scaling applied") self.logger.info("Red offset of %.2f applied" % red_offset) else: redfluxes = None allx = xfr allfx = xfr[nrefx:-nrefx] ally = yfr[nrefx:-nrefx] if nqsb > 0: bluex = xfb[qselblue] allx = np.append(bluex, allx) allfx = np.append(bluex[nrefx:], allfx) ally = np.append(bluefluxes[nrefx:], ally) if nqsr > 0: redx = xfd[qselred] allx = np.append(allx, redx) allfx = np.append(allfx, redx[:-nrefx]) ally = np.append(ally, redfluxes[:-nrefx]) s = np.argsort(allx) allx = allx[s] s = np.argsort(allfx) allfx = allfx[s] ally = ally[s] bkpt = np.min(allx) + np.arange(knots+1) * \ (np.max(allx) - np.min(allx)) / knots sftall, _ = Bspline.iterfit(allfx, ally, fullbkpt=bkpt) yfitall, _ = sftall.value(allx) if self.config.instrument.plot_level >= 1: # output filename stub fltfnam = "flat_%05d_%s_%s_%s" % \ (self.action.args.ccddata.header['FRAMENO'], self.action.args.illum, self.action.args.grating, self.action.args.ifuname) if xbin == 1: stride = int(len(allx) / 8000.) else: stride = 1 xplt = allfx[::stride] yplt = ally[::stride] fxplt = allx[::stride] fplt = yfitall[::stride] yran = [np.nanmin(ally), np.nanmax(ally)] p = figure( title=self.action.args.plotlabel + ' Master Illumination', x_axis_label='Wave (A)', y_axis_label='Flux (e-)', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.circle(xplt, yplt, size=1, line_alpha=0., fill_color='black', legend_label='Data') p.line(fxplt, fplt, line_color='red', legend_label='Fit', line_width=2) p.line([minrwave, minrwave], yran, line_color='orange', legend_label='Cor lim') p.line([maxrwave, maxrwave], yran, line_color='orange') p.legend.location = "top_left" bokeh_plot(p, self.context.bokeh_session) if self.config.instrument.plot_level >= 2: input("Next? <cr>: ") else: time.sleep(self.config.instrument.plot_pause) save_plot(p, filename=fltfnam+".png") # OK, Now we have extended to the full range... so... we are going to # make a ratio flat! comflat = np.zeros(newflat.shape, dtype=float) qz = [i for i, v in enumerate(wavemap.data.flat) if v >= 0] comvals = sftall.value(wavemap.data.flat[qz]) comflat.flat[qz] = comvals ratio = np.zeros(newflat.shape, dtype=float) qzer = [i for i, v in enumerate(newflat.flat) if v != 0] ratio.flat[qzer] = comflat.flat[qzer] / newflat.flat[qzer] # trim negative points qq = [i for i, v in enumerate(ratio.flat) if v < 0] if len(qq) > 0: ratio.flat[qq] = 0.0 # trim the high points near edges of slice qq = [i for i, v in enumerate(ratio.flat) if v >= 3. and (posmap.data.flat[i] <= 4/xbin or posmap.data.flat[i] >= 136/xbin)] if len(qq) > 0: ratio.flat[qq] = 0.0 # don't correct low signal points qq = [i for i, v in enumerate(newflat.flat) if v < 30.] if len(qq) > 0: ratio.flat[qq] = 1.0 # get master flat output name mfname = stack_list[0].split('.fits')[0] + '_' + suffix + '.fits' log_string = MakeMasterFlat.__module__ stacked.header['IMTYPE'] = self.action.args.new_type stacked.header['HISTORY'] = log_string stacked.header['MASTFLAT'] = (True, 'master flat image?') stacked.header['WAVMAPF'] = wmf stacked.header['SLIMAPF'] = slf stacked.header['POSMAPF'] = pof # store flat in output frame stacked.data = ratio # output master flat kcwi_fits_writer(stacked, output_file=mfname, output_dir=self.config.instrument.output_directory) self.context.proctab.update_proctab(frame=stacked, suffix=suffix, newtype=self.action.args.new_type, filename=self.action.args.name) self.context.proctab.write_proctab() self.logger.info(log_string) return self.action.args
def _perform(self): self.logger.info("Tracing continuum bars") if self.config.instrument.plot_level >= 1: do_plot = True else: do_plot = False if len(self.action.args.middle_centers) < 1: self.logger.error("No bars found") elif not self.action.args.bar_avg: self.logger.error("No threshold for tracing") else: # initialize samp = int(80 / self.action.args.ybinsize) win = self.action.args.window bar_thresh = self.action.args.bar_avg self.logger.info("Tracing bars with threshold of %.1f" % bar_thresh) xi = [] # x input xo = [] # x output yi = [] # y input (and output) barid = [] # bar id number slid = [] # slice id number # loop over bars for barn, barx in enumerate(self.action.args.middle_centers): # nearest pixel to bar center barxi = int(barx + 0.5) self.logger.info("bar number %d is at %.3f" % (barn, barx)) # middle row data xi.append(barx) xo.append(barx) yi.append(self.action.args.middle_row) barid.append(barn) slid.append(int(barn/5)) # trace up samy = self.action.args.middle_row + samp done = False while samy < (self.action.args.ccddata.data.shape[0] - win) \ and not done: ys = np.median( self.action.args.ccddata.data[(samy - win): (samy + win + 1), (barxi - win): (barxi + win + 1)], axis=0) ys = ys - np.nanmin(ys) if np.nanmax(ys) > bar_thresh and np.nansum(ys) > 0: xs = list(range(barxi - win, barxi + win + 1)) xc = np.nansum(xs * ys) / np.nansum(ys) xi.append(xc) xo.append(barx) yi.append(samy) barid.append(barn) slid.append(int(barn/5)) barxi = int(xc) else: done = True samy += samp # trace down # nearest pixel to bar center barxi = int(barx + 0.5) samy = self.action.args.middle_row - samp done = False while samy >= win and not done: ys = np.median( self.action.args.ccddata.data[(samy - win): (samy + win + 1), (barxi - win): (barxi + win + 1)], axis=0) ys = ys - np.nanmin(ys) if np.nanmax(ys) > bar_thresh and np.nansum(ys) > 0: xs = list(range(barxi - win, barxi + win + 1)) xc = np.sum(xs * ys) / np.sum(ys) xi.append(xc) xo.append(barx) yi.append(samy) barid.append(barn) slid.append(int(barn / 5)) barxi = int(xc) else: done = True samy -= samp # end loop over bars # create source and destination coords yo = yi dst = np.column_stack((xi, yi)) src = np.column_stack((xo, yo)) if do_plot: # output filename stub trcfnam = "bars_%05d_%s_%s_%s" % \ (self.action.args.ccddata.header['FRAMENO'], self.action.args.illum, self.action.args.grating, self.action.args.ifuname) # plot them p = figure(title=self.action.args.plotlabel + 'SPATIAL CONTROL POINTS', x_axis_label="CCD X (px)", y_axis_label="CCD Y (px)", plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.scatter(xi, yi, marker='x', size=2, color='blue') p.scatter(self.action.args.middle_centers, [self.action.args.middle_row]*120, color='red') bokeh_plot(p, self.context.bokeh_session) if self.config.instrument.plot_level >= 2: input("Next? <cr>: ") else: time.sleep(self.config.instrument.plot_pause) save_plot(p, filename=trcfnam+".png") trace = { 'src': src, 'dst': dst, 'barid': barid, 'slid': slid, 'MIDROW': self.action.args.middle_row, 'WINDOW': self.action.args.window, 'REFDELX': self.action.args.reference_delta_x, 'CBARSNO': self.action.args.contbar_image_number, 'CBARSFL': self.action.args.contbar_image} # in this line we pass the trace information to an argument # instead of writing it to a table self.context.trace = trace ofname = strip_fname(self.action.args.contbar_image) + \ "_trace.fits" write_table(table=[src, dst, barid, slid], names=('src', 'dst', 'barid', 'slid'), output_dir=os.path.join( self.config.instrument.cwd, self.config.instrument.output_directory), output_name=ofname, clobber=self.config.instrument.clobber, comment=['Source and destination fiducial points', 'Derived from KCWI continuum bars images', 'For defining spatial transformation'], keywords={'MIDROW': (self.action.args.middle_row, "Middle Row of image"), 'WINDOW': (self.action.args.window, "Window for bar"), 'REFDELX': ( self.action.args.reference_delta_x, "Reference bar sep in px"), 'CBARSNO': ( self.action.args.contbar_image_number, "Cont. bars image number"), 'CBARSFL': (self.action.args.contbar_image, "Cont. bars image")}) if self.config.instrument.saveintims: from kcwidrp.primitives.kcwi_file_primitives import \ kcwi_fits_writer from skimage import transform as tf # fit transform self.logger.info("Fitting spatial control points") tform = tf.estimate_transform('polynomial', src, dst, order=3) self.logger.info("Transforming bars image") warped = tf.warp(self.action.args.ccddata.data, tform) # write out warped image self.action.args.ccddata.data = warped kcwi_fits_writer( self.action.args.ccddata, output_file=self.action.args.name, output_dir=self.config.instrument.output_directory, suffix='warped') self.logger.info("Transformed bars produced") log_string = TraceBars.__module__ self.action.args.ccddata.header['HISTORY'] = log_string self.logger.info(log_string) return self.action.args
def _perform(self): self.logger.info("Solving overall geometry") # Get some geometry constraints goody0 = 0 goody1 = max(self.action.args.xsvals) # N&S limits goodnsy0 = self.action.args.shufrows goodnsy1 = goodnsy0 + self.action.args.shufrows # Calculate wavelength ranges y0wvs = [] y1wvs = [] # N&S ranges y0nswvs = [] y1nswvs = [] # Get wavelength extremes for each bar for fcfs in self.action.args.fincoeff: y0wvs.append(float(np.polyval(fcfs, goody0))) y1wvs.append(float(np.polyval(fcfs, goody1))) y0nswvs.append(float(np.polyval(fcfs, goodnsy0))) y1nswvs.append(float(np.polyval(fcfs, goodnsy1))) # Now get ensemble extremes y0max = max(y0wvs) y0min = min(y0wvs) y1max = max(y1wvs) y1min = min(y1wvs) y0nsmax = max(y0nswvs) y0nsmin = min(y0nswvs) y1nsmax = max(y1nswvs) y1nsmin = min(y1nswvs) # Cube trimming wavelengths trimw0 = y0min trimw1 = y1max # Check for negative dispersion if trimw0 > trimw1: trimw0 = y1min trimw1 = y0max # Calculate output wavelengths dwout = self.action.args.dwout ndels = int((trimw0 - self.config.instrument.WAVEFID) / dwout) self.action.args.wave0out = \ self.config.instrument.WAVEFID + float(ndels) * dwout ndels = int((trimw1 - self.config.instrument.WAVEFID) / dwout) self.action.args.wave1out = \ self.config.instrument.WAVEFID + float(ndels) * dwout self.logger.info( "WAVE RANGE: %.2f - %.2f" % (self.action.args.wave0out, self.action.args.wave1out)) # Calculate wavelength limits self.action.args.wavegood0 = min([y0max, y1max]) self.action.args.wavegood1 = max([y0min, y1min]) self.action.args.waveall0 = min([y0min, y1min]) self.action.args.waveall1 = max([y0max, y1max]) self.action.args.wavemid = np.average([ self.action.args.wavegood0, self.action.args.wavegood1, self.action.args.waveall0, self.action.args.waveall1 ]) self.action.args.wavensgood0 = min([y0nsmax, y1nsmax]) self.action.args.wavensgood1 = max([y0nsmin, y1nsmin]) self.action.args.wavensall0 = min([y0nsmin, y1nsmin]) self.action.args.wavensall1 = max([y0nsmax, y1nsmax]) self.action.args.wavensmid = np.average([ self.action.args.wavensgood0, self.action.args.wavensgood1, self.action.args.wavensall0, self.action.args.wavensall1 ]) self.logger.info( "WAVE GOOD: %.2f - %.2f" % (self.action.args.wavegood0, self.action.args.wavegood1)) self.logger.info( "WAVE ALL: %.2f - %.2f" % (self.action.args.waveall0, self.action.args.waveall1)) self.logger.info("WAVE MID: %.2f" % self.action.args.wavemid) # Start setting up slice transforms self.action.args.x0out = \ int(self.action.args.reference_bar_separation / 2.) + 1 self.refoutx = np.arange(0, 5) * \ self.action.args.reference_bar_separation + self.action.args.x0out # Variables for output control points srcw = [] # Loop over source control points for ixy, xy in enumerate(self.action.args.source_control_points): # Calculate y wavelength yw = float( np.polyval( self.action.args.fincoeff[self.action.args.bar_id[ixy]], xy[1])) # Convert to output pixels yw = (yw - self.action.args.wave0out) / dwout srcw.append([xy[0], yw]) # Use extremes to define output size ysize = int( (self.action.args.waveall1 - self.action.args.wave0out) / dwout) xsize = int(5. * self.action.args.reference_bar_separation) + 1 self.logger.info("Output slices will be %d x %d px" % (xsize, ysize)) # Now loop over slices and get relevant control points for each slice # Output variables xl0_out = [] xl1_out = [] tform_list = [] invtf_list = [] # Loop over 24 slices for isl in range(0, 24): # Get control points xw = [] yw = [] xi = [] yi = [] # Loop over all control points for ixy, xy in enumerate(srcw): # Only use the ones for this slice if self.action.args.slice_id[ixy] == isl: # Index in to reference output x array ib = self.action.args.bar_id[ixy] % 5 # Geometrically corrected control points xw.append(self.refoutx[ib]) yw.append(xy[1]) # Input control points xi.append( self.action.args.destination_control_points[ixy][0]) yi.append( self.action.args.destination_control_points[ixy][1]) # get image limits xl0 = int(min(xi) - self.action.args.reference_bar_separation) if xl0 < 0: xl0 = 0 xl1 = int(max(xi) + self.action.args.reference_bar_separation) if xl1 > (self.action.args.ccddata.data.shape[0] - 1): xl1 = self.action.args.ccddata.data.shape[0] - 1 # Store for output xl0_out.append(xl0) xl1_out.append(xl1) self.logger.info("Slice %d arc image x limits: %d - %d" % (isl, xl0, xl1)) # adjust control points xit = [x - float(xl0) for x in xi] # fit transform dst = np.column_stack((xit, yi)) src = np.column_stack((xw, yw)) self.logger.info("Fitting wavelength and spatial control points") tform = tf.estimate_transform('polynomial', src, dst, order=3) invtf = tf.estimate_transform('polynomial', dst, src, order=3) # Store for output tform_list.append(tform) invtf_list.append(invtf) # Pixel scales pxscl = self.config.instrument.PIXSCALE * self.action.args.xbinsize ifunum = self.action.args.ifunum if ifunum == 2: slscl = self.config.instrument.SLICESCALE / 2.0 elif ifunum == 3: slscl = self.config.instrument.SLICESCALE / 4.0 else: slscl = self.config.instrument.SLICESCALE # Dichroic fraction try: dichroic_fraction = self.action.args.dichroic_fraction except AttributeError: dichroic_fraction = 1. # Package geometry data ofname = self.action.args.name self.action.args.geometry_file = os.path.join( self.config.instrument.output_directory, strip_fname(ofname) + '_geom.pkl') if os.path.exists(self.action.args.geometry_file): self.logger.error("Geometry file already exists: %s" % self.action.args.geometry_file) else: geom = { "geom_file": self.action.args.geometry_file, "xsize": xsize, "ysize": ysize, "pxscl": pxscl, "slscl": slscl, "cbarsno": self.action.args.contbar_image_number, "cbarsfl": self.action.args.contbar_image, "arcno": self.action.args.arc_number, "arcfl": self.action.args.arc_image, "barsep": self.action.args.reference_bar_separation, "bar0": self.action.args.x0out, "waveall0": self.action.args.waveall0, "waveall1": self.action.args.waveall1, "wavegood0": self.action.args.wavegood0, "wavegood1": self.action.args.wavegood1, "wavemid": self.action.args.wavemid, "wavensall0": self.action.args.wavensall0, "wavensall1": self.action.args.wavensall1, "wavensgood0": self.action.args.wavensgood0, "wavensgood1": self.action.args.wavensgood1, "wavensmid": self.action.args.wavensmid, "dich_frac": dichroic_fraction, "dwout": dwout, "wave0out": self.action.args.wave0out, "wave1out": self.action.args.wave1out, "avwvsig": self.action.args.av_bar_sig, "sdwvsig": self.action.args.st_bar_sig, "xl0": xl0_out, "xl1": xl1_out, "tform": tform_list, "invtf": invtf_list } with open(self.action.args.geometry_file, 'wb') as ofile: pickle.dump(geom, ofile) self.logger.info("Geometry written to: %s" % self.action.args.geometry_file) log_string = SolveGeom.__module__ self.action.args.ccddata.header['HISTORY'] = log_string self.logger.info(log_string) return self.action.args
def _perform(self): self.logger.info("Subtracting sky background") # Header keyword to update key = 'SKYCOR' keycom = 'sky corrected?' target_type = 'SKY' skyfile = self.action.args.skyfile skymask = self.action.args.skymask if not self.action.args.skyfile: tab = self.context.proctab.search_proctab( frame=self.action.args.ccddata, target_type=target_type, nearest=True) self.logger.info("%d master sky frames found" % len(tab)) if len(tab) > 0: skyfile = tab['filename'][0] msname = strip_fname(skyfile) + '_' + target_type.lower() + ".fits" if os.path.exists( os.path.join(self.config.instrument.cwd, 'redux', msname)): self.logger.info("Reading image: %s" % msname) msky = kcwi_fits_reader( os.path.join(self.config.instrument.cwd, 'redux', msname))[0] # scale the sky? obtime = self.action.args.ccddata.header['XPOSURE'] sktime = msky.header['XPOSURE'] if obtime <= 0. or sktime <= 0.: self.logger.warning( "Bad exposure times (obj, sky): %.1f, %1f" % (obtime, sktime)) skscl = 1. else: skscl = obtime / sktime self.logger.info("Sky scale factor = %.3f" % skscl) # do the subtraction self.action.args.ccddata.data -= msky.data * skscl # update header keywords self.action.args.ccddata.header[key] = (True, keycom) self.action.args.ccddata.header['SKYMAST'] = ( msname, "Master sky filename") self.action.args.ccddata.header['SKYSCL'] = (skscl, 'sky scale factor') if skymask: self.action.args.ccddata.header['SKYMSKF'] = (skymask, 'sky mask file') else: # update header keywords self.action.args.ccddata.header[key] = (False, keycom) log_string = SubtractSky.__module__ self.action.args.ccddata.header['HISTORY'] = log_string # write out int image kcwi_fits_writer(self.action.args.ccddata, table=self.action.args.table, output_file=self.action.args.name, output_dir=self.config.instrument.output_directory, suffix="intk") self.context.proctab.update_proctab(frame=self.action.args.ccddata, suffix="intk", filename=self.action.args.name) self.context.proctab.write_proctab() self.logger.info(log_string) return self.action.args
def _perform(self): self.logger.info("Subtracting nod-and shuffle sky background") # Header keyword to update key = 'NASSUB' keycom = 'Nod-and-shuffle subtraction done?' target_type = 'SKY' # get header values ofn = self.action.args.name shrows = self.action.args.ccddata.header['SHUFROWS'] nshfup = self.action.args.ccddata.header['NSHFUP'] nshfdn = self.action.args.ccddata.header['NSHFDN'] # units u_out = self.action.args.ccddata.unit # nominal conditions (sky on bottom, object in middle) skyrow0 = 0 skyrow1 = shrows - 1 objrow0 = shrows objrow1 = shrows + shrows - 1 # aborted script with inverted panels (sky in middle, object above) if nshfdn != nshfup + 1: skyrow0 = shrows skyrow1 = shrows + shrows - 1 objrow0 = skyrow1 objrow1 = objrow0 + shrows - 1 # check limits if (skyrow1-skyrow0) != (objrow1-objrow0): self.logger.error("Nod-and-shuffle row limits error") return self.action.args # create intermediate images and headers sky = self.action.args.ccddata.data.copy() obj = self.action.args.ccddata.data.copy() std = self.action.args.ccddata.uncertainty.array.copy() msk = self.action.args.ccddata.mask.copy() flg = self.action.args.ccddata.flags.copy() skyhdr = self.action.args.ccddata.header.copy() objhdr = self.action.args.ccddata.header.copy() # nominal condition if skyrow0 < 10: self.logger.info("standard nod-and-shuffle configuration") skystd = self.action.args.ccddata.uncertainty.array.copy() skymsk = self.action.args.ccddata.mask.copy() skyflg = self.action.args.ccddata.flags.copy() # move sky to object position sky[objrow0:objrow1, :] = obj[skyrow0:skyrow1, :] skystd[objrow0:objrow1, :] = std[skyrow0:skyrow1, :] skymsk[objrow0:objrow1, :] = msk[skyrow0:skyrow1, :] skyflg[objrow0:objrow1, :] = flg[skyrow0:skyrow1, :] # do subtraction self.action.args.ccddata.data -= sky self.action.args.ccddata.uncertainty.array = np.sqrt(std ** 2 + skystd ** 2) self.action.args.ccddata.mask += skymsk self.action.args.ccddata.flags |= skyflg # clean images self.action.args.ccddata.data[skyrow0:skyrow1, :] = 0. self.action.args.ccddata.data[(objrow1+1):-1, :] = 0. self.action.args.ccddata.uncertainty.array[skyrow0:skyrow1, :] = 0. self.action.args.ccddata.uncertainty.array[(objrow1+1):-1, :] = 0. self.action.args.ccddata.mask[skyrow0:skyrow1, :] = 1 self.action.args.ccddata.mask[(objrow1 + 1):-1, :] = 1 self.action.args.ccddata.flags[skyrow0:skyrow1, :] = 64 self.action.args.ccddata.flags[(objrow1 + 1):-1, :] = 64 sky[skyrow0:skyrow1, :] = 0. sky[(objrow1+1), :] = 0. obj[skyrow0:skyrow1, :] = 0. obj[(objrow1+1), :] = 0. else: self.logger.warning("non-standard nod-and-shuffle configuration") skyscl = -1. while skyscl < 0.: if self.config.instrument.plot_level >= 2: q = input("Enter scale factor for sky to match obj " "(float): ") try: skyscl = float(q) except ValueError: self.logger.warning("Invalid input: %s, try again" % q) skyscl = -1.0 else: skyscl = 1.0 self.logger.info("Sky scaling used = %.2f" % skyscl) objstd = self.action.args.ccddata.uncertainty.array.copy() objmsk = self.action.args.ccddata.mask.copy() objflg = self.action.args.ccddata.flags.copy() # move object to sky position obj[skyrow0:skyrow1, :] = obj[objrow0:objrow1, :] objstd[skyrow0:skyrow1, :] = std[objrow0:objrow1, :] objmsk[skyrow0:skyrow1, :] = msk[objrow0:objrow1, :] objflg[skyrow0:skyrow1, :] = flg[objrow0:objrow1, :] # do subtraction sky *= skyscl self.action.args.ccddata.data = obj - sky self.action.args.ccddata.uncertainty.array = np.sqrt(std ** 2 + objstd ** 2) self.action.args.ccddata.mask += objmsk self.action.args.ccddata.flags |= objflg # clean images self.action.args.ccddata.data[objrow0:objrow1, :] = 0. self.action.args.ccddata.data[0:skyrow0, :] = 0. self.action.args.ccddata.uncertainty.array[objrow0:objrow1, :] = 0. self.action.args.ccddata.uncertainty.array[0:skyrow0, :] = 0. self.action.args.ccddata.mask[objrow0:objrow1, :] = 1 self.action.args.ccddata.mask[0:skyrow0, :] = 1 self.action.args.ccddata.flags[objrow0:objrow1, :] = 64 self.action.args.ccddata.flags[0:skyrow0, :] = 64 sky[objrow0:objrow1, :] = 0. sky[0:skyrow0, :] = 0. obj[objrow0:objrow1, :] = 0. obj[0:skyrow0, :] = 0. cmnt = 'Aborted nod-and-shuffle observations' objhdr['COMMENT'] = cmnt skyhdr['COMMENT'] = cmnt self.action.args.ccddata.header['COMMENT'] = cmnt skyhdr['NASSCL'] = (skyscl, 'Scale factor applied to sky panel') self.action.args.ccddata.header['NASSCL'] = ( skyscl, 'Scale factor applied to sky panel') # log self.logger.info("nod-and-shuffle subtracted, rows (sky0, 1, obj0,1): " "%d, %d, %d, %d" % (skyrow0, skyrow1, objrow0, objrow1)) # update headers log_string = NandshuffSubtractSky.__module__ objhdr[key] = (False, keycom) objhdr['HISTORY'] = log_string skyhdr[key] = (False, keycom) skyhdr['SKYOBS'] = (True, 'Sky observation?') skyhdr['HISTORY'] = log_string self.action.args.ccddata.header[key] = (True, keycom) self.action.args.ccddata.header['HISTORY'] = log_string # write out sky image msname = strip_fname(ofn) + '_' + target_type.lower() + '.fits' out_sky = CCDData(sky, meta=skyhdr, unit=u_out) kcwi_fits_writer(out_sky, output_file=msname, output_dir=self.config.instrument.output_directory) # write out object image obname = strip_fname(ofn) + '_obj.fits' out_obj = CCDData(obj, meta=objhdr, unit=u_out) kcwi_fits_writer(out_obj, output_file=obname, output_dir=self.config.instrument.output_directory) # update header keywords self.action.args.ccddata.header['SKYMAST'] = (msname, "Master sky filename") # write out int image kcwi_fits_writer(self.action.args.ccddata, table=self.action.args.table, output_file=self.action.args.name, output_dir=self.config.instrument.output_directory, suffix="intk") self.context.proctab.update_proctab(frame=self.action.args.ccddata, suffix="intk", filename=self.action.args.name) self.context.proctab.write_proctab() self.logger.info(log_string) return self.action.args
def _perform(self): """ Returns an Argument() with the parameters that depends on this operation """ method = 'average' suffix = self.action.args.stack_type.lower() self.logger.info("Stacking flats using method %s" % method) combine_list = list(self.combine_list['filename']) # get flat stack output name stname = strip_fname(combine_list[-1]) + '_' + suffix + '.fits' stack = [] stackf = [] mask = None for flat in combine_list: # get flat intensity (int) image file name in redux directory stackf.append(strip_fname(flat) + '_intd.fits') flatfn = os.path.join(self.config.instrument.cwd, self.config.instrument.output_directory, stackf[-1]) # using [0] gets just the image data f = kcwi_fits_reader(flatfn)[0] # Set mask to None to prevent ccdproc.combine from masking f.mask = None stack.append(f) stacked = ccdproc.combine(stack, method=method, sigma_clip=True, sigma_clip_low_thresh=None, sigma_clip_high_thresh=2.0) # Get the BPM out of one of the flats (bpm is the same for all) # and add it to the stacked flat as the stack's mask last_flat_name = strip_fname(combine_list[-1]) + '_intd.fits' last_flat_path = os.path.join(self.config.instrument.cwd, self.config.instrument.output_directory, last_flat_name) last_flat = kcwi_fits_reader(last_flat_path)[0] stacked.mask = last_flat.mask stacked.header['IMTYPE'] = self.action.args.stack_type stacked.header['NSTACK'] = (len(combine_list), 'number of images stacked') stacked.header['STCKMETH'] = (method, 'method used for stacking') for ii, fname in enumerate(stackf): stacked.header['STACKF%d' % (ii + 1)] = (fname, "stack input file") log_string = StackFlats.__module__ stacked.header['HISTORY'] = log_string # output stacked flat kcwi_fits_writer(stacked, output_file=stname, output_dir=self.config.instrument.output_directory) self.context.proctab.update_proctab(frame=stacked, suffix=suffix, newtype=self.action.args.stack_type, filename=self.action.args.name) self.context.proctab.write_proctab() self.logger.info(log_string) return self.action.args
def _perform(self): """Correct for differential atmospheric refraction""" self.logger.info("Correcting for DAR") # image size image_size = self.action.args.ccddata.data.shape # get wavelengths w0 = self.action.args.ccddata.header['CRVAL3'] dw = self.action.args.ccddata.header['CD3_3'] waves = w0 + np.arange(image_size[0]) * dw wgoo0 = self.action.args.ccddata.header['WAVGOOD0'] wgoo1 = self.action.args.ccddata.header['WAVGOOD1'] wref = self.action.args.ccddata.header['WAVMID'] self.logger.info("Ref WL = %.1f, good WL range = (%.1f - %.1f" % (wref, wgoo0, wgoo1)) # spatial scales in arcsec/item y_scale = self.action.args.ccddata.header['PXSCL'] * 3600. x_scale = self.action.args.ccddata.header['SLSCL'] * 3600. # padding depends on grating if 'H' in self.action.args.grating: padding_as = 2.0 elif 'M' in self.action.args.grating: padding_as = 3.0 else: padding_as = 4.0 padding_x = int(padding_as / x_scale) padding_y = int(padding_as / y_scale) # update WCS crpix1 = self.action.args.ccddata.header['CRPIX1'] crpix2 = self.action.args.ccddata.header['CRPIX2'] self.action.args.ccddata.header['CRPIX1'] = crpix1 + float(padding_x) self.action.args.ccddata.header['CRPIX2'] = crpix2 + float(padding_y) # airmass airmass = self.action.args.ccddata.header['AIRMASS'] self.logger.info("Airmass: %.3f" % airmass) # IFU orientation ifu_pa = self.action.args.ccddata.header['IFUPA'] # Parallactic angle parallactic_angle = self.action.args.ccddata.header['PARANG'] # Projection angle in radians projection_angle_deg = ifu_pa - parallactic_angle projection_angle = math.radians(projection_angle_deg) self.logger.info("DAR Angles: ifu_pa, parang, projang (deg): " "%.2f, %.2f, %.2f" % (ifu_pa, parallactic_angle, projection_angle_deg)) # dispersion over goo wl range in arcsec dispersion_max_as = atm_disper(wgoo1, wgoo0, airmass) # projected onto IFU xdmax_as = dispersion_max_as * math.sin(projection_angle) ydmax_as = dispersion_max_as * math.cos(projection_angle) self.logger.info("DAR over GOOD WL range: total, x, y (asec): " "%.2f, %.2f, %.2f" % (dispersion_max_as, xdmax_as, ydmax_as)) # now in pixels xdmax_px = xdmax_as / x_scale ydmax_px = ydmax_as / y_scale dmax_px = math.sqrt(xdmax_px**2 + ydmax_px**2) self.logger.info("DAR over GOOD WL range: total, x, y (pix): " "%.2f, %.2f, %.2f" % (dmax_px, xdmax_px, ydmax_px)) # prepare output cubes output_image = np.zeros((image_size[0], image_size[1] + 2 * padding_y, image_size[2] + 2 * padding_x), dtype=np.float64) output_stddev = output_image.copy() output_mask = np.zeros((image_size[0], image_size[1] + 2 * padding_y, image_size[2] + 2 * padding_x), dtype=np.uint8) output_flags = np.zeros((image_size[0], image_size[1] + 2 * padding_y, image_size[2] + 2 * padding_x), dtype=np.uint8) # DAR padded pixel flag output_flags += 128 output_image[:, padding_y:(padding_y+image_size[1]), padding_x:(padding_x+image_size[2])] = \ self.action.args.ccddata.data output_stddev[:, padding_y:(padding_y+image_size[1]), padding_x:(padding_x+image_size[2])] = \ self.action.args.ccddata.uncertainty.array output_mask[:, padding_y:(padding_y+image_size[1]), padding_x:(padding_x+image_size[2])] = \ self.action.args.ccddata.mask output_flags[:, padding_y:(padding_y+image_size[1]), padding_x:(padding_x+image_size[2])] = \ self.action.args.ccddata.flags # check for obj, sky cubes output_obj = None output_sky = None if self.action.args.nasmask and self.action.args.numopen > 1: ofn = self.action.args.name objfn = strip_fname(ofn) + '_ocube.fits' full_path = os.path.join(self.config.instrument.cwd, self.config.instrument.output_directory, objfn) if os.path.exists(full_path): obj = kcwi_fits_reader(full_path)[0] output_obj = np.zeros( (image_size[0], image_size[1] + 2 * padding_y, image_size[2] + 2 * padding_x), dtype=np.float64) output_obj[:, padding_y:(padding_y + image_size[1]), padding_x:(padding_x + image_size[2])] = obj.data skyfn = strip_fname(ofn) + '_scube.fits' full_path = os.path.join(self.config.instrument.cwd, self.config.instrument.output_directory, skyfn) if os.path.exists(full_path): sky = kcwi_fits_reader(full_path)[0] output_sky = np.zeros( (image_size[0], image_size[1] + 2 * padding_y, image_size[2] + 2 * padding_x), dtype=np.float64) output_sky[:, padding_y:(padding_y + image_size[1]), padding_x:(padding_x + image_size[2])] = sky.data # check if we have a standard star observation output_del = None stdfile, _ = kcwi_get_std(self.action.args.ccddata.header['OBJECT'], self.logger) if stdfile is not None: afn = self.action.args.ccddata.header['ARCFL'] delfn = strip_fname(afn) + '_dcube.fits' full_path = os.path.join(self.config.instrument.cwd, self.config.instrument.output_directory, delfn) if os.path.exists(full_path): dew = kcwi_fits_reader(full_path)[0] output_del = np.zeros( (image_size[0], image_size[1] + 2 * padding_y, image_size[2] + 2 * padding_x), dtype=np.float64) output_del[:, padding_y:(padding_y + image_size[1]), padding_x:(padding_x + image_size[2])] = dew.data # Perform correction for j, wl in enumerate(waves): dispersion_correction = atm_disper(wref, wl, airmass) x_shift = dispersion_correction * \ math.sin(projection_angle) / x_scale y_shift = dispersion_correction * \ math.cos(projection_angle) / y_scale output_image[j, :, :] = shift(output_image[j, :, :], (y_shift, x_shift)) output_stddev[j, :, :] = shift(output_stddev[j, :, :], (y_shift, x_shift)) output_mask[j, :, :] = shift(output_mask[j, :, :], (y_shift, x_shift)) output_flags[j, :, :] = shift(output_flags[j, :, :], (y_shift, x_shift)) # for obj, sky if they exist if output_obj is not None: for j, wl in enumerate(waves): dispersion_correction = atm_disper(wref, wl, airmass) x_shift = dispersion_correction * \ math.sin(projection_angle) / x_scale y_shift = dispersion_correction * \ math.cos(projection_angle) / y_scale output_obj[j, :, :] = shift(output_obj[j, :, :], (y_shift, x_shift)) if output_sky is not None: for j, wl in enumerate(waves): dispersion_correction = atm_disper(wref, wl, airmass) x_shift = dispersion_correction * \ math.sin(projection_angle) / x_scale y_shift = dispersion_correction * \ math.cos(projection_angle) / y_scale output_sky[j, :, :] = shift(output_sky[j, :, :], (y_shift, x_shift)) # for delta wavelength cube, if it exists if output_del is not None: for j, wl in enumerate(waves): dispersion_correction = atm_disper(wref, wl, airmass) x_shift = dispersion_correction * \ math.sin(projection_angle) / x_scale y_shift = dispersion_correction * \ math.cos(projection_angle) / y_scale output_del[j, :, :] = shift(output_del[j, :, :], (y_shift, x_shift)) self.action.args.ccddata.data = output_image self.action.args.ccddata.uncertainty.array = output_stddev self.action.args.ccddata.mask = output_mask self.action.args.ccddata.flags = output_flags log_string = CorrectDar.__module__ # update header self.action.args.ccddata.header['HISTORY'] = log_string self.action.args.ccddata.header['DARCOR'] = (True, 'DAR corrected?') self.action.args.ccddata.header['DARANG'] = (projection_angle_deg, 'DAR projection angle ' '(deg)') self.action.args.ccddata.header['DARPADX'] = (padding_x, 'DAR X padding (pix)') self.action.args.ccddata.header['DARPADY'] = (padding_y, 'DAR Y padding (pix)') self.action.args.ccddata.header['DAREFWL'] = (wref, 'DAR reference wl (Ang)') # write out corrected image kcwi_fits_writer(self.action.args.ccddata, table=self.action.args.table, output_file=self.action.args.name, output_dir=self.config.instrument.output_directory, suffix="icubed") self.context.proctab.update_proctab(frame=self.action.args.ccddata, suffix="icubed", filename=self.action.args.name) self.context.proctab.write_proctab() # check for sky, obj cube if output_obj is not None: out_obj = CCDData(output_obj, meta=self.action.args.ccddata.header, unit=self.action.args.ccddata.unit) kcwi_fits_writer( out_obj, output_file=self.action.args.name, output_dir=self.config.instrument.output_directory, suffix="ocubed") if output_sky is not None: out_sky = CCDData(output_sky, meta=self.action.args.ccddata.header, unit=self.action.args.ccddata.unit) kcwi_fits_writer( out_sky, output_file=self.action.args.name, output_dir=self.config.instrument.output_directory, suffix="scubed") # check for delta wave cube if output_del is not None: out_del = CCDData(output_del, meta=self.action.args.ccddata.header, unit=self.action.args.ccddata.unit) kcwi_fits_writer( out_del, output_file=self.action.args.name, output_dir=self.config.instrument.output_directory, suffix="dcubed") self.logger.info(log_string) return self.action.args
def _perform(self): self.logger.info("Creating data cube") log_string = MakeCube.__module__ # Are we interactive? if self.config.instrument.plot_level >= 3: do_inter = True else: do_inter = False self.logger.info("Generating data cube") # Find and read geometry transformation tab = self.context.proctab.search_proctab( frame=self.action.args.ccddata, target_type='ARCLAMP', nearest=True) if not len(tab): self.logger.error("No reference geometry, cannot make cube!") self.action.args.ccddata.header['GEOMCOR'] = ( False, 'Geometry corrected?') self.logger.info(log_string) return self.action.args self.logger.info("%d arc frames found" % len(tab)) ofn = strip_fname(tab['filename'][0]) + "_geom.pkl" geom_file = os.path.join(self.config.instrument.cwd, self.config.instrument.output_directory, ofn) if os.path.exists(geom_file): self.logger.info("Reading %s" % geom_file) with open(geom_file, 'rb') as ifile: geom = pickle.load(ifile) # Slice size xsize = geom['xsize'] ysize = geom['ysize'] out_cube = np.zeros((ysize, xsize, 24), dtype=np.float64) out_vube = np.zeros((ysize, xsize, 24), dtype=np.float64) out_mube = np.zeros((ysize, xsize, 24), dtype=np.uint8) out_fube = np.zeros((ysize, xsize, 24), dtype=np.uint8) out_oube = np.zeros((ysize, xsize, 24), dtype=np.float64) out_sube = np.zeros((ysize, xsize, 24), dtype=np.float64) out_dube = np.zeros((ysize, xsize, 24), dtype=np.float64) # Store original data data_img = self.action.args.ccddata.data data_std = self.action.args.ccddata.uncertainty.array data_msk = self.action.args.ccddata.mask data_flg = self.action.args.ccddata.flags # check for obj, sky images obj = None data_obj = None sky = None data_sky = None if self.action.args.nasmask and self.action.args.numopen > 1: ofn = self.action.args.name objfn = strip_fname(ofn) + '_objf.fits' full_path = os.path.join( self.config.instrument.cwd, self.config.instrument.output_directory, objfn) if os.path.exists(full_path): obj = kcwi_fits_reader(full_path)[0] data_obj = obj.data skyfn = strip_fname(ofn) + '_skyf.fits' full_path = os.path.join( self.config.instrument.cwd, self.config.instrument.output_directory, skyfn) if os.path.exists(full_path): sky = kcwi_fits_reader(full_path)[0] data_sky = sky.data # check for geometry maps dew = None data_dew = None if 'ARCLAMP' in self.action.args.imtype: ofn = self.action.args.name dewfn = strip_fname(ofn) + '_delmap.fits' full_path = os.path.join( self.config.instrument.cwd, self.config.instrument.output_directory, dewfn) if os.path.exists(full_path): dew = kcwi_fits_reader(full_path)[0] data_dew = dew.data # Loop over 24 slices my_arguments = [] for isl in range(0, 24): arguments = { 'slice_number': isl, 'geom': geom, 'img': data_img, 'std': data_std, 'msk': data_msk, 'flg': data_flg, 'xsize': xsize, 'ysize': ysize, 'logger': self.logger } if obj is not None: arguments['obj'] = data_obj if sky is not None: arguments['sky'] = data_sky if dew is not None: arguments['del'] = data_dew my_arguments.append(arguments) p = Pool() results = p.map(make_cube_helper, list(my_arguments)) p.close() self.logger.info("Building cube") for partial_cube in results: slice_number = partial_cube[0] out_cube[:, :, slice_number] = partial_cube[1] out_vube[:, :, slice_number] = partial_cube[2] out_mube[:, :, slice_number] = partial_cube[3] out_fube[:, :, slice_number] = partial_cube[4] if obj is not None: out_oube[:, :, slice_number] = partial_cube[5] if sky is not None: out_sube[:, :, slice_number] = partial_cube[6] if dew is not None: out_dube[:, :, slice_number] = partial_cube[7] if self.config.instrument.plot_level >= 3: for isl in range(0, 24): warped = out_cube[:, :, isl] ptitle = self.action.args.plotlabel + \ "WARPED Slice %d" % isl p = figure(tooltips=[("x", "$x"), ("y", "$y"), ("value", "@image")], title=ptitle, x_axis_label="X (px)", y_axis_label="Y (px)", plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.x_range.range_padding = p.y_range.range_padding = 0 p.image([warped], x=0, y=0, dw=xsize, dh=ysize, palette="Spectral11", level="image") bokeh_plot(p, self.context.bokeh_session) if do_inter: q = input("Next? <cr>, q to quit: ") if 'Q' in q.upper(): do_inter = False else: time.sleep(self.config.instrument.plot_pause) # Calculate some WCS parameters # Get object pointing try: if self.action.args.nasmask: rastr = self.action.args.ccddata.header['RABASE'] decstr = self.action.args.ccddata.header['DECBASE'] else: rastr = self.action.args.ccddata.header['RA'] decstr = self.action.args.ccddata.header['DEC'] except KeyError: try: rastr = self.action.args.ccddata.header['TARGRA'] decstr = self.action.args.ccddata.header['TARGDEC'] except KeyError: rastr = '' decstr = '' if len(rastr) > 0 and len(decstr) > 0: try: coord = SkyCoord(rastr, decstr, unit=(u.hourangle, u.deg)) except ValueError: coord = None else: coord = None # Get rotator position if 'ROTPOSN' in self.action.args.ccddata.header: rpos = self.action.args.ccddata.header['ROTPOSN'] else: rpos = 0. if 'ROTREFAN' in self.action.args.ccddata.header: rref = self.action.args.ccddata.header['ROTREFAN'] else: rref = 0. skypa = rpos + rref crota = math.radians(-(skypa + self.config.instrument.ROTOFF)) cdelt1 = -geom['slscl'] cdelt2 = geom['pxscl'] if coord is None: ra = 0. dec = 0. crota = 1 else: ra = coord.ra.degree dec = coord.dec.degree cd11 = cdelt1 * math.cos(crota) cd12 = abs(cdelt2) * np.sign(cdelt1) * math.sin(crota) cd21 = -abs(cdelt1) * np.sign(cdelt2) * math.sin(crota) cd22 = cdelt2 * math.cos(crota) crpix1 = 12. crpix2 = xsize / 2. crpix3 = 1. porg = self.action.args.ccddata.header['PONAME'] ifunum = self.action.args.ifunum if 'IFU' in porg: if ifunum == 1: off1 = 1.0 off2 = 4.0 elif ifunum == 2: off1 = 1.0 off2 = 5.0 elif ifunum == 3: off1 = 0.05 off2 = 5.6 else: self.logger.warning("Unknown IFU number: %d" % ifunum) off1 = 0. off2 = 0. off1 = off1 / float(self.action.args.xbinsize) off2 = off2 / float(self.action.args.ybinsize) crpix1 += off1 crpix2 += off2 # Update header # Geometry corrected? self.action.args.ccddata.header['GEOMCOR'] = ( True, 'Geometry corrected?') # # Spatial geometry self.action.args.ccddata.header['BARSEP'] = ( geom['barsep'], 'separation of bars (binned pix)') self.action.args.ccddata.header['BAR0'] = ( geom['bar0'], 'first bar pixel position') # Wavelength ranges if self.action.args.nasmask: self.action.args.ccddata.header['WAVALL0'] = ( geom['wavensall0'], 'Low inclusive wavelength') self.action.args.ccddata.header['WAVALL1'] = ( geom['wavensall1'], 'High inclusive wavelength') self.action.args.ccddata.header['WAVGOOD0'] = ( geom['wavensgood0'], 'Low good wavelength') self.action.args.ccddata.header['WAVGOOD1'] = ( geom['wavensgood1'], 'High good wavelength') self.action.args.ccddata.header['WAVMID'] = ( geom['wavensmid'], 'middle wavelength') else: self.action.args.ccddata.header['WAVALL0'] = ( geom['waveall0'], 'Low inclusive wavelength') self.action.args.ccddata.header['WAVALL1'] = ( geom['waveall1'], 'High inclusive wavelength') self.action.args.ccddata.header['WAVGOOD0'] = ( geom['wavegood0'], 'Low good wavelength') self.action.args.ccddata.header['WAVGOOD1'] = ( geom['wavegood1'], 'High good wavelength') self.action.args.ccddata.header['WAVMID'] = ( geom['wavemid'], 'middle wavelength') # Dichroic fraction try: dichroic_fraction = geom['dich_frac'] except AttributeError: dichroic_fraction = 1. self.action.args.ccddata.header['DICHFRAC'] = (dichroic_fraction, 'Dichroic Fraction') # Wavelength fit statistics self.action.args.ccddata.header['AVWVSIG'] = ( geom['avwvsig'], 'Avg. bar wave sigma (Ang)') self.action.args.ccddata.header['SDWVSIG'] = ( geom['sdwvsig'], 'Stdev. var wave sigma (Ang)') # Pixel scales self.action.args.ccddata.header['PXSCL'] = ( geom['pxscl'], 'Pixel scale along slice (deg)') self.action.args.ccddata.header['SLSCL'] = ( geom['slscl'], 'Pixel scale perp. to slices (deg)') # Geometry origins self.action.args.ccddata.header['CBARSNO'] = ( geom['cbarsno'], 'Continuum bars image number') self.action.args.ccddata.header['CBARSFL'] = ( geom['cbarsfl'], 'Continuum bars image filename') self.action.args.ccddata.header['ARCNO'] = (geom['arcno'], 'Arc image number') self.action.args.ccddata.header['ARCFL'] = (geom['arcfl'], 'Arc image filename') self.action.args.ccddata.header['GEOMFL'] = ( geom_file.split('/')[-1], 'Geometry file') # WCS self.action.args.ccddata.header['IFUPA'] = ( skypa, 'IFU position angle (degrees)') self.action.args.ccddata.header['IFUROFF'] = ( self.config.instrument.ROTOFF, 'IFU-SKYPA offset (degrees)') self.action.args.ccddata.header['WCSDIM'] = ( 3, 'number of dimensions in WCS') self.action.args.ccddata.header['WCSNAME'] = 'KCWI' self.action.args.ccddata.header['EQUINOX'] = 2000. self.action.args.ccddata.header['RADESYS'] = 'FK5' self.action.args.ccddata.header['CTYPE1'] = 'RA---TAN' self.action.args.ccddata.header['CTYPE2'] = 'DEC--TAN' self.action.args.ccddata.header['CTYPE3'] = ('AWAV', 'Air Wavelengths') self.action.args.ccddata.header['CUNIT1'] = ('deg', 'RA units') self.action.args.ccddata.header['CUNIT2'] = ('deg', 'DEC units') self.action.args.ccddata.header['CUNIT3'] = ('Angstrom', 'Wavelength units') self.action.args.ccddata.header['CNAME1'] = ('KCWI RA', 'RA name') self.action.args.ccddata.header['CNAME2'] = ('KCWI DEC', 'DEC name') self.action.args.ccddata.header['CNAME3'] = ('KCWI Wavelength', 'Wavelength name') self.action.args.ccddata.header['CRVAL1'] = (ra, 'RA zeropoint') self.action.args.ccddata.header['CRVAL2'] = (dec, 'DEC zeropoint') self.action.args.ccddata.header['CRVAL3'] = ( geom['wave0out'], 'Wavelength zeropoint') self.action.args.ccddata.header['CRPIX1'] = (crpix1, 'RA reference pixel') self.action.args.ccddata.header['CRPIX2'] = (crpix2, 'DEC reference pixel') self.action.args.ccddata.header['CRPIX3'] = ( crpix3, 'Wavelength reference pixel') self.action.args.ccddata.header['CD1_1'] = ( cd11, 'RA degrees per column pixel') self.action.args.ccddata.header['CD2_1'] = ( cd21, 'DEC degrees per column pixel') self.action.args.ccddata.header['CD1_2'] = ( cd12, 'RA degrees per row pixel') self.action.args.ccddata.header['CD2_2'] = ( cd22, 'DEC degrees per row pixel') self.action.args.ccddata.header['CD3_3'] = ( geom['dwout'], 'Wavelength Angstroms per pixel') self.action.args.ccddata.header['LONPOLE'] = ( 180.0, 'Native longitude of Celestial pole') self.action.args.ccddata.header['LATPOLE'] = ( 0.0, 'Native latitude of Celestial pole') # write out cube self.action.args.ccddata.header['HISTORY'] = log_string self.action.args.ccddata.data = out_cube self.action.args.ccddata.uncertainty.array = out_vube self.action.args.ccddata.mask = out_mube self.action.args.ccddata.flags = out_fube # write out int image kcwi_fits_writer( self.action.args.ccddata, table=self.action.args.table, output_file=self.action.args.name, output_dir=self.config.instrument.output_directory, suffix="icube") self.context.proctab.update_proctab(frame=self.action.args.ccddata, suffix="icube", filename=self.action.args.name) self.context.proctab.write_proctab() # check for obj, sky outputs if obj is not None: out_obj = CCDData(out_oube, meta=self.action.args.ccddata.header, unit=self.action.args.ccddata.unit) kcwi_fits_writer( out_obj, output_file=self.action.args.name, output_dir=self.config.instrument.output_directory, suffix="ocube") if sky is not None: out_sky = CCDData(out_sube, meta=self.action.args.ccddata.header, unit=self.action.args.ccddata.unit) kcwi_fits_writer( out_sky, output_file=self.action.args.name, output_dir=self.config.instrument.output_directory, suffix="scube") # check for dew outputs if dew is not None: out_dew = CCDData(out_dube, meta=self.action.args.ccddata.header, unit=dew.unit) kcwi_fits_writer( out_dew, output_file=self.action.args.name, output_dir=self.config.instrument.output_directory, suffix="dcube") else: self.logger.error("Geometry file not found: %s" % geom_file) self.logger.info(log_string) return self.action.args
def _perform(self): # Header keyword to update key = 'FLATCOR' keycom = 'flat corrected?' # obj, sky obj = None sky = None self.logger.info("Correcting Illumination") if self.action.args.master_flat: mflat = kcwi_fits_reader( os.path.join(self.config.instrument.cwd, self.config.instrument.output_directory, self.action.args.master_flat))[0] # do the correction self.action.args.ccddata.data *= mflat.data # update header keywords self.action.args.ccddata.header[key] = (True, keycom) self.action.args.ccddata.header['MFFILE'] = ( self.action.args.master_flat, "Master flat filename") # check for obj, sky observations if self.action.args.nasmask and self.action.args.numopen > 1: ofn = self.action.args.name objfn = strip_fname(ofn) + '_obj.fits' full_path = os.path.join( self.config.instrument.cwd, self.config.instrument.output_directory, objfn) if os.path.exists(full_path): obj = kcwi_fits_reader(full_path)[0] # correction obj.data *= mflat.data # update header obj.header[key] = (True, keycom) obj.header['MFFILE'] = ( self.action.args.master_flat, 'Master flat filename') else: obj = None skyfn = strip_fname(ofn) + '_sky.fits' full_path = os.path.join( self.config.instrument.cwd, self.config.instrument.output_directory, skyfn) if os.path.exists(full_path): sky = kcwi_fits_reader(full_path)[0] # correction sky.data *= mflat.data # update header sky.header[key] = (True, keycom) sky.header['MFFILE'] = ( self.action.args.master_flat, 'Master flat filename') else: sky = None else: self.logger.error("No master flat found, " "cannot correct illumination.") self.action.args.ccddata.header[key] = (False, keycom) log_string = CorrectIllumination.__module__ self.action.args.ccddata.header['HISTORY'] = log_string # write out intf image kcwi_fits_writer(self.action.args.ccddata, table=self.action.args.table, output_file=self.action.args.name, output_dir=self.config.instrument.output_directory, suffix="intf") self.context.proctab.update_proctab(frame=self.action.args.ccddata, suffix="intf", filename=self.action.args.name) self.context.proctab.write_proctab() # check for obj, sky images if obj is not None: kcwi_fits_writer(obj, output_file=self.action.args.name, output_dir=self.config.instrument.output_directory, suffix="objf") if sky is not None: kcwi_fits_writer(sky, output_file=self.action.args.name, output_dir=self.config.instrument.output_directory, suffix="skyf") self.logger.info(log_string) return self.action.args
def _perform(self): self.logger.info("Making inverse sensitivity curve") suffix = 'invsens' stdname = self.action.args.stdname # get size sz = self.action.args.ccddata.data.shape # default pixel ranges z = np.arange(sz[0]) z0 = 175 z1 = sz[0] - 175 # get exposure time expt = self.action.args.ccddata.header['XPOSURE'] if expt == 0.: self.logger.warning("No exposure time found, setting to 1s") expt = 1. else: self.logger.info("Using exposure time of %.1f" % expt) # get wavelength scale w0 = self.action.args.ccddata.header['CRVAL3'] dw = self.action.args.ccddata.header['CD3_3'] crpixw = self.action.args.ccddata.header['CRPIX3'] # get all good wavelength range wgoo0 = self.action.args.ccddata.header['WAVGOOD0'] if wgoo0 < 3650: wgoo0 = 3650. wgoo1 = self.action.args.ccddata.header['WAVGOOD1'] # get all inclusive wavelength range wall0 = self.action.args.ccddata.header['WAVALL0'] wall1 = self.action.args.ccddata.header['WAVALL1'] # get DAR padding in y pad_y = self.action.args.ccddata.header['DARPADY'] # get sky subtraction status skycor = self.action.args.ccddata.header['SKYCOR'] # get telescope and atm. correction if 'TELESCOP' in self.action.args.ccddata.header: tel = self.action.args.ccddata.header['TELESCOP'] else: tel = 'KeckI' if 'Keck' in tel: area = 760000.0 else: area = -1.0 # compute good y pixel ranges if w0 > 0. and dw > 0. and wgoo0 > 0. and wgoo1 > 0.: z0 = int((wgoo0 - w0) / dw) + 10 z1 = int((wgoo1 - w0) / dw) - 10 # wavelength scale w = w0 + z * dw # good spatial range gy0 = pad_y if pad_y > 1 else 1 gy1 = sz[1] - (pad_y if pad_y > 2 else 2) # log results self.logger.info("Invsen. Pars: Y0, Y1, Z0, Z1, Wav0, Wav1: " "%d, %d, %d, %d, %.3f, %.3f" % (gy0, gy1, z0, z1, w[z0], w[z1])) # central wavelength cwv = self.action.args.cwave # find standard in slices # sum over wavelength tot = np.sum(self.action.args.ccddata.data[z0:z1, gy0:gy1, :], 0) # yy = np.arange(gy1-gy0) + gy0 mxsl = -1. mxsg = 0. # for each slice for i in range(sz[2]): tstd = float(np.nanstd(tot[:, i])) if tstd > mxsg: mxsg = tstd mxsl = i # relevant slices sl0 = (mxsl - 3) if mxsl >= 3 else 0 sl1 = (mxsl + 3) if (mxsl + 3) <= sz[2] - 1 else sz[2] - 1 # get y position of std cy, _ = find_peaks(tot[:, mxsl], height=np.nanmean(tot[:, mxsl])) cy = int(cy[0]) + gy0 # log results self.logger.info("Std lices: max, sl0, sl1, spatial cntrd: " "%d, %d, %d, %.2f" % (mxsl, sl0, sl1, cy)) # get dwave spectrum ofn = self.action.args.name delfn = strip_fname(ofn) + '_dcubed.fits' full_path = os.path.join(self.config.instrument.cwd, self.config.instrument.output_directory, delfn) if os.path.exists(full_path): dew = kcwi_fits_reader(full_path)[0] dwspec = dew.data[:, cy, mxsl] zeros = np.where(dwspec == 0) if len(zeros) > 0: dwspec[zeros] = dw else: dwspec = np.zeros(sz[0]) + dw # copy of input cube scub = self.action.args.ccddata.data.copy() # sky window width in pixels skywin = int(self.config.instrument.psfwid / self.action.args.xbinsize) # do sky subtraction, if needed if not skycor: self.logger.warning("Sky should have been subtraced already") # apply extinction correction ucub = scub.copy() # uncorrected cube kcwi_correct_extin(scub, self.action.args.ccddata.header, logger=self.logger) # get slice spectra y limits sy0 = (cy - skywin) if (cy - skywin) > 0 else 0 sy1 = (cy + skywin) if (cy + skywin) < (sz[1] - 1) else (sz[1] - 1) # sum over y range slspec = np.sum(scub[:, sy0:sy1, :], 1) ulspec = np.sum(ucub[:, sy0:sy1, :], 1) # sum over slices obsspec = np.sum(slspec[:, sl0:sl1], 1) ubsspec = np.sum(ulspec[:, sl0:sl1], 1) # convert to e-/second obsspec /= expt ubsspec /= expt # check for zeros zeros = np.where(obsspec == 0.) if len(zeros) > 0: obsmean = np.nanmean(obsspec) obsspec[zeros] = obsmean # read in standard star spectrum hdul = pf.open(self.action.args.stdfile) swl = hdul[1].data['WAVELENGTH'] sflx = hdul[1].data['FLUX'] sfw = hdul[1].data['FWHM'] hdul.close() # get region of interest sroi = [i for i, v in enumerate(swl) if w[0] <= v <= w[-1]] nsroi = len(sroi) if nsroi <= 0: self.logger.error("No standard wavelengths in common") return self.action.args # expand range after checking for edges if sroi[0] > 0: sroi.insert(0, sroi[0] - 1) nsroi += 1 if sroi[-1] < (len(swl) - 1): sroi.append(sroi[-1] + 1) nsroi += 1 # very sparsely sampled w.r.t. object if nsroi <= 1: self.logger.error("Not enough standard points") return self.action.args self.logger.info("Number of standard point = %d" % nsroi) # how many points? # do_line = nsroi > 20 swl = swl[sroi] sflx = sflx[sroi] sfw = sfw[sroi] fwhm = np.max(sfw) self.logger.info("Reference spectrum FWHM used = %.1f (A)" % fwhm) # resample standard onto our wavelength grid rsint = interp1d(swl, sflx, kind='cubic', fill_value='extrapolate') rsflx = rsint(w) # get effective inverse sensitivity invsen = rsflx / obsspec # convert to photons/s/cm^2/(wl bin = dw) rspho = 5.03411250e7 * rsflx * w * dw # get effective area earea = ubsspec / rspho # correct to native bins earea *= dw / dwspec # Balmer lines blines = [6563., 4861., 4341., 4102., 3970., 3889., 3835.] # default values (for BM) bwid = 0.008 # fractional width to mask ford = 9 # fit order if 'BL' in self.action.args.grating: bwid = 0.004 ford = 7 elif 'BH' in self.action.args.grating: bwid = 0.012 ford = 9 # Adjust for dichroic fraction try: dichroic_fraction = self.action.args.ccddata.header['DICHFRAC'] except KeyError: dichroic_fraction = 1. ford = int(ford * dichroic_fraction) if ford < 3: ford = 3 self.logger.info("Fitting Invsens with polynomial order %d" % ford) bwids = [bl * bwid for bl in blines] # fit inverse sensitivity and effective area # get initial region of interest wl_good = [i for i, v in enumerate(w) if wgoo0 <= v <= wgoo1] nwl_good = len(wl_good) if nwl_good <= 0: self.logger.error("No good wavelengths to fit") return self.action.args wlm0 = wgoo0 wlm1 = wgoo1 # interactively set wavelength limits if self.config.instrument.plot_level >= 1: yran = [np.min(obsspec), np.max(obsspec)] source = ColumnDataSource(data=dict(x=w, y=obsspec)) done = False while not done: p = figure(tooltips=[("x", "@x{0,0.0}"), ("y", "@y{0,0.0}")], title=self.action.args.plotlabel + ' Obs Spec', x_axis_label='Wave (A)', y_axis_label='Intensity (e-)', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.line('x', 'y', line_color='black', source=source) p.line([wgoo0, wgoo0], yran, line_color='green', legend_label='WAVGOOD') p.line([wgoo1, wgoo1], yran, line_color='green') p.line([wlm0, wlm0], yran, line_color='blue', legend_label='LIMITS') p.line([wlm1, wlm1], yran, line_color='blue') p.line([cwv, cwv], yran, line_color='red', legend_label='CWAV') set_plot_lims(p, xlim=[wall0, wall1], ylim=yran) bokeh_plot(p, self.context.bokeh_session) print("WL limits: %.1f - %.1f" % (wlm0, wlm1)) qstr = input("New? <float> <float>, <cr> - done: ") if len(qstr) <= 0: done = True else: try: wlm0 = float(qstr.split()[0]) wlm1 = float(qstr.split()[1]) if wlm1 < wlm0 or wlm0 < wall0 or wlm1 > wall1: wlm0 = wgoo0 wlm1 = wgoo1 print("range/order error, try again") except (IndexError, ValueError): wlm0 = wgoo0 wlm1 = wgoo1 print("format error, try again") # update region of interest wl_good = [i for i, v in enumerate(w) if wlm0 <= v <= wlm1] nwl_good = len(wl_good) # END: interactively set wavelength limits # Now interactively identify lines if self.config.instrument.plot_level >= 1: yran = [np.min(obsspec), np.max(obsspec)] # source = ColumnDataSource(data=dict(x=w, y=obsspec)) done = False while not done: p = figure(tooltips=[("x", "@x{0.0}"), ("y", "@y{0.0}")], title=self.action.args.plotlabel + ' Obs Spec', x_axis_label='Wave (A)', y_axis_label='Intensity (e-)', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.line(w, obsspec, line_color='black') p.line([wgoo0, wgoo0], yran, line_color='green', legend_label='WAVGOOD') p.line([wgoo1, wgoo1], yran, line_color='green') p.line([wlm0, wlm0], yran, line_color='blue', legend_label='LIMITS') p.line([wlm1, wlm1], yran, line_color='blue') p.line([cwv, cwv], yran, line_color='red', legend_label='CWAV') for il, bl in enumerate(blines): if wall0 < bl < wall1: p.line([bl, bl], yran, line_color='orange') p.line([bl - bwids[il], bl - bwids[il]], yran, line_color='orange', line_dash='dashed') p.line([bl + bwids[il], bl + bwids[il]], yran, line_color='orange', line_dash='dashed') set_plot_lims(p, xlim=[wall0, wall1], ylim=yran) bokeh_plot(p, self.context.bokeh_session) qstr = input("New lines? <float> [<float>] ... (A), " "<cr> - done: ") if len(qstr) <= 0: done = True else: for lstr in qstr.split(): try: new_line = float(lstr) except ValueError: print("bad line: %s" % lstr) continue if wlm0 < new_line < wlm1: blines.append(new_line) bwids.append(bwid * new_line) else: print("line outside range: %s" % lstr) # END: interactively identify lines # set up fitting vectors, flux, waves, measure errors sf = invsen[wl_good] # dependent variable af = earea[wl_good] # effective area wf = w[wl_good] # independent variable bf = obsspec[wl_good] # input electrons rf = rsflx[wl_good] # reference flux mw = np.ones(nwl_good, dtype=float) # weights use = np.ones(nwl_good, dtype=int) # toggles for usage # loop over Balmer lines for il, bl in enumerate(blines): roi = [ i for i, v in enumerate(wf) if (bl - bwids[il]) <= v <= (bl + bwids[il]) ] nroi = len(roi) if nroi > 0: use[roi] = 0 self.logger.info("Masking line at %.1f +- %.4f (A)" % (bl, bwids[il])) # ignore bad points by setting large errors mf = [] ef = [] ww = [] used = [] not_used = [] for i in range(len(use)): if use[i] == 1: used.append(i) else: mw[i] = 1.e-9 mf.append(sf[i]) ef.append(100. * af[i] / area) ww.append(wf[i]) not_used.append(i) # initial polynomial fit of inverse sensitivity wf0 = np.min(wf) res = np.polyfit(wf - wf0, sf, deg=ford, w=mw) finvsen = np.polyval(res, w - wf0) sinvsen = np.polyval(res, wf - wf0) calspec = obsspec * finvsen scalspec = bf * sinvsen # initial polynomial fit of effective area res = np.polyfit(wf - wf0, af, ford, w=mw) fearea = np.polyval(res, w - wf0) # calculate residuals resid = 100.0 * (scalspec - rf) / rf if len(not_used) > 0: rbad = resid[not_used] else: rbad = None rsd_mean = float(np.nanmean(resid[used])) rsd_stdv = float(np.nanstd(resid[used])) self.logger.info("Calibration residuals = %f +- %f %%" % (rsd_mean, rsd_stdv)) # plots peff = None pivs = None pcal = None prsd = None # interactively adjust fit if self.config.instrument.plot_level >= 1: done = False while not done: yran = [np.min(100. * af / area), np.max(100. * af / area)] effmax = np.nanmax(100. * fearea / area) effmean = np.nanmean(100. * fearea / area) peff = figure(title=self.action.args.plotlabel + ' Efficiency', x_axis_label='Wave (A)', y_axis_label='Effective Efficiency (%)', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) peff.line(wf, 100. * af / area, line_color='black', legend_label='Data') peff.line(w, 100. * fearea / area, line_color='red', legend_label='Fit') peff.scatter(ww, ef, marker='x', legend_label='Rej') peff.line([wlm0, wlm0], yran, line_color='green', legend_label='WL lims') peff.line([wlm1, wlm1], yran, line_color='green') peff.line([wall0, wall1], [effmax, effmax], line_color='black', line_dash='dashed') peff.line([wall0, wall1], [effmean, effmean], line_color='black', line_dash='dashdot') set_plot_lims(peff, xlim=[wall0, wall1], ylim=yran) bokeh_plot(peff, self.context.bokeh_session) if self.config.instrument.plot_level >= 1: input("Next? <cr>: ") else: time.sleep(2. * self.config.instrument.plot_pause) yran = [np.min(sf), np.max(sf)] pivs = figure(title=self.action.args.plotlabel + ' Inverse sensitivity', x_axis_label='Wave (A)', y_axis_label='Invserse Sensitivity (Flux/e-/s)', y_axis_type='log', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) pivs.line(wf, sf, line_color='black', legend_label='Data') pivs.line(w, finvsen, line_color='red', legend_label='Fit') pivs.scatter(ww, mf, marker='x', legend_label='Rej') pivs.line([wlm0, wlm0], yran, line_color='green', legend_label='WL lims') pivs.line([wlm1, wlm1], yran, line_color='green') set_plot_lims(pivs, xlim=[wall0, wall1], ylim=yran) bokeh_plot(pivs, self.context.bokeh_session) if self.config.instrument.plot_level >= 1: input("Next? <cr>: ") else: time.sleep(2. * self.config.instrument.plot_pause) yran = [np.min(calspec[wl_good]), np.max(calspec[wl_good])] pcal = figure(title=self.action.args.plotlabel + ' Calibrated', x_axis_label='Wave (A)', y_axis_label='Flux (ergs/s/cm^2/A)', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) pcal.line(w, calspec, line_color='black', legend_label='Obs') pcal.line(w, rsflx, line_color='red', legend_label='Ref') pcal.line([wlm0, wlm0], yran, line_color='green', legend_label='WL lims') pcal.line([wlm1, wlm1], yran, line_color='green') set_plot_lims(pcal, xlim=[wall0, wall1], ylim=yran) bokeh_plot(pcal, self.context.bokeh_session) if self.config.instrument.plot_level >= 1: input("Next? <cr>: ") else: time.sleep(2. * self.config.instrument.plot_pause) yran = [np.min(resid), np.max(resid)] prsd = figure(title=self.action.args.plotlabel + ' Residuals = %.1f +- %.1f (%%)' % (rsd_mean, rsd_stdv), x_axis_label='Wave (A)', y_axis_label='Obs - Ref / Ref (%)', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) prsd.line(wf, resid, line_color='black', legend_label='Obs - Ref (%)') if len(not_used) > 0: prsd.scatter(ww, rbad, marker='x', legend_label='Rej') prsd.line([wlm0, wlm0], yran, line_color='green', legend_label='WL lims') prsd.line([wlm1, wlm1], yran, line_color='green') prsd.line([wall0, wall1], [rsd_mean, rsd_mean], line_color='red') prsd.line([wall0, wall1], [rsd_mean + rsd_stdv, rsd_mean + rsd_stdv], line_color='black', line_dash='dashed') prsd.line([wall0, wall1], [rsd_mean - rsd_stdv, rsd_mean - rsd_stdv], line_color='black', line_dash='dashed') set_plot_lims(prsd, xlim=[wall0, wall1], ylim=yran) bokeh_plot(prsd, self.context.bokeh_session) if self.config.instrument.plot_level >= 1: qstr = input("Current fit order = %d, " "New fit order? <int>, <cr> - done: " % ford) if len(qstr) <= 0: done = True else: try: ford = int(qstr) # update fit of inverse sensitivity res = np.polyfit(wf - wf0, sf, deg=ford, w=mw) finvsen = np.polyval(res, w - wf0) sinvsen = np.polyval(res, wf - wf0) calspec = obsspec * finvsen scalspec = bf * sinvsen # update polynomial fit of effective area res = np.polyfit(wf - wf0, af, ford, w=mw) fearea = np.polyval(res, w - wf0) # re-calculate residuals resid = 100.0 * (scalspec - rf) / rf if len(not_used) > 0: rbad = resid[not_used] else: rbad = None rsd_mean = float(np.nanmean(resid[used])) rsd_stdv = float(np.nanstd(resid[used])) self.logger.info( "Calibration residuals = %f +- %f %%" % (rsd_mean, rsd_stdv)) except ValueError: print("Bad fit order, try again") else: done = True time.sleep(2. * self.config.instrument.plot_pause) # log results effmax = float(np.nanmax(100. * fearea / area)) effmean = float(np.nanmean(100. * fearea / area)) self.logger.info("Peak, mean efficiency (%%): %.1f, %.1f" % (effmax, effmean)) self.logger.info("Fit order = %d" % ford) # output plots pfname = "std_%05d_%s_%s_%s_%d" % ( self.action.args.ccddata.header['FRAMENO'], stdname, self.action.args.grating.strip(), self.action.args.ifuname.strip(), int(self.action.args.cwave)) # Save plots save_plot(peff, filename=pfname + '_eff.png') # Efficiency save_plot(pivs, filename=pfname + '_invsens.png') # Inv. Sens. save_plot(prsd, filename=pfname + '_resid.png') # Residuals save_plot(pcal, filename=pfname + '_cal.png') # Calibrated log_string = MakeInvsens.__module__ self.action.args.ccddata.header['HISTORY'] = log_string self.logger.info(log_string) # write out effective inverse sensitivity # update inverse sensitivity header hdr = self.action.args.ccddata.header.copy() hdr['HISTORY'] = log_string hdr['INVSENS'] = (True, 'effective inv. sens. spectrum?') hdr['INVSW0'] = (w[z0], 'low wavelength for eff. inv. sens.') hdr['INVSW1'] = (w[z1], 'high wavelength for eff. inv. sens.') hdr['INVSZ0'] = (z0, 'low wave pixel for eff. inv. sens.') hdr['INVSZ1'] = (z1, 'high wave pixel for eff. inv. sens.') hdr['INVSY0'] = (gy0, 'low spatial pixel for eff. inv. sens.') hdr['INVSY1'] = (gy1, 'high spatial pixel for eff. inv. sens.') hdr['INVSLMX'] = (mxsl, 'brightest std star slice') hdr['INVSL0'] = (sl0, 'lowest std star slice summed') hdr['INVSL1'] = (sl1, 'highest std star slice summed') hdr['INVSLY'] = (cy, 'spatial pixel position of std within slice') hdr['INVFW0'] = (wlm0, 'low wavelength for fits') hdr['INVFW1'] = (wlm1, 'high wavelength for fits') hdr['INVFORD'] = (ford, 'fit order') hdr['EXPTIME'] = (1., 'effective exposure time (seconds)') hdr['XPOSURE'] = (1., 'effective exposure time (seconds)') # remove old WCS del hdr['RADESYS'] del hdr['EQUINOX'] del hdr['LONPOLE'] del hdr['LATPOLE'] del hdr['NAXIS2'] if 'NAXIS3' in hdr: del hdr['NAXIS3'] del hdr['CTYPE1'] del hdr['CTYPE2'] del hdr['CTYPE3'] del hdr['CUNIT1'] del hdr['CUNIT2'] del hdr['CUNIT3'] del hdr['CNAME1'] del hdr['CNAME2'] del hdr['CNAME3'] del hdr['CRVAL1'] del hdr['CRVAL2'] del hdr['CRVAL3'] del hdr['CRPIX1'] del hdr['CRPIX2'] del hdr['CRPIX3'] del hdr['CD1_1'] del hdr['CD1_2'] del hdr['CD2_1'] del hdr['CD2_2'] del hdr['CD3_3'] # set wavelength axis WCS values hdr['WCSDIM'] = 1 hdr['CTYPE1'] = ('AWAV', 'Air Wavelengths') hdr['CUNIT1'] = ('Angstrom', 'Wavelength units') hdr['CNAME1'] = ('KCWI INVSENS Wavelength', 'Wavelength name') hdr['CRVAL1'] = (w0, 'Wavelength zeropoint') hdr['CRPIX1'] = (crpixw, 'Wavelength reference pixel') hdr['CDELT1'] = (dw, 'Wavelength Angstroms per pixel') ofn = self.action.args.name invsname = strip_fname(ofn) + '_' + suffix + '.fits' eaname = strip_fname(ofn) + '_ea.fits' # set units invsens_u = u.erg / (u.angstrom * u.cm**2 * u.s * u.electron) # output inverse sensitivity out_invsens = CCDData(np.asarray([invsen, finvsen, obsspec]), meta=hdr, unit=invsens_u) kcwi_fits_writer(out_invsens, output_file=invsname, output_dir=self.config.instrument.output_directory) self.context.proctab.update_proctab(frame=out_invsens, suffix=suffix, newtype='INVSENS', filename=self.action.args.name) # output effective area ea_u = u.cm**2 / u.angstrom out_ea = CCDData(np.asarray([earea, fearea]), meta=hdr, unit=ea_u) kcwi_fits_writer(out_ea, output_file=eaname, output_dir=self.config.instrument.output_directory) self.context.proctab.update_proctab(frame=out_ea, suffix='ea', newtype='EAREA', filename=self.action.args.name) return self.action.args
def _perform(self): """ Returns an Argument() with the parameters that depends on this operation """ self.logger.info("Creating master sky") suffix = self.action.args.new_type.lower() # get root for maps tab = self.context.proctab.search_proctab( frame=self.action.args.ccddata, target_type='ARCLAMP', target_group=self.action.args.groupid) if len(tab) <= 0: self.logger.error("Geometry not solved!") return self.action.args groot = strip_fname(tab['filename'][0]) # Wavelength map image wmf = groot + '_wavemap.fits' self.logger.info("Reading image: %s" % wmf) wavemap = kcwi_fits_reader(os.path.join( self.config.instrument.cwd, 'redux', wmf))[0] # Slice map image slf = groot + '_slicemap.fits' self.logger.info("Reading image: %s" % slf) slicemap = kcwi_fits_reader(os.path.join( self.config.instrument.cwd, 'redux', slf))[0] # Position map image pof = groot + '_posmap.fits' self.logger.info("Reading image: %s" % pof) posmap = kcwi_fits_reader(os.path.join( self.config.instrument.cwd, 'redux', pof))[0] posmax = np.nanmax(posmap.data) posbuf = int(10. / self.action.args.xbinsize) # wavelength region wavegood0 = wavemap.header['WAVGOOD0'] wavegood1 = wavemap.header['WAVGOOD1'] waveall0 = wavemap.header['WAVALL0'] waveall1 = wavemap.header['WAVALL1'] # get image size sm_sz = self.action.args.ccddata.data.shape # sky masking # default is no masking (True = mask, False = don't mask) binary_mask = np.zeros(sm_sz, dtype=bool) # was sky masking requested? if self.action.args.skymask: if os.path.exists(self.action.args.skymask): self.logger.info("Reading sky mask file: %s" % self.action.args.skymask) hdul = fits.open(self.action.args.skymask) binary_mask = hdul[0].data # verify size match bm_sz = binary_mask.shape if bm_sz[0] != sm_sz[0] or bm_sz[1] != sm_sz[1]: self.logger.warning("Sky mask size mis-match: " "masking disabled") binary_mask = np.zeros(sm_sz, dtype=bool) else: self.logger.warning("Sky mask image not found: %s" % self.action.args.skymask) # count masked pixels tmsk = len(np.nonzero(np.where(binary_mask.flat, True, False))[0]) self.logger.info("Number of pixels masked = %d" % tmsk) finiteflux = np.isfinite(self.action.args.ccddata.data.flat) # get un-masked points mapped to exposed regions on CCD # handle dichroic bad region if self.action.args.dich: if self.action.args.camera == 0: # Blue q = [i for i, v in enumerate(slicemap.data.flat) if 0 <= v <= 23 and posbuf < posmap.data.flat[i] < (posmax - posbuf) and waveall0 <= wavemap.data.flat[i] <= waveall1 and not (v > 20 and wavemap.data.flat[i] > 5600.) and finiteflux[i] and not binary_mask.flat[i]] else: # Red q = [i for i, v in enumerate(slicemap.data.flat) if 0 <= v <= 23 and posbuf < posmap.data.flat[i] < (posmax - posbuf) and waveall0 <= wavemap.data.flat[i] <= waveall1 and not (v > 20 and wavemap.data.flat[i] < 5600.) and finiteflux[i] and not binary_mask.flat[i]] else: q = [i for i, v in enumerate(slicemap.data.flat) if 0 <= v <= 23 and posbuf < posmap.data.flat[i] < (posmax - posbuf) and waveall0 <= wavemap.data.flat[i] <= waveall1 and finiteflux[i] and not binary_mask.flat[i]] # get all points mapped to exposed regions on the CCD (for output) qo = [i for i, v in enumerate(slicemap.data.flat) if 0 <= v <= 23 and posmap.data.flat[i] >= 0 and waveall0 <= wavemap.data.flat[i] <= waveall1 and finiteflux[i]] # extract relevant image values fluxes = self.action.args.ccddata.data.flat[q] # relevant wavelengths waves = wavemap.data.flat[q] self.logger.info("Number of fit waves = %d" % len(waves)) # keep output wavelengths owaves = wavemap.data.flat[qo] self.logger.info("Number of output waves = %d" % len(owaves)) # sort on wavelength s = np.argsort(waves) waves = waves[s] fluxes = fluxes[s] # knots per pixel knotspp = self.config.instrument.KNOTSPP n = int(sm_sz[0] * knotspp) # calculate break points for b splines bkpt = np.min(waves) + np.arange(n+1) * \ (np.max(waves) - np.min(waves)) / n # log self.logger.info("Nknots = %d, min = %.2f, max = %.2f (A)" % (n, np.min(bkpt), np.max(bkpt))) # do bspline fit sft0, gmask = Bspline.iterfit(waves, fluxes, fullbkpt=bkpt, upper=1, lower=1) gp = [i for i, v in enumerate(gmask) if v] yfit1, _ = sft0.value(waves) self.logger.info("Number of good points = %d" % len(gp)) # check result if np.max(yfit1) < 0: self.logger.warning("B-spline failure") if n > 2000: if n == 5000: n = 2000 if n == 8000: n = 5000 # calculate breakpoints bkpt = np.min(waves) + np.arange(n + 1) * \ (np.max(waves) - np.min(waves)) / n # log self.logger.info("Nknots = %d, min = %.2f, max = %.2f (A)" % (n, np.min(bkpt), np.max(bkpt))) # do bspline fit sft0, gmask = Bspline.iterfit(waves, fluxes, fullbkpt=bkpt, upper=1, lower=1) yfit1, _ = sft0.value(waves) if np.max(yfit1) <= 0: self.logger.warning("B-spline final failure, sky is zero") # get values at original wavelengths yfit, _ = sft0.value(owaves) # for plotting gwaves = waves[gp] gfluxes = fluxes[gp] npts = len(gwaves) stride = int(npts / 8000.) xplt = gwaves[::stride] yplt = gfluxes[::stride] fplt, _ = sft0.value(xplt) yrng = [np.min(yplt), np.max(yplt)] self.logger.info("Stride = %d" % stride) # plot, if requested if self.config.instrument.plot_level >= 1: # output filename stub skyfnam = "sky_%05d_%s_%s_%s" % \ (self.action.args.ccddata.header['FRAMENO'], self.action.args.illum, self.action.args.grating, self.action.args.ifuname) p = figure( title=self.action.args.plotlabel + ' Master Sky', x_axis_label='Wave (A)', y_axis_label='Flux (e-)', plot_width=self.config.instrument.plot_width, plot_height=self.config.instrument.plot_height) p.circle(xplt, yplt, size=1, line_alpha=0., fill_color='purple', legend_label='Data') p.line(xplt, fplt, line_color='red', legend_label='Fit') p.line([wavegood0, wavegood0], yrng, line_color='green') p.line([wavegood1, wavegood1], yrng, line_color='green') p.y_range.start = yrng[0] p.y_range.end = yrng[1] bokeh_plot(p, self.context.bokeh_session) if self.config.instrument.plot_level >= 2: input("Next? <cr>: ") else: time.sleep(self.config.instrument.plot_pause) save_plot(p, filename=skyfnam+".png") # create sky image sky = np.zeros(self.action.args.ccddata.data.shape, dtype=float) sky.flat[qo] = yfit # store original data, header img = self.action.args.ccddata.data hdr = self.action.args.ccddata.header.copy() self.action.args.ccddata.data = sky # get master sky output name ofn_full = self.action.args.name ofn = os.path.basename(ofn_full) msname = strip_fname(ofn) + '_' + suffix + '.fits' log_string = MakeMasterSky.__module__ self.action.args.ccddata.header['IMTYPE'] = self.action.args.new_type self.action.args.ccddata.header['HISTORY'] = log_string self.action.args.ccddata.header['SKYMODEL'] = (True, 'sky model image?') self.action.args.ccddata.header['SKYIMAGE'] = \ (ofn, 'image used for sky model') if tmsk > 0: self.action.args.ccddata.header['SKYMSK'] = (True, 'was sky masked?') # self.action.args.ccddata.header['SKYMSKF'] = (skymf, # 'sky mask file') else: self.action.args.ccddata.header['SKYMSK'] = (False, 'was sky masked?') self.action.args.ccddata.header['WAVMAPF'] = wmf self.action.args.ccddata.header['SLIMAPF'] = slf self.action.args.ccddata.header['POSMAPF'] = pof # output master sky kcwi_fits_writer(self.action.args.ccddata, output_file=msname, output_dir=self.config.instrument.output_directory) self.context.proctab.update_proctab(frame=self.action.args.ccddata, suffix=suffix, newtype=self.action.args.new_type, filename=self.action.args.name) self.context.proctab.write_proctab() # restore original image self.action.args.ccddata.data = img self.action.args.ccddata.header = hdr self.logger.info(log_string) return self.action.args