for ord in range(0, n_ord * 2, 2): for coeff in range(ordfit): i = (ord * n_coeff) + coeff poly_c[i % n_coeff, i // n_coeff] = hdr_wave["LOCTR" + str(i)] poly_c = poly_c[:, poly_c[0, :] != 0] datac = (pyfits.getdata(slope_file)) hdr = pyfits.getheader(slope_file) p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p) # convert NaN to zeros data0 = np.where(~np.isfinite(data), np.zeros_like(data), data) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) data1 = spirouImage.ResizeImage(p, data0, **bkwargs) # dxmap that will contain the dx per pixel # if the file does not exist, we fill the map # with zeros if glob.glob(outname) != []: print(outname + ' exists... we use its values as a starting point')
def main(night_name=None, files=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) # force plotting to 1 p['DRS_PLOT'] = 1 # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data data, hdr, nx, ny = spirouImage.ReadImage(p) # ---------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- data = spirouImage.FixNonPreProcess(p, data, filename=p['FITSFILENAME']) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # now change the value of sigdet if require if p['IC_EXT_SIGDET'] > 0: p['SIGDET'] = float(p['IC_EXT_SIGDET']) # ---------------------------------------------------------------------- # Resize image # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- data = spirouImage.ConvertToADU(spirouImage.FlipImage(p, data), p=p) # convert NaN to zeros data2 = np.where(~np.isfinite(data), np.zeros_like(data), data) # resize image # bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], # ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], # getshape=False) # data2 = spirouImage.ResizeImage(data0, **bkwargs) # log change in data size # WLOG(p, '', ('Image format changed to ' # '{0}x{1}').format(*data2.shape[::-1])) # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- # get the number of bad pixels n_bad_pix = np.nansum(data2 == 0) n_bad_pix_frac = n_bad_pix * 100 / np.product(data2.shape) # Log number wmsg = 'Nb dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) satseuil = 64536. col = 2100 seuil = 10000 slice0 = 5 plt.ion() plt.clf() plt.imshow(data2, origin='lower', clim=(1., seuil)) plt.colorbar() plt.axis([0, nx, 0, ny]) plt.plot(np.ones(4096) * col, np.arange(4096), c='red') plt.figure() plt.clf() centpart = data2[:, col - slice0:col + slice0] # centpart = data2[col - slice:col + slice,:] # weights = np.where((centpart < satseuil) & (centpart > 0), 1, 0.0001) # y = np.average(centpart, axis=1, weights=weights) ## weighted average y = np.nanmedian(centpart, axis=1) # y=average(centpart,axis=1,weights=where((centpart>0),1,0.0001)) # ## weighted average plt.plot(np.arange(ny), y) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p, outputs=None) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, ufile=None, xsize=None, ysize=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # need custom args (to accept full path or wild card if ufile is None or xsize is None or ysize is None: pos = [0, 1, 2] names, types = ['ufile', 'xsize', 'ysize'], [str, int, int] customargs = spirouStartup.GetCustomFromRuntime(p, pos, types, names, last_multi=True) else: customargs = dict(ufile=ufile, xsize=xsize, ysize=ysize) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs) # add constants not currently in constants file # ------------------------------------------------------------------ # Read image file # ------------------------------------------------------------------ # read the image data rout = spirouImage.ReadImage(p, filename=p['UFILE']) image, hdr, nx, ny = rout # ---------------------------------------------------------------------- # un-Resize image # ---------------------------------------------------------------------- # create an array of given size size = np.product([p['YSIZE'], p['XSIZE']]) newimage = np.repeat(np.nan, size).reshape(p['YSIZE'], p['XSIZE']) # insert image at given pixels xlow, xhigh = p['IC_CCDX_LOW'], p['IC_CCDX_HIGH'] ylow, yhigh = p['IC_CCDY_LOW'], p['IC_CCDY_HIGH'] newimage[ylow:yhigh, xlow:xhigh] = image # rotate the image newimage = spirouImage.FlipImage(p, newimage) # ------------------------------------------------------------------ # Save rotated image # ------------------------------------------------------------------ # construct rotated file name outfits = ufile.replace('.fits', '_old.fits') outfitsname = os.path.split(outfits)[-1] # log that we are saving rotated image WLOG(p, '', 'Saving Rotated Image in ' + outfitsname) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # write to file p = spirouImage.WriteImage(p, outfits, newimage, hdict) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- wmsg = 'Recipe {0} has been successfully completed' WLOG(p, 'info', wmsg.format(p['PROGRAM'])) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): """ cal_SLIT_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_SLIT_spirou.py [night_directory] [files] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) p = spirouStartup.InitialFileSetup(p, calibdb=True) # set the fiber type p['FIB_TYP'] = 'AB' p.set_source('FIB_TYP', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add') # ---------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- data = spirouImage.FixNonPreProcess(p, data) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # ---------------------------------------------------------------------- # Correction of DARK # ---------------------------------------------------------------------- p, datac = spirouImage.CorrectForDark(p, data, hdr) # ---------------------------------------------------------------------- # Resize image # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p) # convert NaN to zeros data0 = np.where(~np.isfinite(data), np.zeros_like(data), data) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) data2 = spirouImage.ResizeImage(p, data0, **bkwargs) # log change in data size WLOG(p, '', ('Image format changed to ' '{0}x{1}').format(*data2.shape)) # ---------------------------------------------------------------------- # Correct for the BADPIX mask (set all bad pixels to zero) # ---------------------------------------------------------------------- p, data2 = spirouImage.CorrectForBadPix(p, data2, hdr) # ---------------------------------------------------------------------- # Background computation # ---------------------------------------------------------------------- if p['IC_DO_BKGR_SUBTRACTION']: # log that we are doing background measurement WLOG(p, '', 'Doing background measurement on raw frame') # get the bkgr measurement bargs = [p, data2, hdr] # background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs) p, background = spirouBACK.MeasureBackgroundMap(*bargs) else: background = np.zeros_like(data2) p['BKGRDFILE'] = 'None' p.set_source('BKGRDFILE', __NAME__ + '.main()') # correct data2 with background data2 = data2 - background # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- # get the number of bad pixels n_bad_pix = np.nansum(~np.isfinite(data2)) n_bad_pix_frac = n_bad_pix * 100 / np.product(data2.shape) # Log number wmsg = 'Nb dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- loc = ParamDict() # ---------------------------------------------------------------------- # Loop around fiber types # ---------------------------------------------------------------------- # set fiber p['FIBER'] = p['FIB_TYP'] # ------------------------------------------------------------------ # Get localisation coefficients # ------------------------------------------------------------------ # original there is a loop but it is not used --> removed p = spirouImage.FiberParams(p, p['FIBER'], merge=True) # get localisation fit coefficients p, loc = spirouLOCOR.GetCoeffs(p, hdr, loc) # ------------------------------------------------------------------ # Calculating the tilt # ------------------------------------------------------------------ # get the tilt by extracting the AB fibers and correlating them loc = spirouImage.GetTilt(p, loc, data2) # fit the tilt with a polynomial loc = spirouImage.FitTilt(p, loc) # log the tilt dispersion wmsg = 'Tilt dispersion = {0:.3f} deg' WLOG(p, 'info', wmsg.format(loc['RMS_TILT'])) # ------------------------------------------------------------------ # Plotting # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # plots setup: start interactive plot sPlt.start_interactive_session(p) # plot image with selected order shown sPlt.slit_sorder_plot(p, loc, data2) # plot slit tilt angle and fit sPlt.slit_tilt_angle_and_fit_plot(p, loc) # end interactive section sPlt.end_interactive_session(p) # ------------------------------------------------------------------ # Replace tilt by the global fit # ------------------------------------------------------------------ loc['TILT'] = loc['YFIT_TILT'] oldsource = loc.get_source('tilt') loc.set_source('TILT', oldsource + '+{0}/main()'.format(__NAME__)) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # check that tilt rms is below required if loc['RMS_TILT'] > p['QC_SLIT_RMS']: # add failed message to fail message list fmsg = 'abnormal RMS of SLIT angle ({0:.2f} > {1:.2f} deg)' fail_msg.append(fmsg.format(loc['RMS_TILT'], p['QC_SLIT_RMS'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(loc['RMS_TILT']) qc_names.append('RMS_TILT') qc_logic.append('RMS_TILT > {0:.2f}'.format(p['QC_SLIT_RMS'])) # ---------------------------------------------------------------------- # check that tilt is less than max tilt required max_tilt = np.max(loc['TILT']) if max_tilt > p['QC_SLIT_MAX']: # add failed message to fail message list fmsg = 'abnormal SLIT angle ({0:.2f} > {1:.2f} deg)' fail_msg.append(fmsg.format(max_tilt, p['QC_SLIT_MAX'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(max_tilt) qc_names.append('max_tilt') qc_logic.append('max_tilt > {0:.2f}'.format(p['QC_SLIT_MAX'])) # ---------------------------------------------------------------------- # check that tilt is greater than min tilt required min_tilt = np.min(loc['TILT']) if min_tilt < p['QC_SLIT_MIN']: # add failed message to fail message list fmsg = 'abnormal SLIT angle ({0:.2f} < {1:.2f} deg)' fail_msg.append(fmsg.format(max_tilt, p['QC_SLIT_MIN'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(min_tilt) qc_names.append('min_tilt') qc_logic.append('min_tilt > {0:.2f}'.format(p['QC_SLIT_MIN'])) # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ---------------------------------------------------------------------- # Save and record of tilt table # ---------------------------------------------------------------------- # copy the tilt along the orders tiltima = np.ones((int(loc['NUMBER_ORDERS']/2), data2.shape[1])) tiltima *= loc['TILT'][:, None] # get the raw tilt file name raw_tilt_file = os.path.basename(p['FITSFILENAME']) # construct file name and path tiltfits, tag = spirouConfig.Constants.SLIT_TILT_FILE(p) tiltfitsname = os.path.basename(tiltfits) # Log that we are saving tilt file wmsg = 'Saving tilt information in file: {0}' WLOG(p, '', wmsg.format(tiltfitsname)) # Copy keys from fits file hdict = spirouImage.CopyOriginalKeys(hdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBACK'], value=p['BKGRDFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # add qc parameters # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # add tilt parameters as 1d list hdict = spirouImage.AddKey1DList(p, hdict, p['KW_TILT'], values=loc['TILT']) # write tilt file to file p = spirouImage.WriteImage(p, tiltfits, tiltima, hdict) # ---------------------------------------------------------------------- # Update the calibration data base # ---------------------------------------------------------------------- if p['QC']: keydb = 'TILT' # copy localisation file to the calibDB folder spirouDB.PutCalibFile(p, tiltfits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, tiltfitsname, hdr) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None, fiber_type=None, **kwargs): """ cal_DRIFT_E2DS_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_DRIFT_E2DS_spirou.py [night_directory] [files] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :param fiber_type: string, if None does all fiber types (defined in constants_SPIROU FIBER_TYPES (default is AB, A, B, C if defined then only does this fiber type (but must be in FIBER_TYPES) :param kwargs: any keyword to overwrite constant in param dict "p" :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) p = spirouStartup.InitialFileSetup(p, calibdb=True) # deal with fiber type if fiber_type is None: fiber_type = p['FIBER_TYPES'] if type(fiber_type) == str: if fiber_type.upper() == 'ALL': fiber_type = p['FIBER_TYPES'] elif fiber_type in p['FIBER_TYPES']: fiber_type = [fiber_type] else: emsg = 'fiber_type="{0}" not understood' WLOG(p, 'error', emsg.format(fiber_type)) # set fiber type p['FIB_TYPE'] = fiber_type p.set_source('FIB_TYPE', __NAME__ + '__main__()') # Overwrite keys from source for kwarg in kwargs: p[kwarg] = kwargs[kwarg] # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add') # ---------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- data = spirouImage.FixNonPreProcess(p, data) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # now change the value of sigdet if require if p['IC_EXT_SIGDET'] > 0: p['SIGDET'] = float(p['IC_EXT_SIGDET']) # get DPRTYPE from header (Will have it if valid) p = spirouImage.ReadParam(p, hdr, 'KW_DPRTYPE', required=False, dtype=str) # check the DPRTYPE is not None if (p['DPRTYPE'] == 'None') or (['DPRTYPE'] is None): emsg1 = 'Error: {0} is not set in header for file {1}' eargs = [p['KW_DPRTYPE'][0], p['FITSFILENAME']] emsg2 = '\tPlease run pre-processing on file.' emsg3 = ('\tIf pre-processing fails or skips file, file is not ' 'currrently as valid DRS fits file.') WLOG(p, 'error', [emsg1.format(*eargs), emsg2, emsg3]) else: p['DPRTYPE'] = p['DPRTYPE'].strip() # ---------------------------------------------------------------------- # Correction of DARK # ---------------------------------------------------------------------- p, datac = spirouImage.CorrectForDark(p, data, hdr) # ---------------------------------------------------------------------- # Resize image # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to ADU data = spirouImage.ConvertToADU(spirouImage.FlipImage(p, datac), p=p) # convert NaN to zeros data0 = np.where(~np.isfinite(data), np.zeros_like(data), data) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) data1 = spirouImage.ResizeImage(p, data0, **bkwargs) # log change in data size wmsg = 'Image format changed to {1}x{0}' WLOG(p, '', wmsg.format(*data1.shape)) # ---------------------------------------------------------------------- # Correct for the BADPIX mask (set all bad pixels to zero) # ---------------------------------------------------------------------- p, data1 = spirouImage.CorrectForBadPix(p, data1, hdr) # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- # get the number of bad pixels n_bad_pix = np.sum(~np.isfinite(data1)) n_bad_pix_frac = n_bad_pix * 100 / np.product(data1.shape) # Log number wmsg = 'Nb dead pixels = {0} / {1:.4f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ---------------------------------------------------------------------- # Get the miny, maxy and max_signal for the central column # ---------------------------------------------------------------------- # get the central column y = data1[p['IC_CENT_COL'], :] # get the min max and max signal using box smoothed approach miny, maxy, max_signal, diff_maxmin = spirouBACK.MeasureMinMaxSignal(p, y) # Log max average flux/pixel wmsg = 'Maximum average flux/pixel in the spectrum: {0:.1f} [ADU]' WLOG(p, 'info', wmsg.format(max_signal / p['NBFRAMES'])) # ---------------------------------------------------------------------- # Background computation # ---------------------------------------------------------------------- if p['IC_DO_BKGR_SUBTRACTION']: # log that we are doing background measurement WLOG(p, '', 'Doing background measurement on raw frame') # get the bkgr measurement bargs = [p, data1, hdr] # background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs) p, background = spirouBACK.MeasureBackgroundMap(*bargs) else: background = np.zeros_like(data1) p['BKGRDFILE'] = 'None' p.set_source('BKGRDFILE', __NAME__ + '.main()') # apply background correction to data (and set to zero where negative) data1 = data1 - background # ---------------------------------------------------------------------- # Read tilt slit angle # ---------------------------------------------------------------------- # define loc storage parameter dictionary loc = ParamDict() # get tilts (if the mode requires it) if p['IC_EXTRACT_TYPE'] not in EXTRACT_SHAPE_TYPES: p, loc['TILT'] = spirouImage.ReadTiltFile(p, hdr) loc.set_source('TILT', __NAME__ + '/main() + /spirouImage.ReadTiltFile') else: loc['TILT'] = None loc.set_source('TILT', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Earth Velocity calculation # ---------------------------------------------------------------------- if p['IC_IMAGE_TYPE'] == 'H4RG': p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, hdr) # ---------------------------------------------------------------------- # Get all fiber data (for all fibers) # ---------------------------------------------------------------------- # TODO: This is temp solution for options 5a and 5b loc_fibers = spirouLOCOR.GetFiberData(p, hdr) # ------------------------------------------------------------------ # Deal with debananafication # ------------------------------------------------------------------ # if mode 4a or 4b we need to straighten in x only if p['IC_EXTRACT_TYPE'] in ['4a', '4b']: # get the shape parameters p, shapem_x = spirouImage.GetShapeX(p, hdr) p, shape_local = spirouImage.GetShapeLocal(p, hdr) # log progress WLOG(p, '', 'Debananafying (straightening) image') # apply shape transforms targs = dict(lin_transform_vect=shape_local, dxmap=shapem_x) data2 = spirouImage.EATransform(data1, **targs) # if mode 5a or 5b we need to straighten in x and y using the # polynomial fits for location elif p['IC_EXTRACT_TYPE'] in ['5a', '5b']: # get the shape parameters p, shapem_x = spirouImage.GetShapeX(p, hdr) p, shapem_y = spirouImage.GetShapeY(p, hdr) p, shape_local = spirouImage.GetShapeLocal(p, hdr) p, fpmaster = spirouImage.GetFPMaster(p, hdr) # get the bad pixel map bkwargs = dict(return_map=True, quiet=True) p, badpix = spirouImage.CorrectForBadPix(p, data1, hdr, **bkwargs) # log progress WLOG(p, '', 'Cleaning image') # clean the image data1 = spirouEXTOR.CleanHotpix(data1, badpix) # log progress WLOG(p, '', 'Debananafying (straightening) image') # apply shape transforms targs = dict(lin_transform_vect=shape_local, dxmap=shapem_x, dymap=shapem_y) data2 = spirouImage.EATransform(data1, **targs) # in any other mode we do not straighten else: data2 = np.array(data1) # ---------------------------------------------------------------------- # Fiber loop # ---------------------------------------------------------------------- # loop around fiber types for fiber in p['FIB_TYPE']: # set fiber p['FIBER'] = fiber p.set_source('FIBER', __NAME__ + '/main()()') # ------------------------------------------------------------------ # Read wavelength solution # ------------------------------------------------------------------ # set source of wave file wsource = __NAME__ + '/main() + /spirouImage.GetWaveSolution' # Force A and B to AB solution if fiber in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = fiber # get wave image wkwargs = dict(hdr=hdr, return_wavemap=True, return_filename=True, return_header=True, fiber=wave_fiber) wout = spirouImage.GetWaveSolution(p, **wkwargs) loc['WAVEPARAMS'], loc['WAVE'], loc['WAVEFILE'] = wout[:3] loc['WAVEHDR'], loc['WSOURCE'] = wout[3:] source_names = ['WAVE', 'WAVEFILE', 'WAVEPARAMS', 'WAVEHDR'] loc.set_sources(source_names, wsource) # get dates loc['WAVE_ACQTIMES'] = spirouDB.GetTimes(p, loc['WAVEHDR']) loc.set_source('WAVE_ACQTIMES', __NAME__ + '.main()') # get the recipe that produced the wave solution if 'WAVECODE' in loc['WAVEHDR']: loc['WAVE_CODE'] = loc['WAVEHDR']['WAVECODE'] else: loc['WAVE_CODE'] = 'UNKNOWN' loc.set_source('WAVE_CODE', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Get WFP keys # ---------------------------------------------------------------------- # Read the WFP keys - if they don't exist set to None and deal # with later p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_DRIFT', name='WFP_DRIFT', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_FWHM', name='WFP_FWHM', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_CONTRAST', name='WFP_CONTRAST', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_MAXCPP', name='WFP_MAXCPP', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_MASK', name='WFP_MASK', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_LINES', name='WFP_LINES', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_TARG_RV', name='WFP_TARG_RV', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_WIDTH', name='WFP_WIDTH', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_STEP', name='WFP_STEP', required=False) # ---------------------------------------------------------------------- # Read Flat file # ---------------------------------------------------------------------- fout = spirouImage.ReadFlatFile(p, hdr, return_header=True) p, loc['FLAT'], flathdr = fout loc.set_source('FLAT', __NAME__ + '/main() + /spirouImage.ReadFlatFile') # get flat extraction mode if p['KW_E2DS_EXTM'][0] in flathdr: flat_ext_mode = flathdr[p['KW_E2DS_EXTM'][0]] else: flat_ext_mode = None # ------------------------------------------------------------------ # Check extraction method is same as flat extraction method # ------------------------------------------------------------------ # get extraction method and function extmethod, extfunc = spirouEXTOR.GetExtMethod(p, p['IC_EXTRACT_TYPE']) if not DEBUG: # compare flat extraction mode to extraction mode spirouEXTOR.CompareExtMethod(p, flat_ext_mode, extmethod, 'FLAT', 'EXTRACTION') # ------------------------------------------------------------------ # Read Blaze file # ------------------------------------------------------------------ p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hdr) blazesource = __NAME__ + '/main() + /spirouImage.ReadBlazeFile' loc.set_source('BLAZE', blazesource) # ------------------------------------------------------------------ # Get fiber specific parameters from loc_fibers # ------------------------------------------------------------------ # get this fibers parameters p = spirouImage.FiberParams(p, p['FIBER'], merge=True) # get localisation parameters for key in loc_fibers[fiber]: loc[key] = loc_fibers[fiber][key] loc.set_source(key, loc_fibers[fiber].sources[key]) # get locofile source p['LOCOFILE'] = loc['LOCOFILE'] p.set_source('LOCOFILE', loc.sources['LOCOFILE']) # get the order_profile order_profile = loc_fibers[fiber]['ORDER_PROFILE'] # ------------------------------------------------------------------ # Set up Extract storage # ------------------------------------------------------------------ # Create array to store extraction (for each order and each pixel # along order) loc['E2DS'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) loc['E2DSFF'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) loc['E2DSLL'] = [] loc['SPE1'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) loc['SPE3'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) loc['SPE4'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) loc['SPE5'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) # Create array to store the signal to noise ratios for each order loc['SNR'] = np.zeros(loc['NUMBER_ORDERS']) # ------------------------------------------------------------------ # Extract orders # ------------------------------------------------------------------ # source for parameter dictionary source = __NAME__ + '/main()' # get limits of order extraction valid_orders = spirouEXTOR.GetValidOrders(p, loc) # loop around each order for order_num in valid_orders: # ------------------------------------------------------------- # IC_EXTRACT_TYPE decides the extraction routine # ------------------------------------------------------------- eargs = [p, loc, data2, order_num] ekwargs = dict(mode=p['IC_EXTRACT_TYPE'], order_profile=order_profile) with warnings.catch_warnings(record=True) as w: eout = spirouEXTOR.Extraction(*eargs, **ekwargs) # deal with different return if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES: e2ds, e2dsll, cpt = eout else: e2ds, cpt = eout e2dsll = None # ------------------------------------------------------------- # calculate the noise range1, range2 = p['IC_EXT_RANGE1'], p['IC_EXT_RANGE2'] # set the noise noise = p['SIGDET'] * np.sqrt(range1 + range2) # get window size blaze_win1 = int(data2.shape[0] / 2) - p['IC_EXTFBLAZ'] blaze_win2 = int(data2.shape[0] / 2) + p['IC_EXTFBLAZ'] # get average flux per pixel flux = np.nansum( e2ds[blaze_win1:blaze_win2]) / (2 * p['IC_EXTFBLAZ']) # calculate signal to noise ratio = flux/sqrt(flux + noise^2) snr = flux / np.sqrt(flux + noise**2) # log the SNR RMS wmsg = 'On fiber {0} order {1}: S/N= {2:.1f} Nbcosmic= {3}' wargs = [p['FIBER'], order_num, snr, cpt] WLOG(p, '', wmsg.format(*wargs)) # add calculations to storage loc['E2DS'][order_num] = e2ds loc['E2DSFF'][order_num] = e2ds / loc['FLAT'][order_num] loc['SNR'][order_num] = snr # save the longfile if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES: loc['E2DSLL'].append(e2dsll) # set sources loc.set_sources(['e2ds', 'SNR'], source) # Log if saturation level reached satvalue = (flux / p['GAIN']) / (range1 + range2) if satvalue > (p['QC_LOC_FLUMAX'] * p['NBFRAMES']): wmsg = 'SATURATION LEVEL REACHED on Fiber {0} order={1}' WLOG(p, 'warning', wmsg.format(fiber, order_num)) # ------------------------------------------------------------------ # Thermal correction # ------------------------------------------------------------------ # get fiber type if fiber in ['AB', 'A', 'B']: fibertype = p['DPRTYPE'].split('_')[0] else: fibertype = p['DPRTYPE'].split('_')[1] # apply thermal correction based on fiber type if fibertype in p['THERMAL_CORRECTION_TYPE1']: # log progress wmsg = 'Correcting thermal background for {0}={1} mode={2}' wargs = [fiber, fibertype, 1] WLOG(p, 'info', wmsg.format(*wargs)) # correct E2DS tkwargs = dict(image=loc['E2DS'], mode=1, fiber=fiber, hdr=hdr) p, loc['E2DS'] = spirouBACK.ThermalCorrect(p, **tkwargs) # correct E2DSFF tkwargs = dict(image=loc['E2DSFF'], mode=1, fiber=fiber, hdr=hdr, flat=loc['FLAT']) p, loc['E2DSFF'] = spirouBACK.ThermalCorrect(p, **tkwargs) elif fibertype in p['THERMAL_CORRECTION_TYPE2']: # log progress wmsg = 'Correcting thermal background for {0}={1} mode={2}' wargs = [fiber, fibertype, 2] WLOG(p, 'info', wmsg.format(*wargs)) # correct E2DS tkwargs = dict(image=loc['E2DS'], mode=2, fiber=fiber, hdr=hdr) p, loc['E2DS'] = spirouBACK.ThermalCorrect(p, **tkwargs) # correct E2DSFF tkwargs = dict(image=loc['E2DSFF'], mode=2, fiber=fiber, hdr=hdr, flat=loc['FLAT']) p, loc['E2DSFF'] = spirouBACK.ThermalCorrect(p, **tkwargs) else: # log progress wmsg = 'Not correcting thermal background for {0}={1}' wargs = [fiber, fibertype] WLOG(p, 'info', wmsg.format(*wargs)) # set filename for output outfile = 'THERMALFILE_{0}'.format(fiber) p[outfile] = 'None' p.set_source(outfile, __NAME__ + '.main()') # ------------------------------------------------------------------ # Plots # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot all orders or one order if p['IC_FF_PLOT_ALL_ORDERS']: # plot image with all order fits (slower) sPlt.ext_aorder_fit(p, loc, data1, max_signal / 10.) else: # plot image with selected order fit and edge fit (faster) sPlt.ext_sorder_fit(p, loc, data1, max_signal / 10.) # plot e2ds against wavelength sPlt.ext_spectral_order_plot(p, loc) if p['IC_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES: sPlt.ext_debanana_plot(p, loc, data2, max_signal / 10.) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # if array is completely NaNs it shouldn't pass if np.sum(np.isfinite(loc['E2DS'])) == 0: fail_msg.append('E2DS image is all NaNs') passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append('NaN') qc_names.append('image') qc_logic.append('image is all NaN') # ---------------------------------------------------------------------- # saturation check: check that the max_signal is lower than # qc_max_signal max_qcflux = p['QC_MAX_SIGNAL'] * p['NBFRAMES'] if max_signal > max_qcflux: fmsg = 'Too much flux in the image ({0:.2f} > {1:.2f})' fail_msg.append(fmsg.format(max_signal, max_qcflux)) passed = False # Question: Why is this test ignored? # For some reason this test is ignored in old code passed = True WLOG(p, 'info', fail_msg[-1]) qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(max_signal) qc_names.append('max_signal') qc_logic.append('QC_MAX_SIGNAL > {0:.3f}'.format(max_qcflux)) # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Store extraction in file(s) # ------------------------------------------------------------------ raw_ext_file = os.path.basename(p['FITSFILENAME']) # construct filename e2dsfits, tag1 = spirouConfig.Constants.EXTRACT_E2DS_FILE(p) e2dsfitsname = os.path.split(e2dsfits)[-1] e2dsfffits, tag2 = spirouConfig.Constants.EXTRACT_E2DSFF_FILE(p) e2dsfffitsname = os.path.split(e2dsfffits)[-1] e2dsllfits, tag4 = spirouConfig.Constants.EXTRACT_E2DSLL_FILE(p) e2dsfllitsname = os.path.split(e2dsllfits)[-1] # log that we are saving E2DS spectrum wmsg = 'Saving E2DS spectrum of Fiber {0} in {1}' WLOG(p, '', wmsg.format(p['FIBER'], e2dsfitsname)) wmsg = 'Saving E2DSFF spectrum of Fiber {0} in {1}' WLOG(p, '', wmsg.format(p['FIBER'], e2dsfffitsname)) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_FIBER'], value=p['FIBER']) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBACK'], value=p['BKGRDFILE']) if p['IC_EXTRACT_TYPE'] not in EXTRACT_SHAPE_TYPES: hdict = spirouImage.AddKey(p, hdict, p['KW_CDBTILT'], value=p['TILTFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFLAT'], value=p['FLATFILE']) if p['IC_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES: hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEX'], value=p['SHAPEXFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEY'], value=p['SHAPEYFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPE'], value=p['SHAPEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFPMASTER'], value=p['FPMASTERFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBTHERMAL'], value=p['THERMALFILE_{0}'.format(fiber)]) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # construct loco filename locofile, _ = spirouConfig.Constants.EXTRACT_LOCO_FILE(p) locofilename = os.path.basename(locofile) # add barycentric keys to header hdict = spirouImage.AddKey(p, hdict, p['KW_BERV'], value=loc['BERV']) hdict = spirouImage.AddKey(p, hdict, p['KW_BJD'], value=loc['BJD']) hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_MAX'], value=loc['BERV_MAX']) hdict = spirouImage.AddKey(p, hdict, p['KW_B_OBS_HOUR'], value=loc['BERVHOUR']) # add barycentric estimate keys to header hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_EST'], value=loc['BERV_EST']) hdict = spirouImage.AddKey(p, hdict, p['KW_BJD_EST'], value=loc['BJD_EST']) hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_MAX_EST'], value=loc['BERV_MAX_EST']) hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_SOURCE'], value=loc['BERV_SOURCE']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # copy extraction method and function to header # (for reproducibility) hdict = spirouImage.AddKey(p, hdict, p['KW_E2DS_EXTM'], value=extmethod) hdict = spirouImage.AddKey(p, hdict, p['KW_E2DS_FUNC'], value=extfunc) # add localization file name to header hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_FILE'], value=locofilename) # add wave solution date hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME1'], value=loc['WAVE_ACQTIMES'][0]) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME2'], value=loc['WAVE_ACQTIMES'][1]) # add wave solution number of orders hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_ORD_N'], value=loc['WAVEPARAMS'].shape[0]) # add wave solution degree of fit hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_LL_DEG'], value=loc['WAVEPARAMS'].shape[1] - 1) # ------------------------------------------------------------------------- # add keys of the wave solution FP CCF hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_FILE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_DRIFT'], value=p['WFP_DRIFT']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_FWHM'], value=p['WFP_FWHM']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_CONTRAST'], value=p['WFP_CONTRAST']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_MAXCPP'], value=p['WFP_MAXCPP']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_MASK'], value=p['WFP_MASK']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_LINES'], value=p['WFP_LINES']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_TARG_RV'], value=p['WFP_TARG_RV']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_WIDTH'], value=p['WFP_WIDTH']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_STEP'], value=p['WFP_STEP']) # write 1D list of the SNR hdict = spirouImage.AddKey1DList(p, hdict, p['KW_E2DS_SNR'], values=loc['SNR']) # add localization file keys to header root = p['KW_ROOT_DRS_LOC'][0] hdict = spirouImage.CopyRootKeys(p, hdict, locofile, root=root) # add wave solution coefficients hdict = spirouImage.AddKey2DList(p, hdict, p['KW_WAVE_PARAM'], values=loc['WAVEPARAMS']) # Save E2DS file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) p = spirouImage.WriteImage(p, e2dsfits, loc['E2DS'], hdict) # Save E2DSFF file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) p = spirouImage.WriteImage(p, e2dsfffits, loc['E2DSFF'], hdict) # Save E2DSLL file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES: llstack = np.vstack(loc['E2DSLL']) p = spirouImage.WriteImage(p, e2dsllfits, llstack, hdict) # ------------------------------------------------------------------ # 1-dimension spectral S1D (uniform in wavelength) # ------------------------------------------------------------------ # get arguments for E2DS to S1D e2dsargs = [loc['WAVE'], loc['E2DSFF'], loc['BLAZE']] # get 1D spectrum xs1d1, ys1d1 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='wave') # Plot the 1D spectrum if p['DRS_PLOT'] > 0: sPlt.ext_1d_spectrum_plot(p, xs1d1, ys1d1) # construct file name s1dfile1, tag3 = spirouConfig.Constants.EXTRACT_S1D_FILE1(p) s1dfilename1 = os.path.basename(s1dfile1) # add header keys # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) # log writing to file wmsg = 'Saving 1D spectrum (uniform in wavelength) for Fiber {0} in {1}' WLOG(p, '', wmsg.format(p['FIBER'], s1dfilename1)) # Write to file columns = ['wavelength', 'flux', 'eflux'] values = [xs1d1, ys1d1, np.zeros_like(ys1d1)] units = ['nm', None, None] s1d1 = spirouImage.MakeTable(p, columns, values, units=units) spirouImage.WriteTable(p, s1d1, s1dfile1, header=hdict) # ------------------------------------------------------------------ # 1-dimension spectral S1D (uniform in velocity) # ------------------------------------------------------------------ # get arguments for E2DS to S1D e2dsargs = [loc['WAVE'], loc['E2DSFF'], loc['BLAZE']] # get 1D spectrum xs1d2, ys1d2 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='velocity') # Plot the 1D spectrum if p['DRS_PLOT'] > 0: sPlt.ext_1d_spectrum_plot(p, xs1d2, ys1d2) # construct file name s1dfile2, tag4 = spirouConfig.Constants.EXTRACT_S1D_FILE2(p) s1dfilename2 = os.path.basename(s1dfile2) # add header keys hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) # log writing to file wmsg = 'Saving 1D spectrum (uniform in velocity) for Fiber {0} in {1}' WLOG(p, '', wmsg.format(p['FIBER'], s1dfilename2)) # Write to file columns = ['wavelength', 'flux', 'eflux'] values = [xs1d2, ys1d2, np.zeros_like(ys1d2)] units = ['nm', None, None] s1d2 = spirouImage.MakeTable(p, columns, values, units=units) spirouImage.WriteTable(p, s1d2, s1dfile2, header=hdict) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): """ cal_shape_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_shape_spirou.py [night_directory] [fitsfilename] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) p = spirouStartup.InitialFileSetup(p) # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add') # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Get basic image properties for FP file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # ---------------------------------------------------------------------- # Correction of reference FP # ---------------------------------------------------------------------- # set the number of frames p['NBFRAMES'] = 1 p.set_source('NBFRAMES', __NAME__ + '.main()') # Correction of DARK p, datac = spirouImage.CorrectForDark(p, data, hdr) # Resize hc data # rotate the image and convert from ADU/s to e- data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) data1 = spirouImage.ResizeImage(p, data, **bkwargs) # log change in data size WLOG(p, '', ('FPref Image format changed to {0}x{1}').format(*data1.shape)) # Correct for the BADPIX mask (set all bad pixels to zero) bargs = [p, data1, hdr] p, data1 = spirouImage.CorrectForBadPix(*bargs) p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True) # log progress WLOG(p, '', 'Cleaning FPref hot pixels') # correct hot pixels data1 = spirouEXTOR.CleanHotpix(data1, badpixmask) # Log the number of dead pixels # get the number of bad pixels with warnings.catch_warnings(record=True) as _: n_bad_pix = np.nansum(data1 <= 0) n_bad_pix_frac = n_bad_pix * 100 / np.product(data1.shape) # Log number wmsg = 'Nb FPref dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ---------------------------------------------------------------------- # Get master FP file # ---------------------------------------------------------------------- # log progress WLOG(p, '', 'Getting FP Master from calibDB') # get master fp p, masterfp = spirouImage.GetFPMaster(p, hdr) # ---------------------------------------------------------------------- # Get transform parameters # ---------------------------------------------------------------------- # log progress wargs = [p['ARG_FILE_NAMES'][0], p['FPMASTERFILE']] WLOG(p, 'info', 'Calculating transforming for {0} onto {1}'.format(*wargs)) gout = spirouImage.GetLinearTransformParams(p, masterfp, data1) transform, xres, yres = gout # ------------------------------------------------------------------ # Need to straighten the fp data for debug # ------------------------------------------------------------------ p, shapem_x = spirouImage.GetShapeX(p, hdr) p, shapem_y = spirouImage.GetShapeY(p, hdr) data2 = spirouImage.EATransform(data1, transform, dxmap=shapem_x, dymap=shapem_y) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # if transform is None means the fp image quality was too poor if transform is None: fail_msg.append('FP Image quality too poor (sigma clip failed)') passed = False qc_pass.append(0) else: qc_pass.append(1) qc_values.append('None') qc_names.append('Image Quality') qc_logic.append('Image too poor') # ---------------------------------------------------------------------- # get residual qc parameter qc_res = p['SHAPE_QC_LINEAR_TRANS_RES_THRES'] # assess quality of x residuals if xres > qc_res: fail_msg.append('x-resdiuals too high {0} > {1}'.format(xres, qc_res)) passed = False qc_pass.append(0) else: qc_pass.append(1) qc_values.append(xres) qc_names.append('XRES') qc_logic.append('XRES > {0}'.format(qc_res)) # assess quality of x residuals if yres > qc_res: fail_msg.append('y-resdiuals too high {0} > {1}'.format(yres, qc_res)) passed = False qc_pass.append(0) else: qc_pass.append(1) qc_values.append(yres) qc_names.append('YRES') qc_logic.append('YRES > {0}'.format(qc_res)) # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Writing shape to file # ------------------------------------------------------------------ # get the raw tilt file name raw_shape_file = os.path.basename(p['FITSFILENAME']) # construct file name and path shapefits, tag = spirouConfig.Constants.SLIT_SHAPE_LOCAL_FILE(p) shapefitsname = os.path.basename(shapefits) # Log that we are saving tilt file wmsg = 'Saving shape information in file: {0}' WLOG(p, '', wmsg.format(shapefitsname)) # Copy keys from fits file hdict = spirouImage.CopyOriginalKeys(hdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEX'], value=p['SHAPEXFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEY'], value=p['SHAPEYFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFPMASTER'], value=p['FPMASTERFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # add the transform parameters hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_DX'], value=transform[0]) hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_DY'], value=transform[1]) hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_A'], value=transform[2]) hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_B'], value=transform[3]) hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_C'], value=transform[4]) hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_D'], value=transform[5]) # write tilt file to file p = spirouImage.WriteImage(p, shapefits, [transform], hdict) # ---------------------------------------------------------------------- # Move to calibDB and update calibDB # ---------------------------------------------------------------------- if p['QC']: # add shape keydb = 'SHAPE' # copy shape file to the calibDB folder spirouDB.PutCalibFile(p, shapefits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, shapefitsname, hdr) # ------------------------------------------------------------------ # Writing sanity check files # ------------------------------------------------------------------ if p['SHAPE_DEBUG_OUTPUTS']: # log WLOG(p, '', 'Saving debug sanity check files') # construct file names dargs = [p, p['ARG_FILE_NAMES'][0]] out1 = spirouConfig.Constants.SLIT_SHAPE_IN_FP_FILE(*dargs) input_fp_file, tag1= out1 out2 = spirouConfig.Constants.SLIT_SHAPE_OUT_FP_FILE(*dargs) output_fp_file, tag2 = out2 # write input fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) p = spirouImage.WriteImage(p, input_fp_file, data1, hdict) # write output fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) p = spirouImage.WriteImage(p, output_fp_file, data2, hdict) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): """ cal_loc_RAW_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_loc_RAW_spirou.py [night_name] [files] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) p = spirouStartup.InitialFileSetup(p, calibdb=True) # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add') # ---------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- data = spirouImage.FixNonPreProcess(p, data) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # ---------------------------------------------------------------------- # Correction of DARK # ---------------------------------------------------------------------- p, datac = spirouImage.CorrectForDark(p, data, hdr) # ---------------------------------------------------------------------- # Interpolation over bad regions (to fill in the holes) # ---------------------------------------------------------------------- # log process # wmsg = 'Interpolating over bad regions' # WLOG(p, '', wmsg) # run interpolation # datac = spirouImage.InterpolateBadRegions(p, datac) # ---------------------------------------------------------------------- # Resize image # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p) # convert NaN to zeros data0 = np.where(~np.isfinite(data), np.zeros_like(data), data) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) data2 = spirouImage.ResizeImage(p, data0, **bkwargs) # log change in data size WLOG(p, '', ('Image format changed to ' '{0}x{1}').format(*data2.shape)) # ---------------------------------------------------------------------- # Correct for the BADPIX mask (set all bad pixels to zero) # ---------------------------------------------------------------------- p, data2 = spirouImage.CorrectForBadPix(p, data2, hdr) # ---------------------------------------------------------------------- # Background computation # ---------------------------------------------------------------------- if p['IC_DO_BKGR_SUBTRACTION']: # log that we are doing background measurement WLOG(p, '', 'Doing background measurement on raw frame') # get the bkgr measurement bargs = [p, data2, hdr] # background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs) p, background = spirouBACK.MeasureBackgroundMap(*bargs) else: background = np.zeros_like(data2) p['BKGRDFILE'] = 'None' p.set_source('BKGRDFILE', __NAME__ + '.main()') # apply background correction to data data2 = data2 - background # ---------------------------------------------------------------------- # Construct image order_profile # ---------------------------------------------------------------------- # log that we are doing background measurement WLOG(p, '', 'Creating Order Profile') order_profile = spirouLOCOR.BoxSmoothedImage(data2, p['LOC_BOX_SIZE']) # data 2 is now set to the order profile data2o = data2.copy() data2 = order_profile.copy() # ---------------------------------------------------------------------- # Write image order_profile to file # ---------------------------------------------------------------------- # Construct folder and filename rawfits, tag1 = spirouConfig.Constants.LOC_ORDER_PROFILE_FILE(p) rawfitsname = os.path.split(rawfits)[-1] # log saving order profile wmsg = 'Saving processed raw frame in {0}' WLOG(p, '', wmsg.format(rawfitsname)) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) # write to file p = spirouImage.WriteImage(p, rawfits, order_profile, hdict) # ---------------------------------------------------------------------- # Move order_profile to calibDB and update calibDB # ---------------------------------------------------------------------- # set key for calibDB keydb = 'ORDER_PROFILE_{0}'.format(p['FIBER']) # copy dark fits file to the calibDB folder spirouDB.PutCalibFile(p, rawfits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, rawfitsname, hdr) # ###################################################################### # Localization of orders on central column # ###################################################################### # storage dictionary for localization parameters loc = ParamDict() # Plots setup: start interactive plot if p['DRS_PLOT'] > 0: sPlt.start_interactive_session(p) # ---------------------------------------------------------------------- # Measurement and correction of background on the central column # ---------------------------------------------------------------------- loc = spirouBACK.MeasureBkgrdGetCentPixs(p, loc, data2) # ---------------------------------------------------------------------- # Search for order center on the central column - quick estimation # ---------------------------------------------------------------------- # log progress WLOG(p, '', 'Searching order center on central column') # plot the minimum of ycc and ic_locseuil if in debug and plot mode if p['DRS_DEBUG'] > 0 and p['DRS_PLOT']: sPlt.debug_locplot_min_ycc_loc_threshold(p, loc['YCC']) # find the central positions of the orders in the central posc_all = spirouLOCOR.FindPosCentCol(loc['YCC'], p['IC_LOCSEUIL']) # depending on the fiber type we may need to skip some pixels and also # we need to add back on the ic_offset applied start = p['IC_FIRST_ORDER_JUMP'] posc = posc_all[start:] + p['IC_OFFSET'] # work out the number of orders to use (minimum of ic_locnbmaxo and number # of orders found in 'LocateCentralOrderPositions') number_of_orders = np.min([p['IC_LOCNBMAXO'], len(posc)]) # log the number of orders than have been detected wargs = [p['FIBER'], int(number_of_orders / p['NBFIB']), p['NBFIB']] WLOG(p, 'info', ('On fiber {0} {1} orders have been detected ' 'on {2} fiber(s)').format(*wargs)) # ---------------------------------------------------------------------- # Search for order center and profile on specific columns # ---------------------------------------------------------------------- # get fit polynomial orders for position and width fitpos, fitwid = p['IC_LOCDFITC'], p['IC_LOCDFITW'] # Create arrays to store position and width of order for each order loc['CTRO'] = np.zeros((number_of_orders, data2.shape[1]), dtype=float) loc['SIGO'] = np.zeros((number_of_orders, data2.shape[1]), dtype=float) # Create arrays to store coefficients for position and width loc['ACC'] = np.zeros((number_of_orders, fitpos + 1)) loc['ASS'] = np.zeros((number_of_orders, fitpos + 1)) # Create arrays to store rms values for position and width loc['RMS_CENTER'] = np.zeros(number_of_orders) loc['RMS_FWHM'] = np.zeros(number_of_orders) # Create arrays to store point to point max value for position and width loc['MAX_PTP_CENTER'] = np.zeros(number_of_orders) loc['MAX_PTP_FRACCENTER'] = np.zeros(number_of_orders) loc['MAX_PTP_FWHM'] = np.zeros(number_of_orders) loc['MAX_PTP_FRACFWHM'] = np.zeros(number_of_orders) # Create arrays to store rejected points loc['MAX_RMPTS_POS'] = np.zeros(number_of_orders) loc['MAX_RMPTS_WID'] = np.zeros(number_of_orders) # set the central col centers in the cpos_orders array loc['CTRO'][:, p['IC_CENT_COL']] = posc[0:number_of_orders] # set source for all locs loc.set_all_sources(__NAME__ + '/main()') # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # storage for plotting loc['XPLOT'], loc['YPLOT'] = [], [] # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # loop around each order rorder_num = 0 for order_num in range(number_of_orders): # find the row centers of the columns loc = spirouLOCOR.FindOrderCtrs(p, data2, loc, order_num) # only keep the orders with non-zero width mask = loc['SIGO'][order_num, :] != 0 loc['X'] = np.arange(data2.shape[1])[mask] loc.set_source('X', __NAME__ + '/main()') # check that we have enough data points to fit data if len(loc['X']) > (fitpos + 1): # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # initial fit params iofargs = [p, loc, mask, rorder_num] # initial fit for center positions for this order loc, cf_data = spirouLOCOR.InitialOrderFit(*iofargs, kind='center') # initial fit for widths for this order loc, wf_data = spirouLOCOR.InitialOrderFit(*iofargs, kind='fwhm') # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Log order number and fit at central pixel and width and rms wargs = [ rorder_num, cf_data['cfitval'], wf_data['cfitval'], cf_data['rms'] ] WLOG(p, '', ('ORDER: {0} center at pixel {1:.1f} width ' '{2:.1f} rms {3:.3f}').format(*wargs)) # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # sigma fit params sigfargs = [p, loc, cf_data, mask, order_num, rorder_num] # sigma clip fit for center positions for this order cf_data = spirouLOCOR.SigClipOrderFit(*sigfargs, kind='center') # load results into storage arrags for this order loc['ACC'][rorder_num] = cf_data['a'] loc['RMS_CENTER'][rorder_num] = cf_data['rms'] loc['MAX_PTP_CENTER'][rorder_num] = cf_data['max_ptp'] loc['MAX_PTP_FRACCENTER'][rorder_num] = cf_data['max_ptp_frac'] loc['MAX_RMPTS_POS'][rorder_num] = cf_data['max_rmpts'] # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # sigma fit params sigfargs = [p, loc, wf_data, mask, order_num, rorder_num] # sigma clip fit for width positions for this order wf_data = spirouLOCOR.SigClipOrderFit(*sigfargs, kind='fwhm') # load results into storage arrags for this order loc['ASS'][rorder_num] = wf_data['a'] loc['RMS_FWHM'][rorder_num] = wf_data['rms'] loc['MAX_PTP_FWHM'][rorder_num] = wf_data['max_ptp'] loc['MAX_PTP_FRACFWHM'][rorder_num] = wf_data['max_ptp_frac'] loc['MAX_RMPTS_WID'][rorder_num] = wf_data['max_rmpts'] # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # increase the roder_num iterator rorder_num += 1 # else log that the order is unusable else: WLOG(p, '', 'Order found too much incomplete, discarded') # ---------------------------------------------------------------------- # Plot the image (ready for fit points to be overplotted later) if p['DRS_PLOT'] > 0: # get saturation threshold satseuil = p['IC_SATSEUIL'] * p['GAIN'] * p['NBFRAMES'] # plot image above saturation threshold sPlt.locplot_im_sat_threshold(p, loc, data2, satseuil) # ---------------------------------------------------------------------- # Log that order geometry has been measured WLOG(p, 'info', ('On fiber {0} {1} orders geometry have been ' 'measured').format(p['FIBER'], rorder_num)) # Get mean rms mean_rms_center = np.nansum( loc['RMS_CENTER'][:rorder_num]) * 1000 / rorder_num mean_rms_fwhm = np.nansum(loc['RMS_FWHM'][:rorder_num]) * 1000 / rorder_num # Log mean rms values wmsg = 'Average uncertainty on {0}: {1:.2f} [mpix]' WLOG(p, 'info', wmsg.format('position', mean_rms_center)) WLOG(p, 'info', wmsg.format('width', mean_rms_fwhm)) # ---------------------------------------------------------------------- # Plot of RMS for positions and widths # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: sPlt.locplot_order_number_against_rms(p, loc, rorder_num) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # check that max number of points rejected in center fit is below threshold if np.nansum(loc['MAX_RMPTS_POS']) > p['QC_LOC_MAXLOCFIT_REMOVED_CTR']: fmsg = 'abnormal points rejection during ctr fit ({0:.2f} > {1:.2f})' fail_msg.append( fmsg.format(np.nansum(loc['MAX_RMPTS_POS']), p['QC_LOC_MAXLOCFIT_REMOVED_CTR'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(np.nansum(loc['MAX_RMPTS_POS'])) qc_names.append('sum(MAX_RMPTS_POS)') qc_logic.append('sum(MAX_RMPTS_POS) > {0:.2f}' ''.format(p['QC_LOC_MAXLOCFIT_REMOVED_CTR'])) # ---------------------------------------------------------------------- # check that max number of points rejected in width fit is below threshold if np.nansum(loc['MAX_RMPTS_WID']) > p['QC_LOC_MAXLOCFIT_REMOVED_WID']: fmsg = 'abnormal points rejection during width fit ({0:.2f} > {1:.2f})' fail_msg.append( fmsg.format(np.nansum(loc['MAX_RMPTS_WID']), p['QC_LOC_MAXLOCFIT_REMOVED_WID'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(np.nansum(loc['MAX_RMPTS_WID'])) qc_names.append('sum(MAX_RMPTS_WID)') qc_logic.append('sum(MAX_RMPTS_WID) > {0:.2f}' ''.format(p['QC_LOC_MAXLOCFIT_REMOVED_WID'])) # ---------------------------------------------------------------------- # check that the rms in center fit is lower than qc threshold if mean_rms_center > p['QC_LOC_RMSMAX_CENTER']: fmsg = 'too high rms on center fitting ({0:.2f} > {1:.2f})' fail_msg.append(fmsg.format(mean_rms_center, p['QC_LOC_RMSMAX_CENTER'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(mean_rms_center) qc_names.append('mean_rms_center') qc_logic.append('mean_rms_center > {0:.2f}' ''.format(p['QC_LOC_RMSMAX_CENTER'])) # ---------------------------------------------------------------------- # check that the rms in center fit is lower than qc threshold if mean_rms_fwhm > p['QC_LOC_RMSMAX_FWHM']: fmsg = 'too high rms on profile fwhm fitting ({0:.2f} > {1:.2f})' fail_msg.append(fmsg.format(mean_rms_fwhm, p['QC_LOC_RMSMAX_CENTER'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(mean_rms_fwhm) qc_names.append('mean_rms_fwhm') qc_logic.append('mean_rms_fwhm > {0:.2f}' ''.format(p['QC_LOC_RMSMAX_CENTER'])) # ---------------------------------------------------------------------- # check for abnormal number of identified orders if rorder_num != p['QC_LOC_NBO']: fmsg = ('abnormal number of identified orders (found {0:.2f} ' 'expected {1:.2f})') fail_msg.append(fmsg.format(rorder_num, p['QC_LOC_NBO'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(rorder_num) qc_names.append('rorder_num') qc_logic.append('rorder_num != {0:.2f}'.format(p['QC_LOC_NBO'])) # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ---------------------------------------------------------------------- # Save and record of image of localization with order center and keywords # ---------------------------------------------------------------------- raw_loco_file = os.path.basename(p['FITSFILENAME']) # construct filename locofits, tag2 = spirouConfig.Constants.LOC_LOCO_FILE(p) locofitsname = os.path.split(locofits)[-1] # log that we are saving localization file WLOG(p, '', ('Saving localization information ' 'in file: {0}').format(locofitsname)) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # define new keys to add hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=raw_loco_file) hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_SIGDET']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_CONAD']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_BCKGRD'], value=loc['MEAN_BACKGRD']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_NBO'], value=rorder_num) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_C'], value=p['IC_LOCDFITC']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_W'], value=p['IC_LOCDFITW']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_E']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DELTA']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_MAXFLX'], value=float(loc['MAX_SIGNAL'])) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_SMAXPTS_CTR'], value=np.nansum(loc['MAX_RMPTS_POS'])) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_SMAXPTS_WID'], value=np.nansum(loc['MAX_RMPTS_WID'])) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_RMS_CTR'], value=mean_rms_center) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_RMS_WID'], value=mean_rms_fwhm) # write 2D list of position fit coefficients hdict = spirouImage.AddKey2DList(p, hdict, p['KW_LOCO_CTR_COEFF'], values=loc['ACC'][0:rorder_num]) # write 2D list of width fit coefficients hdict = spirouImage.AddKey2DList(p, hdict, p['KW_LOCO_FWHM_COEFF'], values=loc['ASS'][0:rorder_num]) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # write center fits and add header keys (via hdict) center_fits = spirouLOCOR.CalcLocoFits(loc['ACC'], data2.shape[1]) p = spirouImage.WriteImage(p, locofits, center_fits, hdict) # ---------------------------------------------------------------------- # Save and record of image of sigma # ---------------------------------------------------------------------- # construct filename locofits2, tag3 = spirouConfig.Constants.LOC_LOCO_FILE2(p) locofits2name = os.path.split(locofits2)[-1] # log that we are saving localization file wmsg = 'Saving FWHM information in file: {0}' WLOG(p, '', wmsg.format(locofits2name)) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # define new keys to add hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBACK'], value=p['BKGRDFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # add outputs hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_SIGDET']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_CONAD']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_NBO'], value=rorder_num) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_C'], value=p['IC_LOCDFITC']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_W'], value=p['IC_LOCDFITW']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_E']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_MAXFLX'], value=float(loc['MAX_SIGNAL'])) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_SMAXPTS_CTR'], value=np.nansum(loc['MAX_RMPTS_POS'])) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_SMAXPTS_WID'], value=np.nansum(loc['MAX_RMPTS_WID'])) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_RMS_CTR'], value=mean_rms_center) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_RMS_WID'], value=mean_rms_fwhm) # write 2D list of position fit coefficients hdict = spirouImage.AddKey2DList(p, hdict, p['KW_LOCO_CTR_COEFF'], values=loc['ACC'][0:rorder_num]) # write 2D list of width fit coefficients hdict = spirouImage.AddKey2DList(p, hdict, p['KW_LOCO_FWHM_COEFF'], values=loc['ASS'][0:rorder_num]) # add quality control hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) # write image and add header keys (via hdict) width_fits = spirouLOCOR.CalcLocoFits(loc['ASS'], data2.shape[1]) p = spirouImage.WriteImage(p, locofits2, width_fits, hdict) # ---------------------------------------------------------------------- # Save and Record of image of localization # ---------------------------------------------------------------------- if p['IC_LOCOPT1']: # construct filename locofits3, tag4 = spirouConfig.Constants.LOC_LOCO_FILE3(p) locofits3name = os.path.split(locofits3)[-1] # log that we are saving localization file wmsg1 = 'Saving localization image with superposition of orders in ' wmsg2 = 'file: {0}'.format(locofits3name) WLOG(p, '', [wmsg1, wmsg2]) # superpose zeros over the fit in the image data4 = spirouLOCOR.ImageLocSuperimp(data2o, loc['ACC'][0:rorder_num]) # save this image to file hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_FIBER'], value=p['FIBER']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) p = spirouImage.WriteImage(p, locofits3, data4, hdict) # ---------------------------------------------------------------------- # Update the calibration database # ---------------------------------------------------------------------- if p['QC'] == 1: keydb = 'LOC_' + p['FIBER'] # copy localisation file to the calibDB folder spirouDB.PutCalibFile(p, locofits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, locofitsname, hdr) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): """ cal_SLIT_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_SLIT_spirou.py [night_directory] [files] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) p = spirouStartup.InitialFileSetup(p, calibdb=True) # set the fiber type p['FIBER'] = 'AB' p.set_source('FIBER', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add') # ---------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- data = spirouImage.FixNonPreProcess(p, data) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # ---------------------------------------------------------------------- # Correction of DARK # ---------------------------------------------------------------------- p, datac = spirouImage.CorrectForDark(p, data, hdr) datac = data # ---------------------------------------------------------------------- # Resize image # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p) # convert NaN to zeros data0 = np.where(~np.isfinite(data), np.zeros_like(data), data) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) data2 = spirouImage.ResizeImage(p, data0, **bkwargs) # log change in data size WLOG(p, '', ('Image format changed to ' '{0}x{1}').format(*data2.shape)) # ---------------------------------------------------------------------- # Correct for the BADPIX mask (set all bad pixels to zero) # ---------------------------------------------------------------------- p, data2 = spirouImage.CorrectForBadPix(p, data2, hdr) p, badpixmap = spirouImage.CorrectForBadPix(p, data2, hdr, return_map=True) # ---------------------------------------------------------------------- # Background computation # ---------------------------------------------------------------------- if p['IC_DO_BKGR_SUBTRACTION']: # log that we are doing background measurement WLOG(p, '', 'Doing background measurement on raw frame') # get the bkgr measurement bargs = [p, data2, hdr, badpixmap] # background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs) p, background = spirouBACK.MeasureBackgroundMap(*bargs) else: background = np.zeros_like(data2) p['BKGRDFILE'] = 'None' p.set_source('BKGRDFILE', __NAME__ + '.main()') # apply background correction to data data2 = data2 - background # save data to loc loc = ParamDict() loc['DATA'] = data2 loc.set_source('DATA', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- # get the number of bad pixels n_bad_pix = np.nansum(data2 <= 0) n_bad_pix_frac = n_bad_pix * 100 / np.product(data2.shape) # Log number wmsg = 'Nb dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ------------------------------------------------------------------ # Get localisation coefficients # ------------------------------------------------------------------ # original there is a loop but it is not used --> removed p = spirouImage.FiberParams(p, p['FIBER'], merge=True) # get localisation fit coefficients p, loc = spirouLOCOR.GetCoeffs(p, hdr, loc) # ------------------------------------------------------------------ # Calculate shape map # ------------------------------------------------------------------ loc = spirouImage.GetShapeMap(p, loc) # ------------------------------------------------------------------ # Plotting # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # plots setup: start interactive plot sPlt.start_interactive_session(p) # plot the shape process for each order sPlt.slit_shape_angle_plot(p, loc) # end interactive section sPlt.end_interactive_session(p) # ------------------------------------------------------------------ # Writing to file # ------------------------------------------------------------------ # get the raw tilt file name raw_shape_file = os.path.basename(p['FITSFILENAME']) # construct file name and path shapefits, tag = spirouConfig.Constants.SLIT_XSHAPE_FILE(p) shapefitsname = os.path.basename(shapefits) # Log that we are saving tilt file wmsg = 'Saving shape information in file: {0}' WLOG(p, '', wmsg.format(shapefitsname)) # Copy keys from fits file # Copy keys from fits file hdict = spirouImage.CopyOriginalKeys(hdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPE'], value=raw_shape_file) # write tilt file to file p = spirouImage.WriteImage(p, shapefits, loc['DXMAP'], hdict) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # TODO: Decide on some quality control criteria? # set passed variable and fail message list passed, fail_msg = True, [] # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Move to calibDB and update calibDB # ---------------------------------------------------------------------- if p['QC']: keydb = 'SHAPE' # copy shape file to the calibDB folder spirouDB.PutCalibFile(p, shapefits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, shapefitsname, hdr) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, hcfile=None, fpfiles=None): """ cal_SLIT_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_SLIT_spirou.py [night_directory] [files] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) if hcfile is None or fpfiles is None: names, types = ['hcfile', 'fpfiles'], [str, str] customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1], types, names, last_multi=True) else: customargs = dict(hcfile=hcfile, fpfiles=fpfiles) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='fpfiles') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, hcfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['HCFILE']) p, fpfilenames = spirouStartup.MultiFileSetup(p, files=p['FPFILES']) # set fiber (it doesn't matter with the 2D image but we need this to get # the lamp type for FPFILES and HCFILES, AB == C p['FIBER'] = 'AB' p['FIB_TYP'] = [p['FIBER']] fsource = __NAME__ + '/main()' p.set_sources(['FIBER', 'FIB_TYP'], fsource) # set the hcfilename to the first hcfilenames fpfitsfilename = fpfilenames[0] # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # add a force plot off p['PLOT_PER_ORDER'] = PLOT_PER_ORDER p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Read FP and HC files # ---------------------------------------------------------------------- # read and combine all FP files except the first (fpfitsfilename) rargs = [p, 'add', fpfitsfilename, fpfilenames[1:]] p, fpdata, fphdr = spirouImage.ReadImageAndCombine(*rargs) # read first file (hcfitsfilename) hcdata, hchdr, _, _ = spirouImage.ReadImage(p, hcfitsfilename) # add data and hdr to loc loc = ParamDict() loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr # set the source sources = ['HCDATA', 'HCHDR'] loc.set_sources(sources, 'spirouImage.ReadImageAndCombine()') sources = ['FPDATA', 'FPHDR'] loc.set_sources(sources, 'spirouImage.ReadImage()') # --------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- hcdata = spirouImage.FixNonPreProcess(p, hcdata) fpdata = spirouImage.FixNonPreProcess(p, fpdata) # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, fphdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, fphdr, name='exptime') # get gain p = spirouImage.GetGain(p, fphdr, name='gain') # get lamp parameters p = spirouTHORCA.GetLampParams(p, hchdr) # ---------------------------------------------------------------------- # Correction of DARK # ---------------------------------------------------------------------- # p, hcdatac = spirouImage.CorrectForDark(p, hcdata, hchdr) hcdatac = hcdata p['DARKFILE'] = 'None' # p, fpdatac = spirouImage.CorrectForDark(p, fpdata, fphdr) fpdatac = fpdata # ---------------------------------------------------------------------- # Resize hc data # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- hcdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, hcdatac), p=p) # convert NaN to zeros hcdata0 = np.where(~np.isfinite(hcdata), np.zeros_like(hcdata), hcdata) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) hcdata2 = spirouImage.ResizeImage(p, hcdata0, **bkwargs) # log change in data size WLOG(p, '', ('HC Image format changed to ' '{0}x{1}').format(*hcdata2.shape)) # ---------------------------------------------------------------------- # Resize fp data # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- fpdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, fpdatac), p=p) # convert NaN to zeros fpdata0 = np.where(~np.isfinite(fpdata), np.zeros_like(fpdata), fpdata) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) fpdata2 = spirouImage.ResizeImage(p, fpdata0, **bkwargs) # log change in data size WLOG(p, '', ('FP Image format changed to ' '{0}x{1}').format(*fpdata2.shape)) # ---------------------------------------------------------------------- # Correct for the BADPIX mask (set all bad pixels to zero) # ---------------------------------------------------------------------- # p, hcdata2 = spirouImage.CorrectForBadPix(p, hcdata2, hchdr) # p, fpdata2 = spirouImage.CorrectForBadPix(p, fpdata2, fphdr) p['BADPFILE'] = 'None' # save data to loc loc['HCDATA'] = hcdata2 loc.set_source('HCDATA', __NAME__ + '/main()') # save data to loc loc['FPDATA'] = fpdata2 loc.set_source('FPDATA', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- # get the number of bad pixels n_bad_pix = np.nansum(hcdata2 <= 0) n_bad_pix_frac = n_bad_pix * 100 / np.product(hcdata2.shape) # Log number wmsg = 'Nb HC dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- # get the number of bad pixels n_bad_pix = np.nansum(fpdata2 <= 0) n_bad_pix_frac = n_bad_pix * 100 / np.product(fpdata2.shape) # Log number wmsg = 'Nb FP dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ------------------------------------------------------------------ # Get localisation coefficients # ------------------------------------------------------------------ # original there is a loop but it is not used --> removed p = spirouImage.FiberParams(p, p['FIBER'], merge=True) # get localisation fit coefficients p, loc = spirouLOCOR.GetCoeffs(p, fphdr, loc) # ------------------------------------------------------------------ # Get master wave solution map # ------------------------------------------------------------------ # get master wave map masterwavefile = spirouDB.GetDatabaseMasterWave(p) # log process wmsg1 = 'Getting master wavelength grid' wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile)) WLOG(p, '', [wmsg1, wmsg2]) # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # read master wave map wout = spirouImage.GetWaveSolution(p, filename=masterwavefile, return_wavemap=True, quiet=True, return_header=True, fiber=wave_fiber) loc['MASTERWAVEP'], loc['MASTERWAVE'] = wout[:2] loc['MASTERWAVEHDR'], loc['WSOURCE'] = wout[2:] # set sources wsource = ['MASTERWAVEP', 'MASTERWAVE', 'MASTERWAVEHDR'] loc.set_sources(wsource, 'spirouImage.GetWaveSolution()') # ---------------------------------------------------------------------- # Read UNe solution # ---------------------------------------------------------------------- wave_u_ne, amp_u_ne = spirouImage.ReadLineList(p) loc['LL_LINE'], loc['AMPL_LINE'] = wave_u_ne, amp_u_ne source = __NAME__ + '.main() + spirouImage.ReadLineList()' loc.set_sources(['LL_LINE', 'AMPL_LINE'], source) # ---------------------------------------------------------------------- # Read cavity length file # ---------------------------------------------------------------------- loc['CAVITY_LEN_COEFFS'] = spirouImage.ReadCavityLength(p) source = __NAME__ + '.main() + spirouImage.ReadCavityLength()' loc.set_source('CAVITY_LEN_COEFFS', source) # ------------------------------------------------------------------ # Calculate shape map # ------------------------------------------------------------------ loc = spirouImage.GetShapeMap(p, loc) # ------------------------------------------------------------------ # Plotting # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # plots setup: start interactive plot sPlt.start_interactive_session(p) # plot the shape process for one order sPlt.slit_shape_angle_plot(p, loc) # end interactive section sPlt.end_interactive_session(p) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # TODO: Decide on some quality control criteria? # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # add to qc header lists qc_values.append('None') qc_names.append('None') qc_logic.append('None') qc_pass.append(1) # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Writing DXMAP to file # ------------------------------------------------------------------ # get the raw tilt file name raw_shape_file = os.path.basename(p['FITSFILENAME']) # construct file name and path shapefits, tag = spirouConfig.Constants.SLIT_XSHAPE_FILE(p) shapefitsname = os.path.basename(shapefits) # Log that we are saving tilt file wmsg = 'Saving shape information in file: {0}' WLOG(p, '', wmsg.format(shapefitsname)) # Copy keys from fits file hdict = spirouImage.CopyOriginalKeys(fphdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='hcfile', values=p['HCFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE2'], dim1name='fpfile', values=p['FPFILES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # write tilt file to file p = spirouImage.WriteImage(p, shapefits, loc['DXMAP'], hdict) # ------------------------------------------------------------------ # Writing sanity check files # ------------------------------------------------------------------ if p['SHAPE_DEBUG_OUTPUTS']: # log WLOG(p, '', 'Saving debug sanity check files') # construct file names input_fp_file, tag1 = spirouConfig.Constants.SLIT_SHAPE_IN_FP_FILE(p) output_fp_file, tag2 = spirouConfig.Constants.SLIT_SHAPE_OUT_FP_FILE(p) input_hc_file, tag3 = spirouConfig.Constants.SLIT_SHAPE_IN_HC_FILE(p) output_hc_file, tag4 = spirouConfig.Constants.SLIT_SHAPE_OUT_HC_FILE(p) overlap_file, tag5 = spirouConfig.Constants.SLIT_SHAPE_OVERLAP_FILE(p) # write input fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) p = spirouImage.WriteImage(p, input_fp_file, loc['FPDATA'], hdict) # write output fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) p = spirouImage.WriteImage(p, output_fp_file, loc['FPDATA2'], hdict) # write input fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3) p = spirouImage.WriteImage(p, input_hc_file, loc['HCDATA'], hdict) # write output fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) p = spirouImage.WriteImage(p, output_hc_file, loc['HCDATA2'], hdict) # write overlap file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag5) p = spirouImage.WriteImage(p, overlap_file, loc['ORDER_OVERLAP'], hdict) # ---------------------------------------------------------------------- # Move to calibDB and update calibDB # ---------------------------------------------------------------------- if p['QC']: keydb = 'SHAPE' # copy shape file to the calibDB folder spirouDB.PutCalibFile(p, shapefits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, shapefitsname, fphdr) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, hcfile=None, fpfiles=None): """ cal_SLIT_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_SLIT_spirou.py [night_directory] [files] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) if hcfile is None or fpfiles is None: names, types = ['hcfile', 'fpfiles'], [str, str] customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1], types, names, last_multi=True) else: customargs = dict(hcfile=hcfile, fpfile=fpfiles) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='fpfiles') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, hcfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['HCFILE']) p, fpfitsfiles = spirouStartup.MultiFileSetup(p, files=p['FPFILES']) # set fiber (it doesn't matter with the 2D image but we need this to get # the lamp type for FPFILES and HCFILES, AB == C p['FIBER'] = 'AB' p['FIB_TYP'] = [p['FIBER']] fsource = __NAME__ + '/main()' p.set_sources(['FIBER', 'FIB_TYP'], fsource) # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # add a force plot off p['PLOT_PER_ORDER'] = PLOT_PER_ORDER p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Read FP and HC files # ---------------------------------------------------------------------- # read input fp and hc data rkwargs = dict(filename=fpfitsfiles[0], filenames=fpfitsfiles[1:], framemath='add') p, fpdata, fphdr = spirouImage.ReadImageAndCombine(p, **rkwargs) hcdata, hchdr, _, _ = spirouImage.ReadImage(p, hcfitsfilename) # add data and hdr to loc loc = ParamDict() loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr # set the source sources = ['HCDATA', 'HCHDR'] loc.set_sources(sources, 'spirouImage.ReadImage()') sources = ['FPDATA', 'FPHDR'] loc.set_sources(sources, 'spirouImage.ReadImage()') # --------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- hcdata = spirouImage.FixNonPreProcess(p, hcdata) fpdata = spirouImage.FixNonPreProcess(p, fpdata) # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # add a force plot off p['PLOT_PER_ORDER'] = PLOT_PER_ORDER p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, fphdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, fphdr, name='exptime') # get gain p = spirouImage.GetGain(p, fphdr, name='gain') # get lamp parameters p = spirouTHORCA.GetLampParams(p, hchdr) # get FP_FP DPRTYPE p = spirouImage.ReadParam(p, fphdr, 'KW_DPRTYPE', 'DPRTYPE', dtype=str) # ---------------------------------------------------------------------- # Correction of reference FP # ---------------------------------------------------------------------- # set the number of frames p['NBFRAMES'] = len(fpfitsfiles) p.set_source('NBFRAMES', __NAME__ + '.main()') # Correction of DARK p, fpdatac = spirouImage.CorrectForDark(p, fpdata, fphdr) # Resize hc data # rotate the image and convert from ADU/s to e- fpdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, fpdatac), p=p) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) fpdata1 = spirouImage.ResizeImage(p, fpdata, **bkwargs) # log change in data size WLOG(p, '', ('FPref Image format changed to {0}x{1}').format(*fpdata1.shape)) # Correct for the BADPIX mask (set all bad pixels to zero) bargs = [p, fpdata1, fphdr] p, fpdata1 = spirouImage.CorrectForBadPix(*bargs) p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True) # log progress WLOG(p, '', 'Cleaning FPref hot pixels') # correct hot pixels fpdata1 = spirouEXTOR.CleanHotpix(fpdata1, badpixmask) # add to loc loc['FPDATA1'] = fpdata1 loc.set_source('FPDATA1', __NAME__ + '.main()') # Log the number of dead pixels # get the number of bad pixels with warnings.catch_warnings(record=True) as _: n_bad_pix = np.nansum(fpdata1 <= 0) n_bad_pix_frac = n_bad_pix * 100 / np.product(fpdata1.shape) # Log number wmsg = 'Nb FPref dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ---------------------------------------------------------------------- # Correction of HC # ---------------------------------------------------------------------- # set the number of frames p['NBFRAMES'] = 1 p.set_source('NBFRAMES', __NAME__ + '.main()') # Correction of DARK p, hcdatac = spirouImage.CorrectForDark(p, hcdata, hchdr) # Resize hc data # rotate the image and convert from ADU/s to e- hcdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, hcdatac), p=p) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) hcdata1 = spirouImage.ResizeImage(p, hcdata, **bkwargs) # log change in data size WLOG(p, '', ('HC Image format changed to {0}x{1}').format(*hcdata1.shape)) # Correct for the BADPIX mask (set all bad pixels to zero) bargs = [p, hcdata1, hchdr] p, hcdata1 = spirouImage.CorrectForBadPix(*bargs) p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True) # log progress WLOG(p, '', 'Cleaning HC hot pixels') # correct hot pixels hcdata1 = spirouEXTOR.CleanHotpix(hcdata1, badpixmask) # add to loc loc['HCDATA1'] = hcdata1 loc.set_source('HCDATA1', __NAME__ + '.main()') # Log the number of dead pixels # get the number of bad pixels with warnings.catch_warnings(record=True) as _: n_bad_pix = np.nansum(hcdata1 <= 0) n_bad_pix_frac = n_bad_pix * 100 / np.product(hcdata1.shape) # Log number wmsg = 'Nb HC dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ------------------------------------------------------------------------- # get all FP_FP files # ------------------------------------------------------------------------- fpfilenames = spirouImage.FindFiles(p, filetype=p['DPRTYPE'], allowedtypes=p['ALLOWED_FP_TYPES']) # convert filenames to a numpy array fpfilenames = np.array(fpfilenames) # julian date to know which file we need to # process together fp_time = np.zeros(len(fpfilenames)) basenames, fp_exp, fp_pp_version, nightnames = [], [], [], [] # log progress WLOG(p, '', 'Reading all fp file headers') # looping through the file headers for it in range(len(fpfilenames)): # log progress wmsg = '\tReading file {0} / {1}' WLOG(p, 'info', wmsg.format(it + 1, len(fpfilenames))) # get fp filename fpfilename = fpfilenames[it] # get night name night_name = os.path.dirname(fpfilenames[it]).split(p['TMP_DIR'])[-1] # read data data_it, hdr_it, _, _ = spirouImage.ReadImage(p, fpfilename) # get header hdr = spirouImage.ReadHeader(p, filepath=fpfilenames[it]) # add MJDATE to dark times fp_time[it] = float(hdr[p['KW_ACQTIME'][0]]) # add other keys (for tabular output) basenames.append(os.path.basename(fpfilenames[it])) nightnames.append(night_name) fp_exp.append(float(hdr[p['KW_EXPTIME'][0]])) fp_pp_version.append(hdr[p['KW_PPVERSION'][0]]) # ------------------------------------------------------------------------- # match files by date # ------------------------------------------------------------------------- # log progress wmsg = 'Matching FP files by observation time (+/- {0} hrs)' WLOG(p, '', wmsg.format(p['DARK_MASTER_MATCH_TIME'])) # get the time threshold time_thres = p['FP_MASTER_MATCH_TIME'] # get items grouped by time matched_id = spirouImage.GroupFilesByTime(p, fp_time, time_thres) # ------------------------------------------------------------------------- # construct the master fp file (+ correct for dark/badpix) # ------------------------------------------------------------------------- cargs = [fpdata1, fpfilenames, matched_id] fpcube, transforms = spirouImage.ConstructMasterFP(p, *cargs) # log process wmsg1 = 'Master FP construction complete.' wmsg2 = '\tAdding {0} group images to form FP master image' WLOG(p, 'info', [wmsg1, wmsg2.format(len(fpcube))]) # sum the cube to make fp data masterfp = np.sum(fpcube, axis=0) # add to loc loc['MASTERFP'] = masterfp loc.set_source('MASTERFP', __NAME__ + '.main()') # ------------------------------------------------------------------ # Get localisation coefficients # ------------------------------------------------------------------ # original there is a loop but it is not used --> removed p = spirouImage.FiberParams(p, p['FIBER'], merge=True) # get localisation fit coefficients p, loc = spirouLOCOR.GetCoeffs(p, fphdr, loc) # ------------------------------------------------------------------ # Get master wave solution map # ------------------------------------------------------------------ # get master wave map masterwavefile = spirouDB.GetDatabaseMasterWave(p) # log process wmsg1 = 'Getting master wavelength grid' wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile)) WLOG(p, '', [wmsg1, wmsg2]) # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # read master wave map wout = spirouImage.GetWaveSolution(p, filename=masterwavefile, return_wavemap=True, quiet=True, return_header=True, fiber=wave_fiber) loc['MASTERWAVEP'], loc['MASTERWAVE'] = wout[:2] loc['MASTERWAVEHDR'], loc['WSOURCE'] = wout[2:] # set sources wsource = ['MASTERWAVEP', 'MASTERWAVE', 'MASTERWAVEHDR'] loc.set_sources(wsource, 'spirouImage.GetWaveSolution()') # ---------------------------------------------------------------------- # Read UNe solution # ---------------------------------------------------------------------- wave_u_ne, amp_u_ne = spirouImage.ReadLineList(p) loc['LL_LINE'], loc['AMPL_LINE'] = wave_u_ne, amp_u_ne source = __NAME__ + '.main() + spirouImage.ReadLineList()' loc.set_sources(['LL_LINE', 'AMPL_LINE'], source) # ---------------------------------------------------------------------- # Read cavity length file # ---------------------------------------------------------------------- loc['CAVITY_LEN_COEFFS'] = spirouImage.ReadCavityLength(p) source = __NAME__ + '.main() + spirouImage.ReadCavityLength()' loc.set_source('CAVITY_LEN_COEFFS', source) # ---------------------------------------------------------------------- # Calculate shape map # ---------------------------------------------------------------------- # calculate dx map loc = spirouImage.GetXShapeMap(p, loc) # if dx map is None we shouldn't continue if loc['DXMAP'] is None: fargs = [ loc['MAXDXMAPINFO'][0], loc['MAXDXMAPINFO'][1], loc['MAXDXMAPSTD'], p['SHAPE_QC_DXMAP_STD'] ] fmsg = ('The std of the dxmap for order {0} y-pixel {1} is too large.' ' std = {2} (limit = {3})'.format(*fargs)) wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(fmsg)) WLOG(p, 'warning', 'Cannot continue. Exiting.') # End Message p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals()) # calculate dymap loc = spirouImage.GetYShapeMap(p, loc, fphdr) # ------------------------------------------------------------------ # Need to straighten the dxmap # ------------------------------------------------------------------ # copy it first loc['DXMAP0'] = np.array(loc['DXMAP']) # straighten it loc['DXMAP'] = spirouImage.EATransform(loc['DXMAP'], dymap=loc['DYMAP']) # ------------------------------------------------------------------ # Need to straighten the hc data and fp data for debug # ------------------------------------------------------------------ # log progress WLOG(p, '', 'Shape finding complete. Applying transforms.') # apply very last update of the debananafication tkwargs = dict(dxmap=loc['DXMAP'], dymap=loc['DYMAP']) loc['HCDATA2'] = spirouImage.EATransform(loc['HCDATA1'], **tkwargs) loc['FPDATA2'] = spirouImage.EATransform(loc['FPDATA1'], **tkwargs) loc.set_sources(['HCDATA2', 'FPDATA2'], __NAME__ + '.main()') # ------------------------------------------------------------------ # Plotting # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # plots setup: start interactive plot sPlt.start_interactive_session(p) # plot the shape process for one order sPlt.slit_shape_angle_plot(p, loc) # end interactive section sPlt.end_interactive_session(p) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # TODO: Decide on some quality control criteria? # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # add to qc header lists qc_values.append(loc['MAXDXMAPSTD']) qc_names.append('DXMAP STD') qc_logic.append('DXMAP STD < {0}'.format(p['SHAPE_QC_DXMAP_STD'])) qc_pass.append(1) # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Writing FP big table # ------------------------------------------------------------------ # construct big fp table colnames = [ 'FILENAME', 'NIGHT', 'MJDATE', 'EXPTIME', 'PVERSION', 'GROUPID', 'DXREF', 'DYREF', 'A', 'B', 'C', 'D' ] values = [ basenames, nightnames, fp_time, fp_exp, fp_pp_version, matched_id, transforms[:, 0], transforms[:, 1], transforms[:, 2], transforms[:, 3], transforms[:, 4], transforms[:, 5] ] fptable = spirouImage.MakeTable(p, colnames, values) # ------------------------------------------------------------------ # Writing DXMAP to file # ------------------------------------------------------------------ # get the raw tilt file name raw_shape_file = os.path.basename(p['FITSFILENAME']) # construct file name and path shapexfits, tag = spirouConfig.Constants.SLIT_XSHAPE_FILE(p) shapexfitsname = os.path.basename(shapexfits) # Log that we are saving tilt file wmsg = 'Saving shape x information in file: {0}' WLOG(p, '', wmsg.format(shapexfitsname)) # Copy keys from fits file hdict = spirouImage.CopyOriginalKeys(fphdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='hcfile', values=p['HCFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE2'], dim1name='fpfile', values=p['FPFILES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # write tilt file to file p = spirouImage.WriteImageTable(p, shapexfits, image=loc['DXMAP'], table=fptable, hdict=hdict) # ------------------------------------------------------------------ # Writing DYMAP to file # ------------------------------------------------------------------ # get the raw tilt file name raw_shape_file = os.path.basename(p['FITSFILENAME']) # construct file name and path shapeyfits, tag = spirouConfig.Constants.SLIT_YSHAPE_FILE(p) shapeyfitsname = os.path.basename(shapeyfits) # Log that we are saving tilt file wmsg = 'Saving shape y information in file: {0}' WLOG(p, '', wmsg.format(shapeyfitsname)) # Copy keys from fits file hdict = spirouImage.CopyOriginalKeys(fphdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='hcfile', values=p['HCFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE2'], dim1name='fpfile', values=p['FPFILES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # write tilt file to file p = spirouImage.WriteImageTable(p, shapeyfits, image=loc['DYMAP'], table=fptable, hdict=hdict) # ------------------------------------------------------------------ # Writing Master FP to file # ------------------------------------------------------------------ # get the raw tilt file name raw_shape_file = os.path.basename(p['FITSFILENAME']) # construct file name and path fpmasterfits, tag = spirouConfig.Constants.SLIT_MASTER_FP_FILE(p) fpmasterfitsname = os.path.basename(fpmasterfits) # Log that we are saving tilt file wmsg = 'Saving master FP file: {0}' WLOG(p, '', wmsg.format(fpmasterfitsname)) # Copy keys from fits file hdict = spirouImage.CopyOriginalKeys(fphdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='hcfile', values=p['HCFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE2'], dim1name='fpfile', values=p['FPFILES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # write tilt file to file p = spirouImage.WriteImageTable(p, fpmasterfits, image=masterfp, table=fptable, hdict=hdict) # ------------------------------------------------------------------ # Writing sanity check files # ------------------------------------------------------------------ if p['SHAPE_DEBUG_OUTPUTS']: # log WLOG(p, '', 'Saving debug sanity check files') # construct file names input_fp_file, tag1 = spirouConfig.Constants.SLIT_SHAPE_IN_FP_FILE(p) output_fp_file, tag2 = spirouConfig.Constants.SLIT_SHAPE_OUT_FP_FILE(p) input_hc_file, tag3 = spirouConfig.Constants.SLIT_SHAPE_IN_HC_FILE(p) output_hc_file, tag4 = spirouConfig.Constants.SLIT_SHAPE_OUT_HC_FILE(p) bdxmap_file, tag5 = spirouConfig.Constants.SLIT_SHAPE_BDXMAP_FILE(p) # write input fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) p = spirouImage.WriteImage(p, input_fp_file, loc['FPDATA1'], hdict) # write output fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) p = spirouImage.WriteImage(p, output_fp_file, loc['FPDATA2'], hdict) # write input fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3) p = spirouImage.WriteImage(p, input_hc_file, loc['HCDATA1'], hdict) # write output fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) p = spirouImage.WriteImage(p, output_hc_file, loc['HCDATA2'], hdict) # write overlap file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag5) p = spirouImage.WriteImage(p, bdxmap_file, loc['DXMAP0'], hdict) # ---------------------------------------------------------------------- # Move to calibDB and update calibDB # ---------------------------------------------------------------------- if p['QC']: # add shape x keydb = 'SHAPEX' # copy shape file to the calibDB folder spirouDB.PutCalibFile(p, shapexfits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, shapexfitsname, fphdr) # add shape y keydb = 'SHAPEY' # copy shape file to the calibDB folder spirouDB.PutCalibFile(p, shapeyfits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, shapeyfitsname, fphdr) # add fp master keydb = 'FPMASTER' # copy shape file to the calibDB folder spirouDB.PutCalibFile(p, fpmasterfits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, fpmasterfitsname, fphdr) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, flatfile=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with arguments being None (i.e. get from sys.argv) name, lname = ['flatfile'], ['Reference file'] req, call, call_priority = [True], [flatfile], [True] # now get custom arguments customargs = spirouStartup.GetCustomFromRuntime(p, [0], [str], name, req, call, call_priority, lname) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='flatfile') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, reffile = spirouStartup.SingleFileSetup(p, filename=p['FLATFILE']) # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Get the required fiber type from the constants file # ---------------------------------------------------------------------- # get the fiber type (set to AB) p['FIBER'] = p['EM_FIB_TYPE'] p['FIBER_TYPES'] = [p['EM_FIB_TYPE']] # ---------------------------------------------------------------------- # Read flat image file # ---------------------------------------------------------------------- # read the image data (for the header only) image, hdr, ny, nx = spirouImage.ReadData(p, reffile) # ---------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- image = spirouImage.FixNonPreProcess(p, image) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- # create loc loc = ParamDict() # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # ---------------------------------------------------------------------- # Resize flat image # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- image2 = spirouImage.ConvertToE(spirouImage.FlipImage(p, image), p=p) # convert NaN to zeros image2 = np.where(~np.isfinite(image2), np.zeros_like(image2), image2) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) image2 = spirouImage.ResizeImage(p, image2, **bkwargs) # save flat to to loc and set source loc['IMAGE'] = image2 loc.set_sources(['image'], __NAME__ + '/main()') # log change in data size wmsg = 'Image format changed to {0}x{1}' WLOG(p, '', wmsg.format(*image2.shape)) # ---------------------------------------------------------------------- # Read shape or tilt slit angle # ---------------------------------------------------------------------- # set source of tilt file tsource = __NAME__ + '/main() + /spirouImage.ReadTiltFile' if p['IC_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES: # log progress WLOG(p, '', 'Debananafying (straightening) image') # get the shape map p, loc['SHAPE'] = spirouImage.ReadShapeMap(p, hdr) loc.set_source('SHAPE', tsource) else: # get tilts p, loc['TILT'] = spirouImage.ReadTiltFile(p, hdr) loc.set_source('TILT', tsource) # ---------------------------------------------------------------------- # Read blaze # ---------------------------------------------------------------------- # get tilts p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hdr) loc.set_source('BLAZE', __NAME__ + '/main() + /spirouImage.ReadBlazeFile') # set number of orders from blaze file loc['NBO'] = loc['BLAZE'].shape[0] loc.set_source('NBO', __NAME__ + '/main()') # ------------------------------------------------------------------ # Read wavelength solution # ------------------------------------------------------------------ # set source of wave file wsource = __NAME__ + '/main() + /spirouImage.GetWaveSolution' # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get wave image wout = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True, return_filename=True, fiber=wave_fiber) loc['WAVEPARAMS'], loc['WAVE'], loc['WAVEFILE'], loc['WSOURCE'] = wout loc.set_sources(['WAVEPARAMS', 'WAVE', 'WAVEFILE', 'WSOURCE'], wsource) # ------------------------------------------------------------------ # Get localisation coefficients # ------------------------------------------------------------------ # storage for fiber parameters loc['ALL_ACC'] = OrderedDict() loc['ALL_ASS'] = OrderedDict() # get this fibers parameters for fiber in p['FIBER_TYPES']: p = spirouImage.FiberParams(p, fiber, merge=True) # get localisation fit coefficients p, loc = spirouLOCOR.GetCoeffs(p, hdr, loc=loc) # save all fibers loc['ALL_ACC'][fiber] = loc['ACC'] loc['ALL_ASS'][fiber] = loc['ASS'] # ------------------------------------------------------------------ # Get telluric and telluric mask and add to loc # ------------------------------------------------------------------ # log process wmsg = 'Loading telluric model and locating "good" tranmission' WLOG(p, '', wmsg) # load telluric and get mask (add to loc) loc = spirouExM.get_telluric(p, loc) # ------------------------------------------------------------------ # Make 2D map of orders # ------------------------------------------------------------------ # log progress WLOG(p, '', 'Making 2D map of order locations') # make the 2D wave-image loc = spirouExM.order_profile(p, loc) # ------------------------------------------------------------------ # Make 2D map of wavelengths accounting for shape / tilt # ------------------------------------------------------------------ # log progress WLOG(p, '', 'Mapping pixels on to wavelength grid') # make the 2D map of wavelength loc = spirouExM.create_wavelength_image(p, loc) # ------------------------------------------------------------------ # Use spectra wavelength to create 2D image from wave-image # ------------------------------------------------------------------ if p['EM_SAVE_MASK_MAP'] or p['EM_SAVE_TELL_SPEC']: # log progress WLOG(p, '', 'Creating image from wave-image interpolation') # create image from waveimage wkwargs = dict(x=loc['TELL_X'], y=loc['TELL_Y']) loc = spirouExM.create_image_from_waveimage(loc, **wkwargs) else: loc['SPE'] = np.zeros_like(image2).astype(float) # ------------------------------------------------------------------ # Create 2D mask (min to max lambda + transmission threshold) # ------------------------------------------------------------------ if p['EM_SAVE_MASK_MAP']: # log progress WLOG(p, '', 'Creating wavelength/tranmission mask') # create mask loc = spirouExM.create_mask(p, loc) else: loc['TELL_MASK_2D'] = np.zeros_like(image2).astype(bool) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # TODO: Needs doing # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # add to qc header lists qc_values.append('None') qc_names.append('None') qc_logic.append('None') qc_pass.append(1) # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Construct parameters for header # ------------------------------------------------------------------ hdict = OrderedDict() # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) # set the input files if loc['SHAPE'] is not None: hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPE'], value=p['SHAPFILE']) else: hdict = spirouImage.AddKey(p, hdict, p['KW_CDBTILT'], value=p['TILTFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['FLATFILE']) # add name of the TAPAS y data hdict = spirouImage.AddKey(p, hdict, p['KW_EM_TELLY'], value=loc['TELLSPE']) # add name of the localisation fits file used hfile = os.path.basename(loc['LOCO_CTR_FILE']) hdict = spirouImage.AddKey(p, hdict, p['kw_EM_LOCFILE'], value=hfile) # add the max and min wavelength threshold hdict = spirouImage.AddKey(p, hdict, p['kw_EM_MINWAVE'], value=p['EM_MIN_LAMBDA']) hdict = spirouImage.AddKey(p, hdict, p['kw_EM_MAXWAVE'], value=p['EM_MAX_LAMBDA']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # add the transmission cut hdict = spirouImage.AddKey(p, hdict, p['kw_EM_TRASCUT'], value=p['EM_TELL_THRESHOLD']) # ------------------------------------------------------------------ # Deal with output preferences # ------------------------------------------------------------------ # add bad pixel map (if required) if p['EM_COMBINED_BADPIX']: # get bad pix mask (True where bad) badpixmask, bhdr, badfile = spirouImage.GetBadPixMap(p, hdr) goodpixels = badpixmask == 0 # apply mask (multiply) loc['TELL_MASK_2D'] = loc['TELL_MASK_2D'] & goodpixels.astype(bool) else: badfile = 'None' # add to hdict hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=badfile) # convert waveimage mask into float array loc['TELL_MASK_2D'] = loc['TELL_MASK_2D'].astype('float') # check EM_OUTPUT_TYPE and deal with set to "all" if p['EM_OUTPUT_TYPE'] not in ["drs", "raw", "preprocess", "all"]: emsg1 = '"EM_OUTPUT_TYPE" not understood' emsg2 = ' must be either "drs", "raw" or "preprocess"' emsg3 = ' currently EM_OUTPUT_TYPE="{0}"'.format(p['EM_OUTPUT_TYPE']) WLOG(p, 'error', [emsg1, emsg2, emsg3]) outputs = [] elif p['EM_OUTPUT_TYPE'] != 'all': outputs = [str(p['EM_OUTPUT_TYPE'])] else: outputs = ["drs", "raw", "preprocess"] # ---------------------------------------------------------------------- # loop around output types # ---------------------------------------------------------------------- for output in outputs: # log progress WLOG(p, '', 'Processing {0} outputs'.format(output)) # change EM_OUTPUT_TYPE p['EM_OUTPUT_TYPE'] = output # copy arrays out_spe = np.array(loc['SPE']) out_wave = np.array(loc['WAVEIMAGE']) out_mask = np.array(loc['TELL_MASK_2D']) # change image size if needed if output in ["raw", "preprocess"]: kk = dict(xsize=image.shape[1], ysize=image.shape[0]) if p['EM_SAVE_TELL_SPEC']: WLOG(p, '', 'Resizing/Flipping SPE') out_spe = spirouExM.unresize(p, out_spe, **kk) WLOG(p, '', 'Rescaling SPE') out_spe = out_spe / (p['GAIN'] * p['EXPTIME']) if p['EM_SAVE_WAVE_MAP']: WLOG(p, '', 'Resizing/Flipping WAVEIMAGE') out_wave = spirouExM.unresize(p, out_wave, **kk) if p['EM_SAVE_MASK_MAP']: WLOG(p, '', 'Resizing/Flipping TELL_MASK_2D') out_mask = spirouExM.unresize(p, out_mask, **kk) # if raw need to rotate (undo pre-processing) if output == "raw": if p['EM_SAVE_TELL_SPEC']: WLOG(p, '', 'Rotating SPE') out_spe = np.rot90(out_spe, 1) if p['EM_SAVE_WAVE_MAP']: WLOG(p, '', 'Rotating WAVEIMAGE') out_wave = np.rot90(out_wave, 1) if p['EM_SAVE_MASK_MAP']: WLOG(p, '', 'Rotating TELL_MASK_2D') out_mask = np.rot90(out_mask, 1) # ---------------------------------------------------------------------- # save 2D spectrum, wavelength image and mask to file # ---------------------------------------------------------------------- # save telluric spectrum if p['EM_SAVE_TELL_SPEC']: # construct spectrum filename specfitsfile, tag = spirouConfig.Constants.EM_SPE_FILE(p) specfilename = os.path.split(specfitsfile)[-1] # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # log progress wmsg = 'Writing spectrum to file {0}' WLOG(p, '', wmsg.format(specfilename)) # write to file p = spirouImage.WriteImage(p, specfitsfile, out_spe, hdict=hdict) # ---------------------------------------------------------------------- # save wave map if p['EM_SAVE_WAVE_MAP']: # construct waveimage filename wavefitsfile, tag = spirouConfig.Constants.EM_WAVE_FILE(p) wavefilename = os.path.split(wavefitsfile)[-1] # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # log progress wmsg = 'Writing wave image to file {0}' WLOG(p, '', wmsg.format(wavefilename)) # write to file p = spirouImage.WriteImage(p, wavefitsfile, out_wave, hdict=hdict) # ---------------------------------------------------------------------- # save mask file if p['EM_SAVE_MASK_MAP']: # construct tell mask 2D filename maskfitsfile, tag = spirouConfig.Constants.EM_MASK_FILE(p) maskfilename = os.path.split(maskfitsfile)[-1] # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # log progress wmsg = 'Writing telluric mask to file {0}' WLOG(p, '', wmsg.format(maskfilename)) # convert boolean mask to integers writablemask = np.array(out_mask, dtype=float) # write to file p = spirouImage.WriteImage(p, maskfitsfile, writablemask, hdict=hdict) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): """ cal_FF_RAW_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_FF_RAW_spirou.py [night_directory] [files] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) p = spirouStartup.InitialFileSetup(p, calibdb=True) # run specific start up p['FIB_TYPE'] = p['FIBER_TYPES'] p.set_source('FIB_TYPE', __NAME__ + '__main__()') # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add') # ---------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- data = spirouImage.FixNonPreProcess(p, data) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # ---------------------------------------------------------------------- # Correction of DARK # ---------------------------------------------------------------------- p, datac = spirouImage.CorrectForDark(p, data, hdr) # ---------------------------------------------------------------------- # Resize image # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- data = spirouImage.ConvertToADU(spirouImage.FlipImage(p, datac), p=p) # convert NaN to zeros data0 = np.where(~np.isfinite(data), np.zeros_like(data), data) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) data1 = spirouImage.ResizeImage(p, data0, **bkwargs) # log change in data size WLOG(p, '', ('Image format changed to ' '{0}x{1}').format(*data1.shape[::-1])) # ---------------------------------------------------------------------- # Correct for the BADPIX mask (set all bad pixels to zero) # ---------------------------------------------------------------------- p, data1 = spirouImage.CorrectForBadPix(p, data1, hdr) # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- # get the number of bad pixels n_bad_pix = np.sum(~np.isfinite(data1)) n_bad_pix_frac = n_bad_pix * 100 / np.product(data1.shape) # Log number wmsg = 'Nb dead pixels = {0} / {1:.4f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ---------------------------------------------------------------------- # Get the miny, maxy and max_signal for the central column # ---------------------------------------------------------------------- # get the central column y = data1[p['IC_CENT_COL'], :] # get the min max and max signal using box smoothed approach miny, maxy, max_signal, diff_maxmin = spirouBACK.MeasureMinMaxSignal(p, y) # Log max average flux/pixel wmsg = ('Maximum average flux (95th percentile) /pixel in the spectrum: ' '{0:.1f} [ADU]') WLOG(p, 'info', wmsg.format(max_signal / p['NBFRAMES'])) # ---------------------------------------------------------------------- # Background computation # ---------------------------------------------------------------------- # p['ic_bkgr_percent'] = 3.0 if p['IC_DO_BKGR_SUBTRACTION']: # log that we are doing background measurement WLOG(p, '', 'Doing background measurement on raw frame') # get the bkgr measurement bargs = [p, data1, hdr] # background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs) p, background = spirouBACK.MeasureBackgroundMap(*bargs) else: background = np.zeros_like(data1) p['BKGRDFILE'] = 'None' p.set_source('BKGRDFILE', __NAME__ + '.main()') # apply background correction to data data1 = data1 - background # ---------------------------------------------------------------------- # Read tilt slit angle # ---------------------------------------------------------------------- # define loc storage parameter dictionary loc = ParamDict() # get tilts if p['IC_FF_EXTRACT_TYPE'] not in EXTRACT_SHAPE_TYPES: p, loc['TILT'] = spirouImage.ReadTiltFile(p, hdr) else: loc['TILT'] = None loc.set_source('TILT', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Get all fiber data (for all fibers) # ---------------------------------------------------------------------- # TODO: This is temp solution for options 5a and 5b loc_fibers = spirouLOCOR.GetFiberData(p, hdr) # ------------------------------------------------------------------ # Deal with debananafication # ------------------------------------------------------------------ # if mode 4a or 4b we need to straighten in x only if p['IC_FF_EXTRACT_TYPE'] in ['4a', '4b']: # get the shape parameters p, shapem_x = spirouImage.GetShapeX(p, hdr) p, shape_local = spirouImage.GetShapeLocal(p, hdr) # log progress WLOG(p, '', 'Debananafying (straightening) image') # apply shape transforms targs = dict(lin_transform_vect=shape_local, dxmap=shapem_x) data2 = spirouImage.EATransform(data1, **targs) # if mode 5a or 5b we need to straighten in x and y using the # polynomial fits for location elif p['IC_FF_EXTRACT_TYPE'] in ['5a', '5b']: # get the shape parameters p, shapem_x = spirouImage.GetShapeX(p, hdr) p, shapem_y = spirouImage.GetShapeY(p, hdr) p, shape_local = spirouImage.GetShapeLocal(p, hdr) p, fpmaster = spirouImage.GetFPMaster(p, hdr) # get the bad pixel map bkwargs = dict(return_map=True, quiet=True) p, badpix = spirouImage.CorrectForBadPix(p, data1, hdr, **bkwargs) # log progress WLOG(p, '', 'Cleaning image') # clean the image data1 = spirouEXTOR.CleanHotpix(data1, badpix) # log progress WLOG(p, '', 'Debananafying (straightening) image') # apply shape transforms targs = dict(lin_transform_vect=shape_local, dxmap=shapem_x, dymap=shapem_y) data2 = spirouImage.EATransform(data1, **targs) # in any other mode we do not straighten else: data2 = np.array(data1) # ---------------------------------------------------------------------- # Fiber loop # ---------------------------------------------------------------------- # loop around fiber types for fiber in p['FIB_TYPE']: # set fiber in p p['FIBER'] = fiber p.set_source('FIBER', __NAME__ + '/main()') # get fiber parameters params2add = spirouImage.FiberParams(p, p['FIBER']) for param in params2add: p[param] = params2add[param] p.set_source(param, __NAME__ + '.main()') # ------------------------------------------------------------------ # Get fiber specific parameters from loc_fibers # ------------------------------------------------------------------ # get this fibers parameters p = spirouImage.FiberParams(p, p['FIBER'], merge=True) # get localisation parameters for key in loc_fibers[fiber]: loc[key] = loc_fibers[fiber][key] loc.set_source(key, loc_fibers[fiber].sources[key]) # get locofile source p['LOCOFILE'] = loc['LOCOFILE'] p.set_source('LOCOFILE', loc.sources['LOCOFILE']) # get the order_profile order_profile = loc_fibers[fiber]['ORDER_PROFILE'] # ------------------------------------------------------------------ # Set up Extract storage # ------------------------------------------------------------------ # Create array to store extraction (for each order and each pixel # along order) loc['E2DS'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) loc['E2DSLL'] = [] # Create array to store the blaze (for each order and at each pixel # along order) loc['BLAZE'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) # Create array to store the flat (for each order and at each pixel # along order) loc['FLAT'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) # Create array to store the signal to noise ratios for each order loc['SNR'] = np.zeros(loc['NUMBER_ORDERS']) # Create array to store the rms for each order loc['RMS'] = np.zeros(loc['NUMBER_ORDERS']) # Manually set the sigdet to be used in extraction weighting if p['IC_FF_SIGDET'] > 0: p['SIGDET'] = float(p['IC_FF_SIGDET']) # ------------------------------------------------------------------ # Extract orders # old code time: 1 loop, best of 3: 22.3 s per loop # new code time: 3.16 s ± 237 ms per loop # ------------------------------------------------------------------ # get limits of order extraction valid_orders = spirouFLAT.GetValidOrders(p, loc) # loop around each order for order_num in valid_orders: # extract this order eargs = [p, loc, data2, order_num] ekwargs = dict(mode=p['IC_FF_EXTRACT_TYPE'], order_profile=order_profile) with warnings.catch_warnings(record=True) as w: eout = spirouEXTOR.Extraction(*eargs, **ekwargs) # deal with different return if p['IC_FF_EXTRACT_TYPE'] in EXTRACT_LL_TYPES: e2ds, e2dsll, cpt = eout else: e2ds, cpt = eout e2dsll = None # calculate the noise range1, range2 = p['IC_EXT_RANGE1'], p['IC_EXT_RANGE2'] noise = p['SIGDET'] * np.sqrt(range1 + range2) # get window size blaze_win1 = int(data2.shape[1] / 2) - p['IC_EXTFBLAZ'] blaze_win2 = int(data2.shape[1] / 2) + p['IC_EXTFBLAZ'] # get average flux per pixel flux = np.nansum( e2ds[blaze_win1:blaze_win2]) / (2 * p['IC_EXTFBLAZ']) # calculate signal to noise ratio = flux/sqrt(flux + noise^2) snr = flux / np.sqrt(flux + noise**2) # remove edge of orders at low S/N with warnings.catch_warnings(record=True) as _: blazemask = e2ds < (flux / p['IC_FRACMINBLAZE']) e2ds = np.where(blazemask, np.nan, e2ds) # e2ds = np.where(e2ds < p['IC_MINBLAZE'], 0., e2ds) # calcualte the blaze function blaze = spirouFLAT.MeasureBlazeForOrder(p, e2ds) # calculate the flat flat = e2ds / blaze # calculate the rms rms = np.nanstd(flat) # log the SNR RMS wmsg = 'On fiber {0} order {1}: S/N= {2:.1f} - FF rms={3:.2f} %' wargs = [fiber, order_num, snr, rms * 100.0] WLOG(p, '', wmsg.format(*wargs)) # add calculations to storage loc['E2DS'][order_num] = e2ds loc['SNR'][order_num] = snr loc['RMS'][order_num] = rms loc['BLAZE'][order_num] = blaze loc['FLAT'][order_num] = flat # save the longfile if p['IC_FF_EXTRACT_TYPE'] in EXTRACT_LL_TYPES: loc['E2DSLL'].append(e2dsll) # set sources source = __NAME__ + '/main()()' loc.set_sources(['e2ds', 'SNR', 'RMS', 'blaze', 'flat'], source) # Log if saturation level reached satvalue = (flux / p['GAIN']) / (range1 + range2) if satvalue > (p['QC_LOC_FLUMAX'] * p['NBFRAMES']): wmsg = 'SATURATION LEVEL REACHED on Fiber {0} order={1}' WLOG(p, 'warning', wmsg.format(fiber, order_num)) # ---------------------------------------------------------------------- # Plots # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot all orders or one order if p['IC_FF_PLOT_ALL_ORDERS']: # plot image with all order fits (slower) sPlt.ff_aorder_fit_edges(p, loc, data1) else: # plot image with selected order fit and edge fit (faster) sPlt.ff_sorder_fit_edges(p, loc, data1) # plot tilt adjusted e2ds and blaze for selected order sPlt.ff_sorder_tiltadj_e2ds_blaze(p, loc) # plot flat for selected order sPlt.ff_sorder_flat(p, loc) # plot the RMS for all but skipped orders # sPlt.ff_rms_plot(p, loc) if p['IC_FF_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES: sPlt.ff_debanana_plot(p, loc, data2) # ------------------------------------------------------------------ # Quality control # ------------------------------------------------------------------ passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # saturation check: check that the max_signal is lower than # qc_max_signal # if max_signal > (p['QC_MAX_SIGNAL'] * p['nbframes']): # fmsg = 'Too much flux in the image (max authorized={0})' # fail_msg.append(fmsg.format(p['QC_MAX_SIGNAL'] * p['nbframes'])) # passed = False # # For some reason this test is ignored in old code # passed = True # WLOG(p, 'info', fail_msg[-1]) # get mask for removing certain orders in the RMS calculation remove_orders = np.array(p['FF_RMS_PLOT_SKIP_ORDERS']) mask = np.in1d(np.arange(len(loc['RMS'])), remove_orders) # apply mask and calculate the maximum RMS max_rms = np.nanmax(loc['RMS'][~mask]) # apply the quality control based on the new RMS if max_rms > p['QC_FF_RMS']: fmsg = 'abnormal RMS of FF ({0:.3f} > {1:.3f})' fail_msg.append(fmsg.format(max_rms, p['QC_FF_RMS'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(max_rms) qc_names.append('max_rms') qc_logic.append('max_rms > {0:.3f}'.format(p['QC_FF_RMS'])) # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: wmsg = 'QUALITY CONTROL SUCCESSFUL - Well Done -' WLOG(p, 'info', wmsg) p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ---------------------------------------------------------------------- # Store Blaze in file # ---------------------------------------------------------------------- # get raw flat filename raw_flat_file = os.path.basename(p['FITSFILENAME']) e2dsllfits, tag4 = spirouConfig.Constants.EXTRACT_E2DSLL_FILE(p) # get extraction method and function efout = spirouEXTOR.GetExtMethod(p, p['IC_FF_EXTRACT_TYPE']) extmethod, extfunc = efout # construct filename blazefits, tag1 = spirouConfig.Constants.FF_BLAZE_FILE(p) blazefitsname = os.path.split(blazefits)[-1] # log that we are saving blaze file wmsg = 'Saving blaze spectrum for fiber: {0} in {1}' WLOG(p, '', wmsg.format(fiber, blazefitsname)) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # define new keys to add hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_FIBER'], value=p['FIBER']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBACK'], value=p['BKGRDFILE']) if p['IC_FF_EXTRACT_TYPE'] not in EXTRACT_SHAPE_TYPES: hdict = spirouImage.AddKey(p, hdict, p['KW_CDBTILT'], value=p['TILTFILE']) if p['IC_FF_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES: hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEX'], value=p['SHAPEXFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEY'], value=p['SHAPEYFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPE'], value=p['SHAPEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFPMASTER'], value=p['FPMASTERFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # add some properties back hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_SIGDET']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_CONAD']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # copy extraction method and function to header # (for reproducibility) hdict = spirouImage.AddKey(p, hdict, p['KW_E2DS_EXTM'], value=extmethod) hdict = spirouImage.AddKey(p, hdict, p['KW_E2DS_FUNC'], value=extfunc) # output keys hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) # write 1D list of the SNR hdict = spirouImage.AddKey1DList(p, hdict, p['KW_EXTRA_SN'], values=loc['SNR']) # write center fits and add header keys (via hdict) p = spirouImage.WriteImage(p, blazefits, loc['BLAZE'], hdict) # ---------------------------------------------------------------------- # Store Flat-field in file # ---------------------------------------------------------------------- # construct filename flatfits, tag2 = spirouConfig.Constants.FF_FLAT_FILE(p) flatfitsname = os.path.split(flatfits)[-1] # log that we are saving blaze file wmsg = 'Saving FF spectrum for fiber: {0} in {1}' WLOG(p, '', wmsg.format(fiber, flatfitsname)) # write 1D list of the RMS (add to hdict from blaze) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_FLAT_RMS'], values=loc['RMS']) # write center fits and add header keys (via same hdict as blaze) p = spirouImage.WriteImage(p, flatfits, loc['FLAT'], hdict) # Save E2DSLL file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) if p['IC_FF_EXTRACT_TYPE'] in EXTRACT_LL_TYPES: llstack = np.vstack(loc['E2DSLL']) p = spirouImage.WriteImage(p, e2dsllfits, llstack, hdict) # ------------------------------------------------------------------ # Update the calibration database # ------------------------------------------------------------------ if p['QC'] == 1: # copy flatfits to calibdb keydb = 'FLAT_' + p['FIBER'] # copy localisation file to the calibDB folder spirouDB.PutCalibFile(p, flatfits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, flatfitsname, hdr) # copy blazefits to calibdb keydb = 'BLAZE_' + p['FIBER'] # copy localisation file to the calibDB folder spirouDB.PutCalibFile(p, blazefits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, blazefitsname, hdr) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())