def main(night_name=None, e2dsfile=None, mask=None, rv=None, width=None, step=None): """ cal_CCF_E2DS_spirou.py main function, if arguments are None uses arguments from run time i.e.: cal_CCF_E2DS_spirou.py [night_directory] [E2DSfilename] [mask] [RV] [width] [step] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param e2dsfile: string, the E2DS file to use :param mask: string, the mask file to use (i.e. "UrNe.mas") :param rv: float, the target RV to use :param width: float, the CCF width to use :param step: float, the CCF step to use :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with arguments being None (i.e. get from sys.argv) pos = [0, 1, 2, 3, 4] fmt = [str, str, float, float, float] name = ['e2dsfile', 'ccf_mask', 'target_rv', 'ccf_width', 'ccf_step'] lname = ['input_file', 'CCF_mask', 'RV', 'CCF_width', 'CCF_step'] req = [True, True, True, False, False] call = [e2dsfile, mask, rv, width, step] call_priority = [True, True, True, True, True] # now get custom arguments customargs = spirouStartup.GetCustomFromRuntime(p, pos, fmt, name, req, call, call_priority, lname) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='e2dsfile', mainfitsdir='reduced') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, e2dsfilename = spirouStartup.SingleFileSetup(p, filename=p['E2DSFILE']) # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Deal with optional run time arguments # ---------------------------------------------------------------------- # define default arguments (if ccf_width and ccf_step are not defined # in function call or run time arguments if 'ccf_width' not in p: p['CCF_WIDTH'] = p['IC_CCF_WIDTH'] if 'ccf_step' not in p: p['CCF_STEP'] = p['IC_CCF_STEP'] # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data e2ds, hdr, nbo, nx = spirouImage.ReadData(p, e2dsfilename) # add to loc loc = ParamDict() loc['E2DS'] = e2ds loc['NUMBER_ORDERS'] = nbo loc.set_sources(['E2DS', 'number_orders'], __NAME__ + '/main()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # get acquisition time p = spirouImage.GetAcqTime(p, hdr, name='acqtime', kind='julian') # get obj name p = spirouImage.ReadParam(p, hdr, 'KW_OBJNAME', name='OBJNAME', dtype=str) bjdref = p['ACQTIME'] # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # ---------------------------------------------------------------------- # Earth Velocity calculation # ---------------------------------------------------------------------- if p['IC_IMAGE_TYPE'] == 'H4RG': p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, hdr) # ---------------------------------------------------------------------- # Read wavelength solution # ---------------------------------------------------------------------- # log WLOG(p, '', 'Reading wavelength solution ') # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get wave image wout = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True, return_filename=True, fiber=wave_fiber) param_ll, wave_ll, wavefile, wsource = wout # save to storage loc['PARAM_LL'], loc['WAVE_LL'], loc['WAVEFILE'], loc['WSOURCE'] = wout source = __NAME__ + '/main() + spirouTHORCA.GetWaveSolution()' loc.set_sources(['WAVE_LL', 'PARAM_LL', 'WAVEFILE', 'WSOURCE'], source) # ---------------------------------------------------------------------- # Read Flat file # ---------------------------------------------------------------------- # TODO We do not need to correct FLAT # log # WLOG(p, '', 'Reading Flat-Field ') # get flat # loc['FLAT'] = spirouImage.ReadFlatFile(p, hdr) # loc.set_source('FLAT', __NAME__ + '/main() + /spirouImage.ReadFlatFile') # get all values in flat that are zero to 1 # loc['FLAT'] = np.where(loc['FLAT'] == 0, 1.0, loc['FLAT']) # get blaze # p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hdr) p, blaze0 = spirouImage.ReadBlazeFile(p, hdr) # ---------------------------------------------------------------------- # Preliminary set up = no flat, no blaze # ---------------------------------------------------------------------- # reset flat to all ones # loc['FLAT'] = np.ones((nbo, nx)) # set blaze to all ones (if not bug in correlbin !!! # TODO Check why Blaze makes bugs in correlbin loc['BLAZE'] = np.ones((nbo, nx)) # set sources # loc.set_sources(['flat', 'blaze'], __NAME__ + '/main()') loc.set_sources(['blaze'], __NAME__ + '/main()') # Modification of E2DS array with N.A.N if np.isnan(np.sum(e2ds)): WLOG(p, 'warning', 'NaN values found in e2ds, converting process') # First basic approach Replacing N.A.N by zeros # e2ds[np.isnan(e2ds)] = 0 # Second approach replacing N.A.N by the Adjusted Blaze e2dsb = e2ds / blaze0 for i in np.arange(len(e2ds)): with warnings.catch_warnings(record=True) as _: rap = np.mean(e2dsb[i][np.isfinite(e2dsb[i])]) if np.isnan(rap): rap = 0.0 e2ds[i] = np.where(np.isfinite(e2dsb[i]), e2ds[i], blaze0[i] * rap) # ---------------------------------------------------------------------- # correct extracted image for flat # ---------------------------------------------------------------------- # loc['E2DSFF'] = e2ds/loc['FLAT'] # loc['E2DSFF'] = e2ds*1. loc['E2DSFF'] = e2ds loc.set_source('E2DSFF', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Compute photon noise uncertainty for reference file # ---------------------------------------------------------------------- # set up the arguments for DeltaVrms2D dargs = [loc['E2DS'], loc['WAVE_LL']] dkwargs = dict(sigdet=p['IC_DRIFT_NOISE'], size=p['IC_DRIFT_BOXSIZE'], threshold=p['IC_DRIFT_MAXFLUX']) # run DeltaVrms2D dvrmsref, wmeanref = spirouRV.DeltaVrms2D(*dargs, **dkwargs) # save to loc loc['DVRMSREF'], loc['WMEANREF'] = dvrmsref, wmeanref loc.set_sources(['dvrmsref', 'wmeanref'], __NAME__ + '/main()()') # log the estimated RV uncertainty # wmsg = 'On fiber {0} estimated RV uncertainty on spectrum is {1:.3f} m/s' # WLOG(p, 'info', wmsg.format(p['FIBER'], wmeanref)) wmsg = 'On fiber estimated RV uncertainty on spectrum is {0:.3f} m/s' WLOG(p, 'info', wmsg.format(wmeanref)) # TEST N.A.N # loc['E2DSFF'][20:22,2000:3000]=np.nan # e2ds[20:30,1000:3000]=np.nan # ---------------------------------------------------------------------- # Reference plots # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot FP spectral order sPlt.drift_plot_selected_wave_ref(p, loc, x=loc['WAVE_LL'], y=loc['E2DS']) # plot photon noise uncertainty sPlt.drift_plot_photon_uncertainty(p, loc) # ---------------------------------------------------------------------- # Get template RV (from ccf_mask) # ---------------------------------------------------------------------- # get the CCF mask from file (check location of mask) loc = spirouRV.GetCCFMask(p, loc) # check and deal with mask in microns (should be in nm) if np.mean(loc['LL_MASK_CTR']) < 2.0: loc['LL_MASK_CTR'] *= 1000.0 loc['LL_MASK_D'] *= 1000.0 # ---------------------------------------------------------------------- # Do correlation # ---------------------------------------------------------------------- # calculate and fit the CCF loc = spirouRV.Coravelation(p, loc) # ---------------------------------------------------------------------- # Correlation stats # ---------------------------------------------------------------------- # get the maximum number of orders to use nbmax = p['CCF_NUM_ORDERS_MAX'] # get the average ccf loc['AVERAGE_CCF'] = np.nansum(loc['CCF'][:nbmax], axis=0) # normalize the average ccf normalized_ccf = loc['AVERAGE_CCF'] / np.max(loc['AVERAGE_CCF']) # get the fit for the normalized average ccf ccf_res, ccf_fit = spirouRV.FitCCF(p, loc['RV_CCF'], normalized_ccf, fit_type=0) loc['CCF_RES'] = ccf_res loc['CCF_FIT'] = ccf_fit # get the max cpp loc['MAXCPP'] = np.nansum(loc['CCF_MAX']) / np.nansum( loc['PIX_PASSED_ALL']) # get the RV value from the normalised average ccf fit center location loc['RV'] = float(ccf_res[1]) # get the contrast (ccf fit amplitude) loc['CONTRAST'] = np.abs(100 * ccf_res[0]) # get the FWHM value loc['FWHM'] = ccf_res[2] * spirouCore.spirouMath.fwhm() # ---------------------------------------------------------------------- # set the source keys = [ 'average_ccf', 'maxcpp', 'rv', 'contrast', 'fwhm', 'ccf_res', 'ccf_fit' ] loc.set_sources(keys, __NAME__ + '/main()') # ---------------------------------------------------------------------- # log the stats wmsg = ('Correlation: C={0:.1f}[%] RV={1:.5f}[km/s] ' 'FWHM={2:.4f}[km/s] maxcpp={3:.1f}') wargs = [loc['CONTRAST'], loc['RV'], loc['FWHM'], loc['MAXCPP']] WLOG(p, 'info', wmsg.format(*wargs)) # ---------------------------------------------------------------------- # rv ccf plot # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # Plot rv vs ccf (and rv vs ccf_fit) sPlt.ccf_rv_ccf_plot(p, loc['RV_CCF'], normalized_ccf, ccf_fit) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # TODO: Needs doing # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # add to qc header lists qc_values.append('None') qc_names.append('None') qc_logic.append('None') qc_pass.append(1) # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ---------------------------------------------------------------------- # archive ccf to table # ---------------------------------------------------------------------- # construct filename res_table_file = spirouConfig.Constants.CCF_TABLE_FILE(p) # log progress WLOG(p, '', 'Archiving CCF on file {0}'.format(res_table_file)) # define column names columns = ['order', 'maxcpp', 'nlines', 'contrast', 'RV', 'sig'] # define values for each column values = [ loc['ORDERS'], loc['CCF_MAX'] / loc['PIX_PASSED_ALL'], loc['TOT_LINE'], np.abs(100 * loc['CCF_ALL_RESULTS'][:, 0]), loc['CCF_ALL_RESULTS'][:, 1], loc['CCF_ALL_RESULTS'][:, 2] ] # define the format for each column formats = ['2.0f', '5.0f', '4.0f', '4.1f', '9.4f', '7.4f'] # construct astropy table from column names, values and formats table = spirouImage.MakeTable(p, columns, values, formats) # save table to file spirouImage.WriteTable(p, table, res_table_file, fmt='ascii') # ---------------------------------------------------------------------- # archive ccf to fits file # ---------------------------------------------------------------------- raw_infile = os.path.basename(p['E2DSFILE']) # construct folder and filename corfile, tag = spirouConfig.Constants.CCF_FITS_FILE(p) corfilename = os.path.split(corfile)[-1] # log that we are archiving the CCF on file WLOG(p, '', 'Archiving CCF on file {0}'.format(corfilename)) # get constants from p mask = p['CCF_MASK'] # if file exists remove it if os.path.exists(corfile): os.remove(corfile) # add the average ccf to the end of ccf data = np.vstack([loc['CCF'], loc['AVERAGE_CCF']]) # add drs keys hdict = spirouImage.CopyOriginalKeys(hdr) hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['E2DSFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_INCCFMASK'], value=p['CCF_MASK']) hdict = spirouImage.AddKey(p, hdict, p['KW_INRV'], value=p['TARGET_RV']) hdict = spirouImage.AddKey(p, hdict, p['KW_INWIDTH'], value=p['CCF_WIDTH']) hdict = spirouImage.AddKey(p, hdict, p['KW_INSTEP'], value=p['CCF_STEP']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # add CCF keys hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CTYPE'], value='km/s') hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CRVAL'], value=loc['RV_CCF'][0]) # the rv step rvstep = np.abs(loc['RV_CCF'][0] - loc['RV_CCF'][1]) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CDELT'], value=rvstep) # add ccf stats hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_RV'], value=loc['CCF_RES'][1]) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_RVC'], value=loc['RV']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_FWHM'], value=loc['FWHM']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_WMREF'], value=loc['WMEANREF']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CONTRAST'], value=loc['CONTRAST']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_MAXCPP'], value=loc['MAXCPP']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_MASK'], value=p['CCF_MASK']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_LINES'], value=np.nansum(loc['TOT_LINE'])) # add berv values hdict = spirouImage.AddKey(p, hdict, p['KW_BERV'], value=loc['BERV']) hdict = spirouImage.AddKey(p, hdict, p['KW_BJD'], value=loc['BJD']) hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_MAX'], value=loc['BERV_MAX']) # write image and add header keys (via hdict) p = spirouImage.WriteImage(p, corfile, data, hdict) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None, fiber_type=None, **kwargs): """ cal_DRIFT_E2DS_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_DRIFT_E2DS_spirou.py [night_directory] [files] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :param fiber_type: string, if None does all fiber types (defined in constants_SPIROU FIBER_TYPES (default is AB, A, B, C if defined then only does this fiber type (but must be in FIBER_TYPES) :param kwargs: any keyword to overwrite constant in param dict "p" :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) p = spirouStartup.InitialFileSetup(p, calibdb=True) # deal with fiber type if fiber_type is None: fiber_type = p['FIBER_TYPES'] if type(fiber_type) == str: if fiber_type.upper() == 'ALL': fiber_type = p['FIBER_TYPES'] elif fiber_type in p['FIBER_TYPES']: fiber_type = [fiber_type] else: emsg = 'fiber_type="{0}" not understood' WLOG(p, 'error', emsg.format(fiber_type)) # set fiber type p['FIB_TYPE'] = fiber_type p.set_source('FIB_TYPE', __NAME__ + '__main__()') # Overwrite keys from source for kwarg in kwargs: p[kwarg] = kwargs[kwarg] # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add') # ---------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- data = spirouImage.FixNonPreProcess(p, data) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # now change the value of sigdet if require if p['IC_EXT_SIGDET'] > 0: p['SIGDET'] = float(p['IC_EXT_SIGDET']) # get DPRTYPE from header (Will have it if valid) p = spirouImage.ReadParam(p, hdr, 'KW_DPRTYPE', required=False, dtype=str) # check the DPRTYPE is not None if (p['DPRTYPE'] == 'None') or (['DPRTYPE'] is None): emsg1 = 'Error: {0} is not set in header for file {1}' eargs = [p['KW_DPRTYPE'][0], p['FITSFILENAME']] emsg2 = '\tPlease run pre-processing on file.' emsg3 = ('\tIf pre-processing fails or skips file, file is not ' 'currrently as valid DRS fits file.') WLOG(p, 'error', [emsg1.format(*eargs), emsg2, emsg3]) else: p['DPRTYPE'] = p['DPRTYPE'].strip() # ---------------------------------------------------------------------- # Correction of DARK # ---------------------------------------------------------------------- p, datac = spirouImage.CorrectForDark(p, data, hdr) # ---------------------------------------------------------------------- # Resize image # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to ADU data = spirouImage.ConvertToADU(spirouImage.FlipImage(p, datac), p=p) # convert NaN to zeros data0 = np.where(~np.isfinite(data), np.zeros_like(data), data) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) data1 = spirouImage.ResizeImage(p, data0, **bkwargs) # log change in data size wmsg = 'Image format changed to {1}x{0}' WLOG(p, '', wmsg.format(*data1.shape)) # ---------------------------------------------------------------------- # Correct for the BADPIX mask (set all bad pixels to zero) # ---------------------------------------------------------------------- p, data1 = spirouImage.CorrectForBadPix(p, data1, hdr) # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- # get the number of bad pixels n_bad_pix = np.sum(~np.isfinite(data1)) n_bad_pix_frac = n_bad_pix * 100 / np.product(data1.shape) # Log number wmsg = 'Nb dead pixels = {0} / {1:.4f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ---------------------------------------------------------------------- # Get the miny, maxy and max_signal for the central column # ---------------------------------------------------------------------- # get the central column y = data1[p['IC_CENT_COL'], :] # get the min max and max signal using box smoothed approach miny, maxy, max_signal, diff_maxmin = spirouBACK.MeasureMinMaxSignal(p, y) # Log max average flux/pixel wmsg = 'Maximum average flux/pixel in the spectrum: {0:.1f} [ADU]' WLOG(p, 'info', wmsg.format(max_signal / p['NBFRAMES'])) # ---------------------------------------------------------------------- # Background computation # ---------------------------------------------------------------------- if p['IC_DO_BKGR_SUBTRACTION']: # log that we are doing background measurement WLOG(p, '', 'Doing background measurement on raw frame') # get the bkgr measurement bargs = [p, data1, hdr] # background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs) p, background = spirouBACK.MeasureBackgroundMap(*bargs) else: background = np.zeros_like(data1) p['BKGRDFILE'] = 'None' p.set_source('BKGRDFILE', __NAME__ + '.main()') # apply background correction to data (and set to zero where negative) data1 = data1 - background # ---------------------------------------------------------------------- # Read tilt slit angle # ---------------------------------------------------------------------- # define loc storage parameter dictionary loc = ParamDict() # get tilts (if the mode requires it) if p['IC_EXTRACT_TYPE'] not in EXTRACT_SHAPE_TYPES: p, loc['TILT'] = spirouImage.ReadTiltFile(p, hdr) loc.set_source('TILT', __NAME__ + '/main() + /spirouImage.ReadTiltFile') else: loc['TILT'] = None loc.set_source('TILT', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Earth Velocity calculation # ---------------------------------------------------------------------- if p['IC_IMAGE_TYPE'] == 'H4RG': p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, hdr) # ---------------------------------------------------------------------- # Get all fiber data (for all fibers) # ---------------------------------------------------------------------- # TODO: This is temp solution for options 5a and 5b loc_fibers = spirouLOCOR.GetFiberData(p, hdr) # ------------------------------------------------------------------ # Deal with debananafication # ------------------------------------------------------------------ # if mode 4a or 4b we need to straighten in x only if p['IC_EXTRACT_TYPE'] in ['4a', '4b']: # get the shape parameters p, shapem_x = spirouImage.GetShapeX(p, hdr) p, shape_local = spirouImage.GetShapeLocal(p, hdr) # log progress WLOG(p, '', 'Debananafying (straightening) image') # apply shape transforms targs = dict(lin_transform_vect=shape_local, dxmap=shapem_x) data2 = spirouImage.EATransform(data1, **targs) # if mode 5a or 5b we need to straighten in x and y using the # polynomial fits for location elif p['IC_EXTRACT_TYPE'] in ['5a', '5b']: # get the shape parameters p, shapem_x = spirouImage.GetShapeX(p, hdr) p, shapem_y = spirouImage.GetShapeY(p, hdr) p, shape_local = spirouImage.GetShapeLocal(p, hdr) p, fpmaster = spirouImage.GetFPMaster(p, hdr) # get the bad pixel map bkwargs = dict(return_map=True, quiet=True) p, badpix = spirouImage.CorrectForBadPix(p, data1, hdr, **bkwargs) # log progress WLOG(p, '', 'Cleaning image') # clean the image data1 = spirouEXTOR.CleanHotpix(data1, badpix) # log progress WLOG(p, '', 'Debananafying (straightening) image') # apply shape transforms targs = dict(lin_transform_vect=shape_local, dxmap=shapem_x, dymap=shapem_y) data2 = spirouImage.EATransform(data1, **targs) # in any other mode we do not straighten else: data2 = np.array(data1) # ---------------------------------------------------------------------- # Fiber loop # ---------------------------------------------------------------------- # loop around fiber types for fiber in p['FIB_TYPE']: # set fiber p['FIBER'] = fiber p.set_source('FIBER', __NAME__ + '/main()()') # ------------------------------------------------------------------ # Read wavelength solution # ------------------------------------------------------------------ # set source of wave file wsource = __NAME__ + '/main() + /spirouImage.GetWaveSolution' # Force A and B to AB solution if fiber in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = fiber # get wave image wkwargs = dict(hdr=hdr, return_wavemap=True, return_filename=True, return_header=True, fiber=wave_fiber) wout = spirouImage.GetWaveSolution(p, **wkwargs) loc['WAVEPARAMS'], loc['WAVE'], loc['WAVEFILE'] = wout[:3] loc['WAVEHDR'], loc['WSOURCE'] = wout[3:] source_names = ['WAVE', 'WAVEFILE', 'WAVEPARAMS', 'WAVEHDR'] loc.set_sources(source_names, wsource) # get dates loc['WAVE_ACQTIMES'] = spirouDB.GetTimes(p, loc['WAVEHDR']) loc.set_source('WAVE_ACQTIMES', __NAME__ + '.main()') # get the recipe that produced the wave solution if 'WAVECODE' in loc['WAVEHDR']: loc['WAVE_CODE'] = loc['WAVEHDR']['WAVECODE'] else: loc['WAVE_CODE'] = 'UNKNOWN' loc.set_source('WAVE_CODE', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Get WFP keys # ---------------------------------------------------------------------- # Read the WFP keys - if they don't exist set to None and deal # with later p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_DRIFT', name='WFP_DRIFT', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_FWHM', name='WFP_FWHM', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_CONTRAST', name='WFP_CONTRAST', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_MAXCPP', name='WFP_MAXCPP', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_MASK', name='WFP_MASK', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_LINES', name='WFP_LINES', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_TARG_RV', name='WFP_TARG_RV', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_WIDTH', name='WFP_WIDTH', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_STEP', name='WFP_STEP', required=False) # ---------------------------------------------------------------------- # Read Flat file # ---------------------------------------------------------------------- fout = spirouImage.ReadFlatFile(p, hdr, return_header=True) p, loc['FLAT'], flathdr = fout loc.set_source('FLAT', __NAME__ + '/main() + /spirouImage.ReadFlatFile') # get flat extraction mode if p['KW_E2DS_EXTM'][0] in flathdr: flat_ext_mode = flathdr[p['KW_E2DS_EXTM'][0]] else: flat_ext_mode = None # ------------------------------------------------------------------ # Check extraction method is same as flat extraction method # ------------------------------------------------------------------ # get extraction method and function extmethod, extfunc = spirouEXTOR.GetExtMethod(p, p['IC_EXTRACT_TYPE']) if not DEBUG: # compare flat extraction mode to extraction mode spirouEXTOR.CompareExtMethod(p, flat_ext_mode, extmethod, 'FLAT', 'EXTRACTION') # ------------------------------------------------------------------ # Read Blaze file # ------------------------------------------------------------------ p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hdr) blazesource = __NAME__ + '/main() + /spirouImage.ReadBlazeFile' loc.set_source('BLAZE', blazesource) # ------------------------------------------------------------------ # Get fiber specific parameters from loc_fibers # ------------------------------------------------------------------ # get this fibers parameters p = spirouImage.FiberParams(p, p['FIBER'], merge=True) # get localisation parameters for key in loc_fibers[fiber]: loc[key] = loc_fibers[fiber][key] loc.set_source(key, loc_fibers[fiber].sources[key]) # get locofile source p['LOCOFILE'] = loc['LOCOFILE'] p.set_source('LOCOFILE', loc.sources['LOCOFILE']) # get the order_profile order_profile = loc_fibers[fiber]['ORDER_PROFILE'] # ------------------------------------------------------------------ # Set up Extract storage # ------------------------------------------------------------------ # Create array to store extraction (for each order and each pixel # along order) loc['E2DS'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) loc['E2DSFF'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) loc['E2DSLL'] = [] loc['SPE1'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) loc['SPE3'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) loc['SPE4'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) loc['SPE5'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) # Create array to store the signal to noise ratios for each order loc['SNR'] = np.zeros(loc['NUMBER_ORDERS']) # ------------------------------------------------------------------ # Extract orders # ------------------------------------------------------------------ # source for parameter dictionary source = __NAME__ + '/main()' # get limits of order extraction valid_orders = spirouEXTOR.GetValidOrders(p, loc) # loop around each order for order_num in valid_orders: # ------------------------------------------------------------- # IC_EXTRACT_TYPE decides the extraction routine # ------------------------------------------------------------- eargs = [p, loc, data2, order_num] ekwargs = dict(mode=p['IC_EXTRACT_TYPE'], order_profile=order_profile) with warnings.catch_warnings(record=True) as w: eout = spirouEXTOR.Extraction(*eargs, **ekwargs) # deal with different return if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES: e2ds, e2dsll, cpt = eout else: e2ds, cpt = eout e2dsll = None # ------------------------------------------------------------- # calculate the noise range1, range2 = p['IC_EXT_RANGE1'], p['IC_EXT_RANGE2'] # set the noise noise = p['SIGDET'] * np.sqrt(range1 + range2) # get window size blaze_win1 = int(data2.shape[0] / 2) - p['IC_EXTFBLAZ'] blaze_win2 = int(data2.shape[0] / 2) + p['IC_EXTFBLAZ'] # get average flux per pixel flux = np.nansum( e2ds[blaze_win1:blaze_win2]) / (2 * p['IC_EXTFBLAZ']) # calculate signal to noise ratio = flux/sqrt(flux + noise^2) snr = flux / np.sqrt(flux + noise**2) # log the SNR RMS wmsg = 'On fiber {0} order {1}: S/N= {2:.1f} Nbcosmic= {3}' wargs = [p['FIBER'], order_num, snr, cpt] WLOG(p, '', wmsg.format(*wargs)) # add calculations to storage loc['E2DS'][order_num] = e2ds loc['E2DSFF'][order_num] = e2ds / loc['FLAT'][order_num] loc['SNR'][order_num] = snr # save the longfile if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES: loc['E2DSLL'].append(e2dsll) # set sources loc.set_sources(['e2ds', 'SNR'], source) # Log if saturation level reached satvalue = (flux / p['GAIN']) / (range1 + range2) if satvalue > (p['QC_LOC_FLUMAX'] * p['NBFRAMES']): wmsg = 'SATURATION LEVEL REACHED on Fiber {0} order={1}' WLOG(p, 'warning', wmsg.format(fiber, order_num)) # ------------------------------------------------------------------ # Thermal correction # ------------------------------------------------------------------ # get fiber type if fiber in ['AB', 'A', 'B']: fibertype = p['DPRTYPE'].split('_')[0] else: fibertype = p['DPRTYPE'].split('_')[1] # apply thermal correction based on fiber type if fibertype in p['THERMAL_CORRECTION_TYPE1']: # log progress wmsg = 'Correcting thermal background for {0}={1} mode={2}' wargs = [fiber, fibertype, 1] WLOG(p, 'info', wmsg.format(*wargs)) # correct E2DS tkwargs = dict(image=loc['E2DS'], mode=1, fiber=fiber, hdr=hdr) p, loc['E2DS'] = spirouBACK.ThermalCorrect(p, **tkwargs) # correct E2DSFF tkwargs = dict(image=loc['E2DSFF'], mode=1, fiber=fiber, hdr=hdr, flat=loc['FLAT']) p, loc['E2DSFF'] = spirouBACK.ThermalCorrect(p, **tkwargs) elif fibertype in p['THERMAL_CORRECTION_TYPE2']: # log progress wmsg = 'Correcting thermal background for {0}={1} mode={2}' wargs = [fiber, fibertype, 2] WLOG(p, 'info', wmsg.format(*wargs)) # correct E2DS tkwargs = dict(image=loc['E2DS'], mode=2, fiber=fiber, hdr=hdr) p, loc['E2DS'] = spirouBACK.ThermalCorrect(p, **tkwargs) # correct E2DSFF tkwargs = dict(image=loc['E2DSFF'], mode=2, fiber=fiber, hdr=hdr, flat=loc['FLAT']) p, loc['E2DSFF'] = spirouBACK.ThermalCorrect(p, **tkwargs) else: # log progress wmsg = 'Not correcting thermal background for {0}={1}' wargs = [fiber, fibertype] WLOG(p, 'info', wmsg.format(*wargs)) # set filename for output outfile = 'THERMALFILE_{0}'.format(fiber) p[outfile] = 'None' p.set_source(outfile, __NAME__ + '.main()') # ------------------------------------------------------------------ # Plots # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot all orders or one order if p['IC_FF_PLOT_ALL_ORDERS']: # plot image with all order fits (slower) sPlt.ext_aorder_fit(p, loc, data1, max_signal / 10.) else: # plot image with selected order fit and edge fit (faster) sPlt.ext_sorder_fit(p, loc, data1, max_signal / 10.) # plot e2ds against wavelength sPlt.ext_spectral_order_plot(p, loc) if p['IC_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES: sPlt.ext_debanana_plot(p, loc, data2, max_signal / 10.) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # if array is completely NaNs it shouldn't pass if np.sum(np.isfinite(loc['E2DS'])) == 0: fail_msg.append('E2DS image is all NaNs') passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append('NaN') qc_names.append('image') qc_logic.append('image is all NaN') # ---------------------------------------------------------------------- # saturation check: check that the max_signal is lower than # qc_max_signal max_qcflux = p['QC_MAX_SIGNAL'] * p['NBFRAMES'] if max_signal > max_qcflux: fmsg = 'Too much flux in the image ({0:.2f} > {1:.2f})' fail_msg.append(fmsg.format(max_signal, max_qcflux)) passed = False # Question: Why is this test ignored? # For some reason this test is ignored in old code passed = True WLOG(p, 'info', fail_msg[-1]) qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(max_signal) qc_names.append('max_signal') qc_logic.append('QC_MAX_SIGNAL > {0:.3f}'.format(max_qcflux)) # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Store extraction in file(s) # ------------------------------------------------------------------ raw_ext_file = os.path.basename(p['FITSFILENAME']) # construct filename e2dsfits, tag1 = spirouConfig.Constants.EXTRACT_E2DS_FILE(p) e2dsfitsname = os.path.split(e2dsfits)[-1] e2dsfffits, tag2 = spirouConfig.Constants.EXTRACT_E2DSFF_FILE(p) e2dsfffitsname = os.path.split(e2dsfffits)[-1] e2dsllfits, tag4 = spirouConfig.Constants.EXTRACT_E2DSLL_FILE(p) e2dsfllitsname = os.path.split(e2dsllfits)[-1] # log that we are saving E2DS spectrum wmsg = 'Saving E2DS spectrum of Fiber {0} in {1}' WLOG(p, '', wmsg.format(p['FIBER'], e2dsfitsname)) wmsg = 'Saving E2DSFF spectrum of Fiber {0} in {1}' WLOG(p, '', wmsg.format(p['FIBER'], e2dsfffitsname)) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_FIBER'], value=p['FIBER']) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBACK'], value=p['BKGRDFILE']) if p['IC_EXTRACT_TYPE'] not in EXTRACT_SHAPE_TYPES: hdict = spirouImage.AddKey(p, hdict, p['KW_CDBTILT'], value=p['TILTFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFLAT'], value=p['FLATFILE']) if p['IC_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES: hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEX'], value=p['SHAPEXFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEY'], value=p['SHAPEYFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPE'], value=p['SHAPEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFPMASTER'], value=p['FPMASTERFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBTHERMAL'], value=p['THERMALFILE_{0}'.format(fiber)]) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # construct loco filename locofile, _ = spirouConfig.Constants.EXTRACT_LOCO_FILE(p) locofilename = os.path.basename(locofile) # add barycentric keys to header hdict = spirouImage.AddKey(p, hdict, p['KW_BERV'], value=loc['BERV']) hdict = spirouImage.AddKey(p, hdict, p['KW_BJD'], value=loc['BJD']) hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_MAX'], value=loc['BERV_MAX']) hdict = spirouImage.AddKey(p, hdict, p['KW_B_OBS_HOUR'], value=loc['BERVHOUR']) # add barycentric estimate keys to header hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_EST'], value=loc['BERV_EST']) hdict = spirouImage.AddKey(p, hdict, p['KW_BJD_EST'], value=loc['BJD_EST']) hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_MAX_EST'], value=loc['BERV_MAX_EST']) hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_SOURCE'], value=loc['BERV_SOURCE']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # copy extraction method and function to header # (for reproducibility) hdict = spirouImage.AddKey(p, hdict, p['KW_E2DS_EXTM'], value=extmethod) hdict = spirouImage.AddKey(p, hdict, p['KW_E2DS_FUNC'], value=extfunc) # add localization file name to header hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_FILE'], value=locofilename) # add wave solution date hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME1'], value=loc['WAVE_ACQTIMES'][0]) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME2'], value=loc['WAVE_ACQTIMES'][1]) # add wave solution number of orders hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_ORD_N'], value=loc['WAVEPARAMS'].shape[0]) # add wave solution degree of fit hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_LL_DEG'], value=loc['WAVEPARAMS'].shape[1] - 1) # ------------------------------------------------------------------------- # add keys of the wave solution FP CCF hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_FILE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_DRIFT'], value=p['WFP_DRIFT']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_FWHM'], value=p['WFP_FWHM']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_CONTRAST'], value=p['WFP_CONTRAST']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_MAXCPP'], value=p['WFP_MAXCPP']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_MASK'], value=p['WFP_MASK']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_LINES'], value=p['WFP_LINES']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_TARG_RV'], value=p['WFP_TARG_RV']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_WIDTH'], value=p['WFP_WIDTH']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_STEP'], value=p['WFP_STEP']) # write 1D list of the SNR hdict = spirouImage.AddKey1DList(p, hdict, p['KW_E2DS_SNR'], values=loc['SNR']) # add localization file keys to header root = p['KW_ROOT_DRS_LOC'][0] hdict = spirouImage.CopyRootKeys(p, hdict, locofile, root=root) # add wave solution coefficients hdict = spirouImage.AddKey2DList(p, hdict, p['KW_WAVE_PARAM'], values=loc['WAVEPARAMS']) # Save E2DS file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) p = spirouImage.WriteImage(p, e2dsfits, loc['E2DS'], hdict) # Save E2DSFF file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) p = spirouImage.WriteImage(p, e2dsfffits, loc['E2DSFF'], hdict) # Save E2DSLL file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES: llstack = np.vstack(loc['E2DSLL']) p = spirouImage.WriteImage(p, e2dsllfits, llstack, hdict) # ------------------------------------------------------------------ # 1-dimension spectral S1D (uniform in wavelength) # ------------------------------------------------------------------ # get arguments for E2DS to S1D e2dsargs = [loc['WAVE'], loc['E2DSFF'], loc['BLAZE']] # get 1D spectrum xs1d1, ys1d1 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='wave') # Plot the 1D spectrum if p['DRS_PLOT'] > 0: sPlt.ext_1d_spectrum_plot(p, xs1d1, ys1d1) # construct file name s1dfile1, tag3 = spirouConfig.Constants.EXTRACT_S1D_FILE1(p) s1dfilename1 = os.path.basename(s1dfile1) # add header keys # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) # log writing to file wmsg = 'Saving 1D spectrum (uniform in wavelength) for Fiber {0} in {1}' WLOG(p, '', wmsg.format(p['FIBER'], s1dfilename1)) # Write to file columns = ['wavelength', 'flux', 'eflux'] values = [xs1d1, ys1d1, np.zeros_like(ys1d1)] units = ['nm', None, None] s1d1 = spirouImage.MakeTable(p, columns, values, units=units) spirouImage.WriteTable(p, s1d1, s1dfile1, header=hdict) # ------------------------------------------------------------------ # 1-dimension spectral S1D (uniform in velocity) # ------------------------------------------------------------------ # get arguments for E2DS to S1D e2dsargs = [loc['WAVE'], loc['E2DSFF'], loc['BLAZE']] # get 1D spectrum xs1d2, ys1d2 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='velocity') # Plot the 1D spectrum if p['DRS_PLOT'] > 0: sPlt.ext_1d_spectrum_plot(p, xs1d2, ys1d2) # construct file name s1dfile2, tag4 = spirouConfig.Constants.EXTRACT_S1D_FILE2(p) s1dfilename2 = os.path.basename(s1dfile2) # add header keys hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) # log writing to file wmsg = 'Saving 1D spectrum (uniform in velocity) for Fiber {0} in {1}' WLOG(p, '', wmsg.format(p['FIBER'], s1dfilename2)) # Write to file columns = ['wavelength', 'flux', 'eflux'] values = [xs1d2, ys1d2, np.zeros_like(ys1d2)] units = ['nm', None, None] s1d2 = spirouImage.MakeTable(p, columns, values, units=units) spirouImage.WriteTable(p, s1d2, s1dfile2, header=hdict) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, reffile=None): """ cal_DRIFT_E2DS_spirou.py main function, if arguments are None uses arguments from run time i.e.: cal_DRIFT_E2DS_spirou.py [night_directory] [reffile] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param reffile: string, the reference file to use :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with reference file being None (i.e. get from sys.argv) if reffile is None: customargs = spirouStartup.GetCustomFromRuntime( p, [0], [str], ['reffile']) else: customargs = dict(reffile=reffile) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='reffile', mainfitsdir='reduced') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, reffilename = spirouStartup.SingleFileSetup(p, filename=p['REFFILE']) p['REFFILENAME'] = reffilename p.set_source('REFFILENAME', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data speref, hdr, nbo, nx = spirouImage.ReadData(p, reffilename) # add to loc loc = ParamDict() loc['SPEREF'] = speref loc['NUMBER_ORDERS'] = nbo loc.set_sources(['speref', 'number_orders'], __NAME__ + '/main()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # get acquisition time p = spirouImage.GetAcqTime(p, hdr, name='acqtime', kind='julian') bjdref = p['ACQTIME'] # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # manually set OBJNAME to FP p['OBJNAME'] = 'FP' # ---------------------------------------------------------------------- # Earth Velocity calculation # ---------------------------------------------------------------------- if p['IC_IMAGE_TYPE'] == 'H4RG': p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, hdr) # ---------------------------------------------------------------------- # Read wavelength solution # ---------------------------------------------------------------------- # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # # get wave image # wout = spirouImage.GetWaveSolution(p, hdr=hdr, fiber=wave_fiber, # return_wavemap=True) # _, loc['WAVE'] = wout # loc.set_source('WAVE', __NAME__+'/main() + /spirouImage.GetWaveSolution') # get wave image wout = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True, return_filename=True, fiber=wave_fiber) param_ll, wave_ll, wavefile, wsource = wout # save to storage loc['PARAM_LL'], loc['WAVE_LL'], loc['WAVEFILE'], loc['WSOURCE'] = wout source = __NAME__ + '/main() + spirouTHORCA.GetWaveSolution()' loc.set_sources(['WAVE_LL', 'PARAM_LL', 'WAVEFILE', 'WSOURCE'], source) # ---------------------------------------------------------------------- # Read Flat file # ---------------------------------------------------------------------- # get flat p, loc['FLAT'] = spirouImage.ReadFlatFile(p, hdr) loc.set_source('FLAT', __NAME__ + '/main() + /spirouImage.ReadFlatFile') # get all values in flat that are zero to 1 loc['FLAT'] = np.where(loc['FLAT'] == 0, 1.0, loc['FLAT']) # ---------------------------------------------------------------------- # Background correction # ---------------------------------------------------------------------- # log that we are performing background correction if p['IC_DRIFT_BACK_CORR']: WLOG(p, '', 'Perform background correction') # get the box size from constants bsize = p['DRIFT_PEAK_MINMAX_BOXSIZE'] # Loop around the orders for order_num in range(loc['NUMBER_ORDERS']): miny, maxy = spirouBACK.MeasureMinMax(loc['SPEREF'][order_num], bsize) loc['SPEREF'][order_num] = loc['SPEREF'][order_num] - miny # ---------------------------------------------------------------------- # Preliminary set up = no flat, no blaze # ---------------------------------------------------------------------- # reset flat to all ones # loc['FLAT'] = np.ones((nbo, nx)) # set blaze to all ones (if not bug in correlbin !!! # TODO Check why Blaze makes bugs in correlbin loc['BLAZE'] = np.ones((nbo, nx)) # set sources # loc.set_sources(['flat', 'blaze'], __NAME__ + '/main()') loc.set_sources(['blaze'], __NAME__ + '/main()') # ------------------------------------------------------------------ # Compute photon noise uncertainty for reference file # ------------------------------------------------------------------ # set up the arguments for DeltaVrms2D dargs = [loc['SPEREF'], loc['WAVE_LL']] dkwargs = dict(sigdet=p['IC_DRIFT_NOISE'], size=p['IC_DRIFT_BOXSIZE'], threshold=p['IC_DRIFT_MAXFLUX']) # run DeltaVrms2D dvrmsref, wmeanref = spirouRV.DeltaVrms2D(*dargs, **dkwargs) # save to loc loc['DVRMSREF'], loc['WMEANREF'] = dvrmsref, wmeanref loc.set_sources(['dvrmsref', 'wmeanref'], __NAME__ + '/main()()') # log the estimated RV uncertainty wmsg = 'On fiber {0} estimated RV uncertainty on spectrum is {1:.3f} m/s' WLOG(p, 'info', wmsg.format(p['FIBER'], wmeanref)) # ------------------------------------------------------------------ # Reference plots # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot FP spectral order # sPlt.drift_plot_selected_wave_ref(p, loc) # plot photon noise uncertainty sPlt.drift_plot_photon_uncertainty(p, loc) # ---------------------------------------------------------------------- # Get template RV (from ccf_mask) # ---------------------------------------------------------------------- # Use CCF Mask function with drift constants p['CCF_MASK'] = p['DRIFT_CCF_MASK'] p['TARGET_RV'] = p['DRIFT_TARGET_RV'] p['CCF_WIDTH'] = p['DRIFT_CCF_WIDTH'] p['CCF_STEP'] = p['DRIFT_CCF_STEP'] # get the CCF mask from file (check location of mask) loc = spirouRV.GetCCFMask(p, loc) # check and deal with mask in microns (should be in nm) if np.mean(loc['LL_MASK_CTR']) < 2.0: loc['LL_MASK_CTR'] *= 1000.0 loc['LL_MASK_D'] *= 1000.0 # ---------------------------------------------------------------------- # Do correlation # ---------------------------------------------------------------------- # calculate and fit the CCF loc['E2DSFF'] = np.array(loc['SPEREF']) loc.set_source('E2DSFF', __NAME__ + '/main()') p['CCF_FIT_TYPE'] = 1 # run the RV coravelation function with these parameters loc = spirouRV.Coravelation(p, loc) # ---------------------------------------------------------------------- # Correlation stats # ---------------------------------------------------------------------- # get the maximum number of orders to use nbmax = p['CCF_NUM_ORDERS_MAX'] # get the average ccf loc['AVERAGE_CCF'] = np.nansum(loc['CCF'][:nbmax], axis=0) # normalize the average ccf normalized_ccf = loc['AVERAGE_CCF'] / np.nanmax(loc['AVERAGE_CCF']) # get the fit for the normalized average ccf ccf_res, ccf_fit = spirouRV.FitCCF(p, loc['RV_CCF'], normalized_ccf, fit_type=1) loc['CCF_RES'] = ccf_res loc['CCF_FIT'] = ccf_fit # get the max cpp loc['MAXCPP'] = np.nansum(loc['CCF_MAX']) / np.nansum( loc['PIX_PASSED_ALL']) # get the RV value from the normalised average ccf fit center location loc['RV'] = float(ccf_res[1]) # get the contrast (ccf fit amplitude) loc['CONTRAST'] = np.abs(100 * ccf_res[0]) # get the FWHM value loc['FWHM'] = ccf_res[2] * spirouCore.spirouMath.fwhm() # ---------------------------------------------------------------------- # set the source keys = [ 'average_ccf', 'maxcpp', 'rv', 'contrast', 'fwhm', 'ccf_res', 'ccf_fit' ] loc.set_sources(keys, __NAME__ + '/main()') # ---------------------------------------------------------------------- # log the stats wmsg = ('Correlation: C={0:.1f}[%] RV={1:.5f}[km/s] ' 'FWHM={2:.4f}[km/s] maxcpp={3:.1f}') wargs = [loc['CONTRAST'], loc['RV'], loc['FWHM'], loc['MAXCPP']] WLOG(p, 'info', wmsg.format(*wargs)) # get the reference RV in m/s rvref = loc['RV'] * 1000. # ---------------------------------------------------------------------- # rv ccf plot # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # Plot rv vs ccf (and rv vs ccf_fit) sPlt.ccf_rv_ccf_plot(p, loc['RV_CCF'], normalized_ccf, ccf_fit) # ------------------------------------------------------------------ # Get all other files that match kw_OUTPUT and kw_EXT_TYPE from # ref file # ------------------------------------------------------------------ # get files listfiles, listtypes = spirouImage.GetSimilarDriftFiles(p, hdr) # get the number of files nfiles = len(listfiles) # Log the number of files found wmsgs = [ 'Number of files found on directory = {0}'.format(nfiles), '\tExtensions allowed:' ] for listtype in listtypes: wmsgs.append('\t\t - {0}'.format(listtype)) WLOG(p, 'info', wmsgs) # ------------------------------------------------------------------ # Set up Extract storage for all files # ------------------------------------------------------------------ # decide whether we need to skip (for large number of files) if len(listfiles) >= p['DRIFT_NLARGE']: skip = p['DRIFT_E2DS_FILE_SKIP'] nfiles = int(nfiles / skip) else: skip = 1 # set up storage loc['MDRIFT'] = np.zeros(nfiles) loc['MERRDRIFT'] = np.zeros(nfiles) loc['DELTATIME'] = np.zeros(nfiles) loc['FLUXRATIO'] = np.zeros(nfiles) # set loc sources keys = ['mdrift', 'merrdrift', 'deltatime'] loc.set_sources(keys, __NAME__ + '/main()()') # ------------------------------------------------------------------ # Loop around all files: correct for dark, reshape, extract and # calculate dvrms and meanpond # ------------------------------------------------------------------ wref = 1 for i_it in range(nfiles): # get file for this iteration fpfile = listfiles[::skip][i_it] # Log the file we are reading wmsg = 'Reading file {0}' WLOG(p, '', wmsg.format(os.path.split(fpfile)[-1])) # ------------------------------------------------------------------ # read e2ds files and get timestamp # ------------------------------------------------------------------ # read data rout = spirouImage.ReadData(p, filename=fpfile, log=False) loc['SPE'], hdri, nxi, nyi = rout # get acqtime bjdspe = spirouImage.GetAcqTime(p, hdri, name='acqtime', return_value=1, kind='julian') # test whether we want to subtract background if p['IC_DRIFT_BACK_CORR']: # Loop around the orders for order_num in range(loc['NUMBER_ORDERS']): # get the box size from constants bsize = p['DRIFT_PEAK_MINMAX_BOXSIZE'] # Measurethe min and max flux miny, maxy = spirouBACK.MeasureMinMax(loc['SPE'][order_num], bsize) # subtract off the background (miny) loc['SPE'][order_num] = loc['SPE'][order_num] - miny # ------------------------------------------------------------------ # calculate flux ratio # ------------------------------------------------------------------ sorder = p['IC_DRIFT_ORDER_PLOT'] fratio = np.nansum(loc['SPE'][sorder]) / np.nansum( loc['SPEREF'][sorder]) loc['FLUXRATIO'][i_it] = fratio # ------------------------------------------------------------------ # Compute photon noise uncertainty for reference file # ------------------------------------------------------------------ # set up the arguments for DeltaVrms2D dargs = [loc['SPE'], loc['WAVE_LL']] dkwargs = dict(sigdet=p['IC_DRIFT_NOISE'], size=p['IC_DRIFT_BOXSIZE'], threshold=p['IC_DRIFT_MAXFLUX']) # run DeltaVrms2D dvrmsspe, wmeanspe = spirouRV.DeltaVrms2D(*dargs, **dkwargs) # ---------------------------------------------------------------------- # Do correlation # ---------------------------------------------------------------------- # calculate and fit the CCF loc['E2DSFF'] = loc['SPE'] * 1. loc.set_source('E2DSFF', __NAME__ + '/main()') loc = spirouRV.Coravelation(p, loc) # ---------------------------------------------------------------------- # Correlation stats # ---------------------------------------------------------------------- # get the maximum number of orders to use nbmax = p['CCF_NUM_ORDERS_MAX'] # get the average ccf loc['AVERAGE_CCF'] = np.nansum(loc['CCF'][:nbmax], axis=0) # normalize the average ccf normalized_ccf = loc['AVERAGE_CCF'] / np.nanmax(loc['AVERAGE_CCF']) # get the fit for the normalized average ccf ccf_res, ccf_fit = spirouRV.FitCCF(p, loc['RV_CCF'], normalized_ccf, fit_type=1) # calculate the mean RV meanrv = ccf_res[1] * 1000. - rvref # ------------------------------------------------------------------ # Calculate delta time # ------------------------------------------------------------------ # calculate the time from reference (in hours) deltatime = (bjdspe - bjdref) * 24 err_meanrv = np.sqrt(dvrmsref + dvrmsspe) merr = 1. / np.sqrt(np.nansum((1. / err_meanrv)**2)) # Log the RV properties wmsg = ('Time from ref= {0:.2f} h ' '- Flux Ratio= {1:.2f} ' '- Drift mean= {2:.2f} +- ' '{3:.2f} m/s') wargs = [deltatime, loc['FLUXRATIO'][i_it], meanrv, merr] WLOG(p, '', wmsg.format(*wargs)) # add this iteration to storage loc['MDRIFT'][i_it] = meanrv loc['MERRDRIFT'][i_it] = merr loc['DELTATIME'][i_it] = deltatime # ------------------------------------------------------------------ # set source loc.set_sources(['mdrift', 'merrdrift'], __NAME__ + '/main()()') # ------------------------------------------------------------------ # peak to peak drift driftptp = np.max(loc['MDRIFT']) - np.min(loc['MDRIFT']) driftrms = np.std(loc['MDRIFT']) # log th etotal drift peak-to-peak and rms wmsg = ('Total drift Peak-to-Peak={0:.3f} m/s RMS={1:.3f} m/s in ' '{2:.2f} hour') wargs = [driftptp, driftrms, np.max(loc['DELTATIME'])] WLOG(p, '', wmsg.format(*wargs)) # ------------------------------------------------------------------ # Plot of mean drift # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot delta time against median drift sPlt.drift_plot_dtime_against_mdrift(p, loc, kind='e2ds') # ------------------------------------------------------------------ # Save drift values to file # ------------------------------------------------------------------ # # get raw input file name # raw_infile = os.path.basename(p['REFFILE']) # # construct filename # driftfits, tag = spirouConfig.Constants.DRIFTCCF_E2DS_FITS_FILE(p) # driftfitsname = os.path.split(driftfits)[-1] # # log that we are saving drift values # wmsg = 'Saving drift values of Fiber {0} in {1}' # WLOG(p, '', wmsg.format(p['FIBER'], driftfitsname)) # # add keys from original header file # hdict = spirouImage.CopyOriginalKeys(hdr) # # add the reference RV # hdict = spirouImage.AddKey(p, hdict, p['KW_REF_RV'], value=rvref) # # # set the version # hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) # hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # # set the input files # hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFLAT'], value=p['FLATFILE']) # hdict = spirouImage.AddKey(p, hdict, p['KW_REFFILE'], value=raw_infile) # # save drift values # p = spirouImage.WriteImage(p, driftfits, loc['DRIFT'], hdict) # ------------------------------------------------------------------ # print .tbl result # ------------------------------------------------------------------ # construct filename drifttbl = spirouConfig.Constants.DRIFTCCF_E2DS_TBL_FILE(p) drifttblname = os.path.split(drifttbl)[-1] # construct and write table columnnames = ['time', 'drift', 'drifterr'] columnformats = ['7.4f', '6.2f', '6.3f'] columnvalues = [loc['DELTATIME'], loc['MDRIFT'], loc['MERRDRIFT']] table = spirouImage.MakeTable(p, columns=columnnames, values=columnvalues, formats=columnformats) # write table wmsg = 'Average Drift saved in {0} Saved ' WLOG(p, '', wmsg.format(drifttblname)) spirouImage.WriteTable(p, table, drifttbl, fmt='ascii.rst') # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files, mainfitsdir='reduced') p = spirouStartup.InitialFileSetup(p, calibdb=True) # set up function name main_name = __NAME__ + '.main()' # ------------------------------------------------------------------ # Load first file # ------------------------------------------------------------------ loc = ParamDict() rd = spirouImage.ReadImage(p, p['FITSFILENAME']) loc['DATA'], loc['DATAHDR'], loc['YDIM'], loc['XDIM'] = rd loc.set_sources(['DATA', 'DATAHDR', 'XDIM', 'YDIM'], main_name) # ------------------------------------------------------------------ # Get the wave solution # ------------------------------------------------------------------ masterwavefile = spirouDB.GetDatabaseMasterWave(p) # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # read master wave map wout = spirouImage.GetWaveSolution(p, filename=masterwavefile, return_wavemap=True, quiet=True, return_header=True, fiber=wave_fiber) _, loc['WAVE'], loc['WAVEFILE'], _ = wout loc.set_sources(['WAVE', 'WAVEFILE'], main_name) # get the wave keys loc = spirouImage.GetWaveKeys(p, loc, loc['DATAHDR']) # ------------------------------------------------------------------ # Construct convolution kernels (used in GetMolecularTellLines) # ------------------------------------------------------------------ loc = spirouTelluric.ConstructConvKernel1(p, loc) # ------------------------------------------------------------------ # Get molecular telluric lines # ------------------------------------------------------------------ loc = spirouTelluric.GetMolecularTellLines(p, loc) # if TAPAS FNAME is not None we generated a new file so should add to tellDB if loc['TAPAS_FNAME'] is not None: # add to the telluric database spirouDB.UpdateDatabaseTellConv(p, loc['TAPAS_FNAME'], loc['DATAHDR']) # put file in telluDB spirouDB.PutTelluFile(p, loc['TAPAS_ABSNAME']) # ---------------------------------------------------------------------- # load the expected atmospheric transmission # ---------------------------------------------------------------------- # read filename from telluDB tapas_file_names = spirouDB.GetDatabaseTellConv(p) tapas_file_name = tapas_file_names[-1] # load atmospheric transmission sp_tapas = np.load(tapas_file_name) loc['TAPAS_ALL_SPECIES'] = sp_tapas # extract the water and other line-of-sight optical depths loc['TAPAS_WATER'] = sp_tapas[1, :] loc['TAPAS_OTHERS'] = np.prod(sp_tapas[2:, :], axis=0) loc.set_sources(['TAPAS_ALL_SPECIES', 'TAPAS_WATER', 'TAPAS_OTHERS'], main_name) # ------------------------------------------------------------------ # Get master wave solution map # ------------------------------------------------------------------ # get master wave map masterwavefile = spirouDB.GetDatabaseMasterWave(p) # log process wmsg1 = 'Shifting transmission map on to master wavelength grid' wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile)) WLOG(p, '', [wmsg1, wmsg2]) # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # read master wave map mout = spirouImage.GetWaveSolution(p, filename=masterwavefile, return_wavemap=True, quiet=True, return_header=True, fiber=wave_fiber) masterwavep, masterwave, masterwaveheader, mwsource = mout # get wave acqtimes master_acqtimes = spirouDB.GetTimes(p, masterwaveheader) # ------------------------------------------------------------------ # Loop around the files # ------------------------------------------------------------------ # construct extension tellu_ext = '{0}_{1}.fits' # get current telluric maps from telluDB # tellu_db_data = spirouDB.GetDatabaseTellMap(p, required=False) # tellu_db_files = tellu_db_data[0] # storage for valid output files loc['OUTPUTFILES'] = [] # loop around the files for basefilename in p['ARG_FILE_NAMES']: # ------------------------------------------------------------------ # Get absolute path of filename # ------------------------------------------------------------------ filename = os.path.join(p['ARG_FILE_DIR'], basefilename) # ------------------------------------------------------------------ # Read obj telluric file and correct blaze (per order) # ------------------------------------------------------------------ # get image sp, shdr, _, _ = spirouImage.ReadImage(p, filename) # ------------------------------------------------------------------ # check that file has valid DPRTYPE # ------------------------------------------------------------------ # get FP_FP DPRTYPE p = spirouImage.ReadParam(p, shdr, 'KW_DPRTYPE', 'DPRTYPE', dtype=str) # if dprtype is incorrect skip if p['DPRTYPE'] not in p['ALLOWED_TELLURIC_DPRTYPES']: wmsg1 = 'Skipping file (DPRTYPE incorrect)' wmsg2 = '\t DPRTYPE = {0}'.format(p['DPRTYPE']) WLOG(p, 'warning', [wmsg1, wmsg2]) continue # get blaze p, blaze = spirouImage.ReadBlazeFile(p, shdr) # get the blaze percentile blaze_p = p['MKTELLU_BLAZE_PERCENTILE'] # loop through blaze orders, normalize blaze by its peak amplitude for order_num in range(sp.shape[0]): # normalize the spectrum spo, bzo = sp[order_num], blaze[order_num] sp[order_num] = spo / np.nanpercentile(spo, blaze_p) # normalize the blaze blaze[order_num] = bzo / np.nanpercentile(bzo, blaze_p) # find where the blaze is bad with warnings.catch_warnings(record=True) as _: badblaze = blaze < p['MKTELLU_CUT_BLAZE_NORM'] # set bad blaze to NaN blaze[badblaze] = np.nan # set to NaN values where spectrum is zero zeromask = sp == 0 sp[zeromask] = np.nan # divide spectrum by blaze with warnings.catch_warnings(record=True) as _: sp = sp / blaze # add sp to loc loc['SP'] = sp loc.set_source('SP', main_name) # ---------------------------------------------------------------------- # Get object name, airmass and berv # ---------------------------------------------------------------------- # Get object name loc['OBJNAME'] = spirouImage.GetObjName(p, shdr) # Get the airmass loc['AIRMASS'] = spirouImage.GetAirmass(p, shdr) # Get the Barycentric correction from header p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, shdr) # set sources source = main_name + '+ spirouImage.ReadParams()' loc.set_sources(['OBJNAME', 'AIRMASS'], source) # ------------------------------------------------------------------ # get output transmission filename outfile, tag1 = spirouConfig.Constants.TELLU_TRANS_MAP_FILE( p, filename) outfilename = os.path.basename(outfile) loc['OUTPUTFILES'].append(outfile) # ---------------------------------------------------------------------- # Load template (if available) # ---------------------------------------------------------------------- # read filename from telluDB template_file = spirouDB.GetDatabaseObjTemp(p, loc['OBJNAME'], required=False) # if we don't have a template flag it if template_file is None: loc['FLAG_TEMPLATE'] = False loc['TEMPLATE'] = None # construct progres string pstring = 'No template found.' else: loc['FLAG_TEMPLATE'] = True # load template template, _, _, _ = spirouImage.ReadImage(p, template_file) # add to loc loc['TEMPLATE'] = template # construct progres string template_bfile = os.path.basename(template_file) pstring = 'Using template {0}'.format(template_bfile) # set the source for flag and template loc.set_sources(['FLAG_TEMPLATE', 'TEMPLATE'], main_name) # ------------------------------------------------------------------ # log processing file wmsg = 'Processing file {0}. {1}' WLOG(p, '', [wmsg.format(outfilename, pstring)]) # ------------------------------------------------------------------ # Check that basefile is not in blacklist # ------------------------------------------------------------------ blacklist_check = spirouTelluric.CheckBlackList(loc['OBJNAME']) if blacklist_check: # log black list file found wmsg = 'File {0} is blacklisted (OBJNAME={1}). Skipping' wargs = [basefilename, loc['OBJNAME']] WLOG(p, 'warning', wmsg.format(*wargs)) # skip this file continue # ------------------------------------------------------------------ # deal with applying template to spectrum # ------------------------------------------------------------------ # Requires from loc: # TEMPLATE (None or template loaded from file) # FLAG_TEMPLATE # WAVE # SP # BERV # # Returns: # SP (modified if template was used) # TEMPLATE # WCONV loc = spirouTelluric.ApplyTemplate(p, loc) # ------------------------------------------------------------------ # calcullate telluric absorption (with a sigma clip loop) # ------------------------------------------------------------------ # Requires from loc: # AIRMASS # WAVE # SP # WCONV # Returns: # PASSED [Bool] True or False # SP_OUT # SED_OUT # RECOV_AIRMASS # RECOV_WATER loc = spirouTelluric.CalcTelluAbsorption(p, loc) # calculate tranmission map from sp and sed transmission_map = loc['SP_OUT'] / loc['SED_OUT'] # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # if array is completely NaNs it shouldn't pass if np.sum(np.isfinite(transmission_map)) == 0: fail_msg.append('transmission map is all NaNs') passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append('NaN') qc_names.append('image') qc_logic.append('image is all NaN') # ---------------------------------------------------------------------- # get SNR for each order from header nbo = loc['DATA'].shape[0] snr_order = p['QC_MK_TELLU_SNR_ORDER'] snr = spirouImage.Read1Dkey(p, shdr, p['kw_E2DS_SNR'][0], nbo) # check that SNR is high enough if snr[snr_order] < p['QC_MK_TELLU_SNR_MIN']: fmsg = 'low SNR in order {0}: ({1:.2f} < {2:.2f})' fargs = [snr_order, snr[snr_order], p['QC_MK_TELLU_SNR_MIN']] fail_msg.append(fmsg.format(*fargs)) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(snr[snr_order]) qc_name_str = 'SNR[{0}]'.format(snr_order) qc_names.append(qc_name_str) qc_logic.append('{0} < {1:.2f}'.format(qc_name_str, p['QC_MK_TELLU_SNR_ORDER'])) # ---------------------------------------------------------------------- # check that the file passed the CalcTelluAbsorption sigma clip loop if not loc['PASSED']: fmsg = 'File {0} did not converge on a solution in function: {1}' fargs = [basefilename, 'spirouTelluric.CalcTelluAbsorption()'] fail_msg.append(fmsg.format(*fargs)) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(basefilename) qc_names.append('FILE') qc_logic.append('FILE did not converge') # ---------------------------------------------------------------------- # check that the airmass is not too different from input airmass airmass_diff = np.abs(loc['RECOV_AIRMASS'] - loc['AIRMASS']) fargs = [ loc['RECOV_AIRMASS'], loc['AIRMASS'], p['QC_MKTELLU_AIRMASS_DIFF'] ] if airmass_diff > p['QC_MKTELLU_AIRMASS_DIFF']: fmsg = ('Recovered airmass to de-similar than input airmass.' 'Recovered: {0:.3f}. Input: {1:.3f}. QC limit = {2}') fail_msg.append(fmsg.format(*fargs)) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(airmass_diff) qc_names.append('airmass_diff') qc_logic.append('airmass_diff > {0:.2f}' ''.format(p['QC_MKTELLU_AIRMASS_DIFF'])) # ---------------------------------------------------------------------- # check that the water vapor is within limits water_cond1 = loc['RECOV_WATER'] < p['MKTELLU_TRANS_MIN_WATERCOL'] water_cond2 = loc['RECOV_WATER'] > p['MKTELLU_TRANS_MAX_WATERCOL'] fargs = [ p['MKTELLU_TRANS_MIN_WATERCOL'], p['MKTELLU_TRANS_MAX_WATERCOL'] ] if water_cond1 or water_cond2: fmsg = ('Recovered water vapor optical depth not between {0:.3f} ' 'and {1:.3f}') fail_msg.append(fmsg.format(*fargs)) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(loc['RECOV_WATER']) qc_names.append('RECOV_WATER') qc_logic.append('RECOV_WATER not between {0:.3f} and {1:.3f}' ''.format(*fargs)) # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') continue # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Save transmission map to file # ------------------------------------------------------------------ # get raw file name raw_in_file = os.path.basename(p['FITSFILENAME']) # copy original keys hdict = spirouImage.CopyOriginalKeys(loc['DATAHDR']) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=os.path.basename(masterwavefile)) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=mwsource) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # add wave solution date hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME1'], value=master_acqtimes[0]) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME2'], value=master_acqtimes[1]) # add wave solution number of orders hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_ORD_N'], value=masterwavep.shape[0]) # add wave solution degree of fit hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_LL_DEG'], value=masterwavep.shape[1] - 1) # add wave solution coefficients hdict = spirouImage.AddKey2DList(p, hdict, p['KW_WAVE_PARAM'], values=masterwavep) # add telluric keys hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_AIRMASS'], value=loc['RECOV_AIRMASS']) hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_WATER'], value=loc['RECOV_WATER']) # write to file p = spirouImage.WriteImage(p, outfile, transmission_map, hdict) # ------------------------------------------------------------------ # Add transmission map to telluDB # ------------------------------------------------------------------ if p['QC']: # copy tellu file to the telluDB folder spirouDB.PutTelluFile(p, outfile) # update the master tellu DB file with transmission map targs = [ p, outfilename, loc['OBJNAME'], loc['RECOV_AIRMASS'], loc['RECOV_WATER'] ] spirouDB.UpdateDatabaseTellMap(*targs) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files, mainfitsdir='reduced') p = spirouStartup.InitialFileSetup(p, calibdb=True) # set up function name main_name = __NAME__ + '.main()' # ---------------------------------------------------------------------- # Load first file # ---------------------------------------------------------------------- loc = ParamDict() rd = spirouImage.ReadImage(p, p['FITSFILENAME']) loc['DATA'], loc['DATAHDR'], loc['XDIM'], loc['YDIM'] = rd loc.set_sources(['DATA', 'DATAHDR', 'XDIM', 'YDIM'], main_name) # ---------------------------------------------------------------------- # Get object name, airmass and berv # ---------------------------------------------------------------------- # Get object name loc['OBJNAME'] = spirouImage.GetObjName(p, loc['DATAHDR']) # Get the airmass loc['AIRMASS'] = spirouImage.GetAirmass(p, loc['DATAHDR']) # Get the Barycentric correction from header p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, loc['DATAHDR']) # set sources source = main_name + '+ spirouImage.ReadParams()' loc.set_sources(['OBJNAME', 'AIRMASS'], source) loc.set_sources(['OBJNAME', 'AIRMASS'], source) # ---------------------------------------------------------------------- # Read wavelength solution # ---------------------------------------------------------------------- # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # used for plotting only wout = spirouImage.GetWaveSolution(p, image=loc['DATA'], hdr=loc['DATAHDR'], return_wavemap=True, fiber=wave_fiber) _, loc['WAVE'], _ = wout loc.set_source('WAVE', main_name) # ---------------------------------------------------------------------- # Get and Normalise the blaze # ---------------------------------------------------------------------- p, loc = spirouTelluric.GetNormalizedBlaze(p, loc, loc['DATAHDR']) # ---------------------------------------------------------------------- # Load transmission files # ---------------------------------------------------------------------- transdata = spirouDB.GetDatabaseTellMap(p) trans_files = transdata[0] # make sure we have unique filenames for trans_files trans_files = np.unique(trans_files) # ---------------------------------------------------------------------- # Start plotting # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # start interactive plot sPlt.start_interactive_session(p) # ---------------------------------------------------------------------- # Load template (if available) # ---------------------------------------------------------------------- # read filename from telluDB template_file = spirouDB.GetDatabaseObjTemp(p, loc['OBJNAME'], required=False) # if we don't have a template flag it if template_file is None: loc['FLAG_TEMPLATE'] = False loc['TEMPLATE'] = None else: loc['FLAG_TEMPLATE'] = True # load template template, _, _, _ = spirouImage.ReadImage(p, template_file) # add to loc loc['TEMPLATE'] = template # set the source for flag and template loc.set_sources(['FLAG_TEMPLATE', 'TEMPLATE'], main_name) # ---------------------------------------------------------------------- # load the expected atmospheric transmission # ---------------------------------------------------------------------- # read filename from telluDB tapas_file_names = spirouDB.GetDatabaseTellConv(p) tapas_file_name = tapas_file_names[-1] # load atmospheric transmission loc['TAPAS_ALL_SPECIES'] = np.load(tapas_file_name) loc.set_source('TAPAS_ALL_SPECIES', main_name) # ---------------------------------------------------------------------- # Generate the absorption map # ---------------------------------------------------------------------- # get number of files nfiles = len(trans_files) npc = p['TELLU_NUMBER_OF_PRINCIPLE_COMP'] # check that we have enough files (greater than number of principle # components) if nfiles <= npc: emsg1 = 'Not enough "TELL_MAP" files in telluDB to run PCA analysis' emsg2 = '\tNumber of files = {0}, number of PCA components = {1}' emsg3 = '\tNumber of files > number of PCA components' emsg4 = '\tAdd more files or reduce number of PCA components' WLOG(p, 'error', [emsg1, emsg2.format(nfiles, npc), emsg3, emsg4]) # check whether we can used pre-saved abso filetime = spirouImage.GetMostRecent(trans_files) tout = spirouConfig.Constants.TELLU_ABSO_SAVE(p, filetime) abso_save_file, absoprefix = tout use_saved = os.path.exists(abso_save_file) # noinspection PyBroadException try: # try loading from file abso = np.load(abso_save_file) # log progress wmsg = 'Loaded abso from file {0}'.format(abso_save_file) WLOG(p, '', wmsg) except: # set up storage for the absorption abso = np.zeros([nfiles, np.product(loc['DATA'].shape)]) # loop around outputfiles and add them to abso for it, filename in enumerate(trans_files): # load data data_it, _, _, _ = spirouImage.ReadImage(p, filename=filename) # push data into array abso[it, :] = data_it.reshape(np.product(loc['DATA'].shape)) # log progres wmsg = 'Saving abso to file {0}'.format(abso_save_file) WLOG(p, '', wmsg) # remove all abso save files (only need most recent one) afolder = os.path.dirname(abso_save_file) afilelist = os.listdir(afolder) for afile in afilelist: if afile.startswith(absoprefix): os.remove(os.path.join(afolder, afile)) # save to file for later use np.save(abso_save_file, abso) # filter out rows of all NaNs # TODO: Why are we getting all NaN e2ds files? abso_filtered = [] for row in range(len(abso)): if np.sum(np.isnan(abso[row])) != len(abso[row]): abso_filtered.append(abso[row]) else: wargs = [trans_files[row]] WLOG(p, '', 'Removing trans file {0} (all NaN)'.format(*wargs)) abso = np.array(abso_filtered) # log the absorption cube with warnings.catch_warnings(record=True) as w: log_abso = np.log(abso) # ---------------------------------------------------------------------- # Locate valid pixels for PCA # ---------------------------------------------------------------------- # determining the pixels relevant for PCA construction keep = np.isfinite(np.sum(abso, axis=0)) # log fraction of valid (non NaN) pixels fraction = np.nansum(keep) / len(keep) wmsg = 'Fraction of valid pixels (not NaNs) for PCA construction = {0:.3f}' WLOG(p, '', wmsg.format(fraction)) # log fraction of valid pixels > 1 - (1/e) with warnings.catch_warnings(record=True) as w: keep &= np.min(log_abso, axis=0) > -1 fraction = np.nansum(keep) / len(keep) wmsg = 'Fraction of valid pixels with transmission > 1 - (1/e) = {0:.3f}' WLOG(p, '', wmsg.format(fraction)) # ---------------------------------------------------------------------- # Perform PCA analysis on the log of the telluric absorption map # ---------------------------------------------------------------------- # Requires p: # TELLU_NUMBER_OF_PRINCIPLE_COMP # ADD_DERIV_PC # FIT_DERIV_PC # Requires loc: # DATA # Returns loc: # PC # NPC # FIT_PC loc = spirouTelluric.CalculateAbsorptionPCA(p, loc, log_abso, keep) # Plot PCA components # debug plot if p['DRS_PLOT'] and (p['DRS_DEBUG'] > 1): # plot the transmission map plot sPlt.tellu_pca_comp_plot(p, loc) # ---------------------------------------------------------------------- # Get master wavelength grid for shifting # ---------------------------------------------------------------------- # get master wave map loc['MASTERWAVEFILE'] = spirouDB.GetDatabaseMasterWave(p) loc.set_source('MASTERWAVEFILE', main_name) # log progress wmsg1 = 'Getting master wavelength grid' wmsg2 = '\tFile = {0}'.format(os.path.basename(loc['MASTERWAVEFILE'])) WLOG(p, '', [wmsg1, wmsg2]) # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # read master wave map mout = spirouImage.GetWaveSolution(p, filename=loc['MASTERWAVEFILE'], return_wavemap=True, quiet=True, fiber=wave_fiber) _, loc['MASTERWAVE'], _ = mout loc.set_source('MASTERWAVE', main_name) # ---------------------------------------------------------------------- # Loop around telluric files # ---------------------------------------------------------------------- for basefilename in p['ARG_FILE_NAMES']: # ------------------------------------------------------------------ # Construct absolute file path # ------------------------------------------------------------------ filename = os.path.join(p['ARG_FILE_DIR'], basefilename) # ------------------------------------------------------------------ # Construct output file names # ------------------------------------------------------------------ outfile1, tag1 = CONSTANTS.TELLU_FIT_OUT_FILE(p, filename) outfilename1 = os.path.basename(outfile1) outfile2, tag2 = CONSTANTS.TELLU_FIT_RECON_FILE(p, filename) outfilename2 = os.path.basename(outfile2) # ------------------------------------------------------------------ # Read filename # ------------------------------------------------------------------ # read image tdata, thdr, _, _ = spirouImage.ReadImage(p, filename) # normalise with blaze function loc['SP'] = tdata / loc['NBLAZE'] loc.set_source('SP', main_name) # ------------------------------------------------------------------ # check that file has valid DPRTYPE # ------------------------------------------------------------------ # get FP_FP DPRTYPE p = spirouImage.ReadParam(p, thdr, 'KW_DPRTYPE', 'DPRTYPE', dtype=str) # if dprtype is incorrect skip if p['DPRTYPE'] not in p['ALLOWED_TELLURIC_DPRTYPES']: wmsg1 = 'Skipping file (DPRTYPE incorrect)' wmsg2 = '\t DPRTYPE = {0}'.format(p['DPRTYPE']) WLOG(p, 'warning', [wmsg1, wmsg2]) continue # ------------------------------------------------------------------ # Set storage # ------------------------------------------------------------------ loc['RECON_ABSO'] = np.ones(np.product(loc['DATA'].shape)) loc['AMPS_ABSOL_TOTAL'] = np.zeros(loc['NPC']) loc.set_sources(['RECON_ABSO', 'AMPS_ABSOL_TOTAL'], main_name) # ------------------------------------------------------------------ # Read wavelength solution # ------------------------------------------------------------------ # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get wavelength solution wout = spirouImage.GetWaveSolution(p, image=tdata, hdr=thdr, return_wavemap=True, return_filename=True, fiber=wave_fiber) _, loc['WAVE_IT'], loc['WAVEFILE'], loc['WSOURCE'] = wout loc.set_sources(['WAVE_IT', 'WAVEFILE', 'WSOURCE'], main_name) # load wave keys loc = spirouImage.GetWaveKeys(p, loc, thdr) # ------------------------------------------------------------------ # Interpolate at shifted wavelengths (if we have a template) # ------------------------------------------------------------------ if loc['FLAG_TEMPLATE']: # Requires p: # TELLU_FIT_KEEP_FRAC # Requires loc: # DATA # TEMPLATE # WAVE_IT # Returns: # TEMPLATE2 loc = spirouTelluric.BervCorrectTemplate(p, loc, thdr) # debug plot if p['DRS_PLOT'] and (p['DRS_DEBUG'] > 1): # plot the transmission map plot sPlt.tellu_fit_tellu_spline_plot(p, loc) # store PC and TAPAS_ALL_SPECIES before shift loc['PC_PRESHIFT'] = np.array(loc['PC']) loc['TAPAS_ALL_PRESHIFT'] = np.array(loc['TAPAS_ALL_SPECIES']) loc.set_sources(['PC_PRESHIFT', 'TAPAS_ALL_PRESHIFT'], main_name) # ------------------------------------------------------------------ # Shift the pca components to correct frame # ------------------------------------------------------------------ # log process wmsg1 = 'Shifting PCA components from master wavelength grid' wmsg2 = '\tFile = {0}'.format(os.path.basename(loc['MASTERWAVEFILE'])) WLOG(p, '', [wmsg1, wmsg2]) # shift pca components (one by one) for comp in range(loc['NPC']): wargs = [p, loc['PC'][:, comp], loc['MASTERWAVE'], loc['WAVE_IT']] shift_pc = spirouTelluric.Wave2Wave(*wargs, reshape=True) loc['PC'][:, comp] = shift_pc.reshape(wargs[1].shape) wargs = [ p, loc['FIT_PC'][:, comp], loc['MASTERWAVE'], loc['WAVE_IT'] ] shift_fpc = spirouTelluric.Wave2Wave(*wargs, reshape=True) loc['FIT_PC'][:, comp] = shift_fpc.reshape(wargs[1].shape) # ------------------------------------------------------------------ # Shift the tapas spectrum to correct frame # ------------------------------------------------------------------ # log process wmsg1 = 'Shifting TAPAS spectrum from master wavelength grid' wmsg2 = '\tFile = {0}'.format(os.path.basename(loc['MASTERWAVEFILE'])) WLOG(p, '', [wmsg1, wmsg2]) # shift tapas for comp in range(len(loc['TAPAS_ALL_SPECIES'])): wargs = [ p, loc['TAPAS_ALL_SPECIES'][comp], loc['MASTERWAVE'], loc['WAVE_IT'] ] stapas = spirouTelluric.Wave2Wave(*wargs, reshape=True) loc['TAPAS_ALL_SPECIES'][comp] = stapas.reshape(wargs[1].shape) # Debug plot to test shifting if p['DRS_PLOT'] and p['DRS_DEBUG'] > 1: sPlt.tellu_fit_debug_shift_plot(p, loc) # ------------------------------------------------------------------ # Calculate reconstructed absorption # ------------------------------------------------------------------ # Requires p: # TELLU_FIT_MIN_TRANSMISSION # TELLU_FIT_NITER # TELLU_LAMBDA_MIN # TELLU_LAMBDA_MAX # TELLU_FIT_VSINI # TRANSMISSION_CUT # FIT_DERIV_PC # LOG_OPT # Requires loc: # FLAG_TEMPLATE # TAPAS_ALL_SPECIES # AMPS_ABSOL_TOTAL # WAVE_IT # TEMPLATE2 # FIT_PC # NPC # PC # Returns loc: # SP2 # TEMPLATE2 # RECON_ABSO # AMPS_ABSOL_TOTAL loc = spirouTelluric.CalcReconAbso(p, loc) # debug plot if p['DRS_PLOT'] > 0: # plot the recon abso plot sPlt.tellu_fit_recon_abso_plot(p, loc) # ------------------------------------------------------------------ # Get molecular absorption # ------------------------------------------------------------------ # Requires p: # TELLU_FIT_LOG_LIMIT # Requeres loc: # RECON_ABSO # TAPAS_ALL_SPECIES # Returns loc: # TAPAS_{molecule} loc = spirouTelluric.CalcMolecularAbsorption(p, loc) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # get SNR for each order from header nbo = loc['DATA'].shape[0] snr_order = p['QC_FIT_TELLU_SNR_ORDER'] snr = spirouImage.Read1Dkey(p, thdr, p['kw_E2DS_SNR'][0], nbo) # check that SNR is high enough if snr[snr_order] < p['QC_FIT_TELLU_SNR_MIN']: fmsg = 'low SNR in order {0}: ({1:.2f} < {2:.2f})' fargs = [snr_order, snr[snr_order], p['QC_FIT_TELLU_SNR_MIN']] fail_msg.append(fmsg.format(*fargs)) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(snr[snr_order]) qc_name_str = 'SNR[{0}]'.format(snr_order) qc_names.append(qc_name_str) qc_logic.append('{0} < {1:.2f}'.format(qc_name_str, p['QC_FIT_TELLU_SNR_ORDER'])) # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') continue # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Get components amplitudes for header # ------------------------------------------------------------------ # get raw file name raw_in_file = os.path.basename(p['FITSFILENAME']) # copy original keys hdict = spirouImage.CopyOriginalKeys(thdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # set tellu keys npc = loc['NPC'] hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_NPC'], value=npc) hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_FIT_DPC'], value=p['FIT_DERIV_PC']) hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_ADD_DPC'], value=p['ADD_DERIV_PC']) if p['ADD_DERIV_PC']: values = loc['AMPS_ABSOL_TOTAL'][:npc - 2] hdict = spirouImage.AddKey1DList(p, hdict, p['KW_TELLU_AMP_PC'], values=values, dim1name='amp') hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_DV_TELL1'], value=loc['AMPS_ABSOL_TOTAL'][npc - 2]) hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_DV_TELL2'], value=loc['AMPS_ABSOL_TOTAL'][npc - 1]) else: values = loc['AMPS_ABSOL_TOTAL'][:npc] hdict = spirouImage.AddKey1DList(p, hdict, p['KW_TELLU_AMP_PC'], values=values, dim1name='PC') # ------------------------------------------------------------------ # Write corrected spectrum to E2DS # ------------------------------------------------------------------ # reform the E2DS sp_out = loc['SP2'] / loc['RECON_ABSO'] sp_out = sp_out.reshape(loc['DATA'].shape) # multiply by blaze sp_out = sp_out * loc['NBLAZE'] hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) # log progress wmsg = 'Saving {0} to file'.format(outfilename1) WLOG(p, '', wmsg) # write sp_out to file p = spirouImage.WriteImage(p, outfile1, sp_out, hdict) # ------------------------------------------------------------------ # 1-dimension spectral S1D (uniform in wavelength) # ------------------------------------------------------------------ # get arguments for E2DS to S1D e2dsargs = [loc['WAVE'], sp_out, loc['BLAZE']] # get 1D spectrum xs1d1, ys1d1 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='wave') # Plot the 1D spectrum if p['DRS_PLOT'] > 0: sPlt.ext_1d_spectrum_plot(p, xs1d1, ys1d1) # construct file name targs = [p, raw_in_file] s1dfile1, tag3 = spirouConfig.Constants.TELLU_FIT_S1D_FILE1(*targs) s1dfilename1 = os.path.basename(s1dfile1) # add header keys # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) # log writing to file wmsg = 'Saving 1D spectrum (uniform in wavelength) in {0}' WLOG(p, '', wmsg.format(s1dfilename1)) # Write to file columns = ['wavelength', 'flux', 'eflux'] values = [xs1d1, ys1d1, np.zeros_like(ys1d1)] units = ['nm', None, None] s1d1 = spirouImage.MakeTable(p, columns, values, units=units) spirouImage.WriteTable(p, s1d1, s1dfile1, header=hdict) # ------------------------------------------------------------------ # 1-dimension spectral S1D (uniform in velocity) # ------------------------------------------------------------------ # get arguments for E2DS to S1D e2dsargs = [loc['WAVE'], sp_out, loc['BLAZE']] # get 1D spectrum xs1d2, ys1d2 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='velocity') # Plot the 1D spectrum if p['DRS_PLOT'] > 0: sPlt.ext_1d_spectrum_plot(p, xs1d2, ys1d2) # construct file name targs = [p, raw_in_file] s1dfile2, tag4 = spirouConfig.Constants.TELLU_FIT_S1D_FILE2(*targs) s1dfilename2 = os.path.basename(s1dfile2) # add header keys hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) # log writing to file wmsg = 'Saving 1D spectrum (uniform in velocity) in {0}' WLOG(p, '', wmsg.format(s1dfilename2)) # Write to file columns = ['wavelength', 'flux', 'eflux'] values = [xs1d2, ys1d2, np.zeros_like(ys1d2)] units = ['nm', None, None] s1d2 = spirouImage.MakeTable(p, columns, values, units=units) spirouImage.WriteTable(p, s1d2, s1dfile2, header=hdict) # ------------------------------------------------------------------ # Write reconstructed absorption to E2DS # ------------------------------------------------------------------ # set up empty storage recon_abso2 = np.zeros_like(loc['DATA']) # get dimensions of data ydim, xdim = loc['DATA'].shape # loop around orders for order_num in range(ydim): # get start and end points start, end = xdim * order_num, xdim * order_num + xdim # save to storage recon_abso2[order_num, :] = loc['RECON_ABSO'][start:end] # add molecular absorption to file for it, molecule in enumerate(p['TELLU_ABSORBERS'][1:]): # get molecule keyword store and key molkey = '{0}_{1}'.format(p['KW_TELLU_ABSO'][0], molecule.upper()) molkws = [molkey, 0, 'Absorption in {0}'.format(molecule.upper())] # load into hdict hdict = spirouImage.AddKey(p, hdict, molkws, value=loc[molkey]) # add water col if molecule == 'h2o': loc['WATERCOL'] = loc[molkey] # set source loc.set_source('WATERCOL', main_name) # add the tau keys hdict = spirouImage.AddKey(p, hdict, p['KW_TAU_H2O'], value=loc['TAU_H2O']) hdict = spirouImage.AddKey(p, hdict, p['KW_TAU_REST'], value=loc['TAU_REST']) # log progress wmsg = 'Saving {0} to file'.format(outfilename2) WLOG(p, '', wmsg) # write recon_abso to file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) p = spirouImage.WriteImage(p, outfile2, recon_abso2, hdict) # ------------------------------------------------------------------ # Update the Telluric database # ------------------------------------------------------------------ if p['QC']: # add TELLU_OBJ to telluric database oparams = dict(objname=loc['OBJNAME'], berv=loc['BERV'], airmass=loc['AIRMASS'], watercol=loc['WATERCOL']) spirouDB.UpdateDatabaseTellObj(p, outfilename1, **oparams) # copy file to database spirouDB.PutTelluFile(p, outfile1) # add TELLU_RECON to telluric database # add TELLU_OBJ to telluric database oparams = dict(objname=loc['OBJNAME'], berv=loc['BERV'], airmass=loc['AIRMASS'], watercol=loc['WATERCOL']) spirouDB.UpdateDatabaseTellRecon(p, outfilename2, **oparams) # copy file to database spirouDB.PutTelluFile(p, outfile2) # ---------------------------------------------------------------------- # End plotting # ---------------------------------------------------------------------- # debug plot if p['DRS_PLOT'] > 0: # end interactive session sPlt.end_interactive_session(p) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())