def main(night_name=None, key=None, filename=None): """ Manually add a file the the calibDB with "key" i.e. adds key night_name filename human-date unix-time to the calibDB and copies "filename" from .../reduced_dir/night_name/ into the calibDB Note filename must be in .../reduced_dir/night_name/ """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with arguments being None (i.e. get from sys.argv) pos = [0, 1] fmt = [str, str] names = ['key', 'filename'] call = [key, filename] # now get custom arguments customargs = spirouStartup.GetCustomFromRuntime(p, pos, fmt, names, calls=call) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='filename', mainfitsdir='reduced') # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data image, hdr, nbo, nx = spirouImage.ReadData(p, p['FITSFILENAME']) # ---------------------------------------------------------------------- # Move to calibDB and update calibDB # ---------------------------------------------------------------------- # set dark key keydb = p['KEY'] # copy dark fits file to the calibDB folder spirouDB.PutCalibFile(p, p['FITSFILENAME']) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, p['FILENAME'], hdr) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(directory=None, filelist=None, **kwargs): # assign function calls (must add positional) fkwargs = dict(directory=directory, filelist=filelist, **kwargs) # deal with command line inputs / function call inputs recipe, p = spirouStartup.spirouStartup2.input_setup(__NAME__, fkwargs) # display everything that comes from "INPUT" for i in range(10): WLOG(p, '', 'Line {0} of code'.format(i + 1)) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p, outputs=None) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, reffile=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with reference file being None (i.e. get from sys.argv) if reffile is None: customargs = spirouStartup.GetCustomFromRuntime(p, [0], [str], ['reffile']) else: customargs = dict(reffile=reffile) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='reffile', mainfitsdir='reduced') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, reffilename = spirouStartup.SingleFileSetup(p, filename=p['REFFILE'], skipcheck=True) p['REFFILENAME'] = reffilename p.set_source('REFFILENAME', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data data, hdr, nbo, nx = spirouImage.ReadData(p, p['REFFILENAME']) # ---------------------------------------------------------------------- # Add keys and save file # ---------------------------------------------------------------------- newfilename = p['REFFILE'].replace('.fits', '_edit.fits') newpath = os.path.join(p['ARG_FILE_DIR'], newfilename) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # set the version hdict = spirouImage.AddKey(p, hdict, HEADER_KEY, value=HEADER_VALUE) # log saving wmsg = 'Writing file {0} to {1}' WLOG(p, '', wmsg.format(newfilename, p['ARG_FILE_DIR'])) # save drift values p = spirouImage.WriteImage(p, newpath, data, hdict) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): """ cal_DARK_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_DARK_spirou.py [night_directory] [fitsfilename] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) p = spirouStartup.InitialFileSetup(p) # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='average') # ---------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- data = spirouImage.FixNonPreProcess(p, data) # ---------------------------------------------------------------------- # Find the amplitude to use for the local background # ---------------------------------------------------------------------- spirouBACK.MakeLocalBackgroundMap(p, data) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p, outputs=None) # return a copy of locally defined variables in the memory return dict(locals())
def main(): # ---------------------------------------------------------------------- # Run Reset # ---------------------------------------------------------------------- # get constants p = spirouStartup.Begin(__NAME__) # log run WLOG(p, 'info', 'Now Running reset script.') # run DRS reset spirouTools.DRS_Reset(log=True, called=True) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p, outputs=None) # return a copy of locally defined variables in the memory return dict(locals())
def main(cores=1, objects=None, filetype='EXT_E2DS_FF_AB'): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # set up function name main_name = __NAME__ + '.main()' # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # now get custom arguments pos, fmt = [0, 1, 2], [int, str, str] names = ['cores', 'objects', 'filetype'] call = [cores, objects, filetype] required = [False, False, False] customargs = spirouStartup.GetCustomFromRuntime(p, pos, fmt, names, calls=call, required=required, require_night_name=False) p = spirouStartup.LoadArguments(p, None, customargs=customargs, mainfitsdir='reduced', require_night_name=False) # keep track of errors errors = [] # ------------------------------------------------------------------------- # find all objects (matching 'objects' if not None) target_list = spirouTelluric.FindObjects(p) # print number found nfound = len(target_list) nstar = 0 for target in list(target_list.keys()): nstar += len(target_list[target]) wmsg = 'Found target "{0}" ({1} observations total)' WLOG(p, 'info', wmsg.format(target, len(target_list[target]))) # list total found wmsg = 'Found {0} observations total for {1} object(s)' WLOG(p, 'info', wmsg.format(nstar, nfound)) # ------------------------------------------------------------------------- # Step 1: Run fit_tellu on all objects # ------------------------------------------------------------------------- # for t_it, target in enumerate(list(target_list.keys())): # # loop around object filenames # for o_it, objfilename in enumerate(target_list[target]): # # Log progress # pargs = [p, 'Fit Tellurics', target, t_it, nfound, # o_it, len(target_list[target])] # spirouTelluric.UpdateProcessDB(*pargs) # # get arguments from filename # args = spirouTelluric.GetDBarguments(p, objfilename) # # run obj_mk_tellu # try: # obj_fit_tellu.main(**args) # except SystemExit as e: # errors.append([pargs[1], objfilename, e]) # # force close all plots # sPlt.closeall() # ------------------------------------------------------------------------- # Step 2: Run mk_obj_template on each science target # ------------------------------------------------------------------------- for t_it, target in enumerate(list(target_list.keys())): # log progress (big) pmsg = 'Make Telluric Template: Processing object = {0} ({1}/{2}' wmsgs = [ '', '=' * 60, '', pmsg.format(target, t_it + 1, nfound), '', '=' * 60, '' ] WLOG(p, 'info', wmsgs, wrap=False) # get last object objfilename = target_list[target][-1] # get arguments from filename args = spirouTelluric.GetDBarguments(p, objfilename) # run obj_mk_obj_template try: obj_mk_obj_template.main(**args) except SystemExit as e: errors.append(['Telluric Template', target, e]) # force close all plots sPlt.closeall() # ------------------------------------------------------------------------- # Step 3: Re-Run fit_tellu on all objects # ------------------------------------------------------------------------- for t_it, target in enumerate(list(target_list.keys())): # loop around object filenames for o_it, objfilename in enumerate(target_list[target]): # Log progress pargs = [ p, 'Fit Tellurics II', target, t_it, nfound, o_it, len(target_list[target]) ] spirouTelluric.UpdateProcessDB(*pargs) # get arguments from filename args = spirouTelluric.GetDBarguments(p, objfilename) # run obj_mk_tellu try: obj_fit_tellu.main(**args) except SystemExit as e: errors.append([pargs[1], objfilename, e]) # force close all plots sPlt.closeall() # ------------------------------------------------------------------------- # Step 4: Re-Run mk_obj_template on each science target # ------------------------------------------------------------------------- for t_it, target in enumerate(list(target_list.keys())): # log progress (big) pmsg = 'Make Telluric Template II: Processing object = {0} ({1}/{2}' wmsgs = [ '', '=' * 60, '', pmsg.format(target, t_it + 1, nfound), '', '=' * 60, '' ] WLOG(p, 'info', wmsgs, wrap=False) # get last object objfilename = target_list[target][-1] # get arguments from filename args = spirouTelluric.GetDBarguments(p, objfilename) # run obj_mk_obj_template try: obj_mk_obj_template.main(**args) except SystemExit as e: errors.append(['Telluric Template', target, e]) # force close all plots sPlt.closeall() # ---------------------------------------------------------------------- # Print all errors # ---------------------------------------------------------------------- if len(errors) > 0: emsgs = ['', '=' * 50, 'Errors were as follows: '] # loop around errors for error in errors: emsgs.append('') emsgs.append('{0}: Object = {1}'.format(error[0], error[1])) emsgs.append('\t{0}'.format(error[2])) emsgs.append('') WLOG(p, 'error', emsgs) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): """ cal_FF_RAW_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_FF_RAW_spirou.py [night_directory] [files] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) p = spirouStartup.InitialFileSetup(p) # get the night name night_name = p['ARG_NIGHT_NAME'] # ---------------------------------------------------------------------- # get the e2ds filenames (uses ARG_FILE_NAMES by default) exists = False e2ds_files, e2dsff_files = dict(), dict() for fiber in p['FIBER_TYPES']: # must set file type p['FIBER'] = fiber p.set_source('FIBER', __NAME__ + '.main()') # construct this fibers file names e2dsfits, tag1 = spirouConfig.Constants.EXTRACT_E2DS_FILE(p) e2dsfffits, tag2 = spirouConfig.Constants.EXTRACT_E2DSFF_FILE(p) # check whether e2ds file exists if os.path.exists(e2dsfits): exists = exists and True else: exists = exists and False # append to files e2ds_files[fiber] = e2dsfits e2dsff_files[fiber] = e2dsfffits # ---------------------------------------------------------------------- # extract the dark or read dark e2ds header # ---------------------------------------------------------------------- # check if e2ds file exists - if not extract if p['ALWAYS_EXTRACT'] or (not exists): llout = cal_extract_RAW_spirou.main(night_name, files) # get qc passed = llout['p']['QC'] # get hdr hdr = llout['hdr'] # else we need to get else: # read file header and push into outputs hdr = spirouImage.ReadHeader(p, e2ds_files['AB']) # get quality control parameters passed = hdr['KW_DRS_QC'] # ------------------------------------------------------------------ # Update the calibration database # ------------------------------------------------------------------ for fiber in p['FIBER_TYPES']: if passed: # copy THERMAL_{FIBER} to calibdb keydb = 'THERMAL_' + fiber # get absolute path abspath = e2ds_files[fiber] basename = os.path.basename(abspath) # copy localisation file to the calibDB folder spirouDB.PutCalibFile(p, abspath) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, basename, hdr) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): """ cal_DARK_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_DARK_spirou.py [night_directory] [fitsfilename] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) p = spirouStartup.InitialFileSetup(p) # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='average') # ---------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- data = spirouImage.FixNonPreProcess(p, data) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # ---------------------------------------------------------------------- # Dark exposure time check # ---------------------------------------------------------------------- # log the Dark exposure time WLOG(p, 'info', 'Dark Time = {0:.3f} s'.format(p['EXPTIME'])) # Quality control: make sure the exposure time is longer than qc_dark_time if p['EXPTIME'] < p['QC_DARK_TIME']: emsg = 'Dark exposure time too short (< {0:.1f} s)' WLOG(p, 'error', emsg.format(p['QC_DARK_TIME'])) # ---------------------------------------------------------------------- # Resize image # ---------------------------------------------------------------------- # # rotate the image and conver from ADU/s to e- # data = data[::-1, ::-1] * p['exptime'] * p['gain'] # convert NaN to zeros nanmask = ~np.isfinite(data) data0 = np.where(nanmask, np.zeros_like(data), data) # resize blue image bkwargs = dict(xlow=p['IC_CCDX_BLUE_LOW'], xhigh=p['IC_CCDX_BLUE_HIGH'], ylow=p['IC_CCDY_BLUE_LOW'], yhigh=p['IC_CCDY_BLUE_HIGH']) datablue, nx2, ny2 = spirouImage.ResizeImage(p, data, **bkwargs) # Make sure we have data in the blue image if nx2 == 0 or ny2 == 0: WLOG(p, 'error', ('IC_CCD(X/Y)_BLUE_(LOW/HIGH) remove ' 'all pixels from image.')) # resize red image rkwargs = dict(xlow=p['IC_CCDX_RED_LOW'], xhigh=p['IC_CCDX_RED_HIGH'], ylow=p['IC_CCDY_RED_LOW'], yhigh=p['IC_CCDY_RED_HIGH']) datared, nx3, ny3 = spirouImage.ResizeImage(p, data, **rkwargs) # Make sure we have data in the red image if nx3 == 0 or ny3 == 0: WLOG(p, 'error', ('IC_CCD(X/Y)_RED_(LOW/HIGH) remove ' 'all pixels from image.')) # ---------------------------------------------------------------------- # Dark Measurement # ---------------------------------------------------------------------- # Log that we are doing dark measurement WLOG(p, '', 'Doing Dark measurement') # measure dark for whole frame p = spirouImage.MeasureDark(p, data, 'Whole det', 'full') # measure dark for blue part p = spirouImage.MeasureDark(p, datablue, 'Blue part', 'blue') # measure dark for rede part p = spirouImage.MeasureDark(p, datared, 'Red part', 'red') # ---------------------------------------------------------------------- # Identification of bad pixels # ---------------------------------------------------------------------- # get number of bad dark pixels (as a fraction of total pixels) with warnings.catch_warnings(record=True) as w: baddark = 100.0 * np.sum(data0 > p['DARK_CUTLIMIT']) baddark /= np.product(data0.shape) # log the fraction of bad dark pixels wmsg = 'Frac pixels with DARK > {0:.2f} ADU/s = {1:.3f} %' WLOG(p, 'info', wmsg.format(p['DARK_CUTLIMIT'], baddark)) # define mask for values above cut limit or NaN with warnings.catch_warnings(record=True) as w: datacutmask = ~((data0 > p['DARK_CUTLIMIT']) | (~np.isfinite(data))) spirouCore.spirouLog.warninglogger(p, w) # get number of pixels above cut limit or NaN n_bad_pix = np.product(data.shape) - np.nansum(datacutmask) # work out fraction of dead pixels + dark > cut, as percentage p['DADEADALL'] = n_bad_pix * 100 / np.product(data.shape) p.set_source('DADEADALL', __NAME__ + '/main()') # log fraction of dead pixels + dark > cut logargs = [p['DARK_CUTLIMIT'], p['DADEADALL']] WLOG(p, 'info', ('Total Frac dead pixels (N.A.N) + DARK > ' '{0:.2f} ADU/s = {1:.3f} %').format(*logargs)) # ---------------------------------------------------------------------- # Plots # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # start interactive plot sPlt.start_interactive_session(p) # plot the image with blue and red regions sPlt.darkplot_image_and_regions(p, data) # plot histograms sPlt.darkplot_histograms(p) # end interactive session sPlt.end_interactive_session(p) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # check that med < qc_max_darklevel if p['MED_FULL'] > p['QC_MAX_DARKLEVEL']: # add failed message to fail message list fmsg = 'Unexpected Median Dark level ({0:5.2f} > {1:5.2f} ADU/s)' fail_msg.append(fmsg.format(p['MED_FULL'], p['QC_MAX_DARKLEVEL'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(p['MED_FULL']) qc_names.append('MED_FULL') qc_logic.append('MED_FULL > {0:.2f}'.format(p['QC_MAX_DARKLEVEL'])) # ---------------------------------------------------------------------- # check that fraction of dead pixels < qc_max_dead if p['DADEADALL'] > p['QC_MAX_DEAD']: # add failed message to fail message list fmsg = 'Unexpected Fraction of dead pixels ({0:5.2f} > {1:5.2f} %)' fail_msg.append(fmsg.format(p['DADEADALL'], p['QC_MAX_DEAD'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(p['DADEADALL']) qc_names.append('DADEADALL') qc_logic.append('DADEADALL > {0:.2f}'.format(p['QC_MAX_DEAD'])) # ---------------------------------------------------------------------- # checl that the precentage of dark pixels < qc_max_dark if baddark > p['QC_MAX_DARK']: fmsg = ('Unexpected Fraction of dark pixels > {0:.2f} ADU/s ' '({1:.2f} > {2:.2f}') fail_msg.append( fmsg.format(p['DARK_CUTLIMIT'], baddark, p['QC_MAX_DARK'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(baddark) qc_names.append('baddark') qc_logic.append('baddark > {0:.2f}'.format(p['QC_MAX_DARK'])) # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ---------------------------------------------------------------------- # Save dark to file # ---------------------------------------------------------------------- # get raw dark filename rawdarkfile = os.path.basename(p['FITSFILENAME']) # construct folder and filename darkfits, tag = spirouConfig.Constants.DARK_FILE(p) darkfitsname = os.path.basename(darkfits) # log saving dark frame WLOG(p, '', 'Saving Dark frame in ' + darkfitsname) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # define new keys to add hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) hdict = spirouImage.AddKey(p, hdict, p['KW_DARK_DEAD'], value=p['DADEAD_FULL']) hdict = spirouImage.AddKey(p, hdict, p['KW_DARK_MED'], value=p['MED_FULL']) hdict = spirouImage.AddKey(p, hdict, p['KW_DARK_B_DEAD'], value=p['DADEAD_BLUE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DARK_B_MED'], value=p['MED_BLUE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DARK_R_DEAD'], value=p['DADEAD_RED']) hdict = spirouImage.AddKey(p, hdict, p['KW_DARK_R_MED'], value=p['MED_RED']) hdict = spirouImage.AddKey(p, hdict, p['KW_DARK_CUT'], value=p['DARK_CUTLIMIT']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # Set to zero dark value > dark_cutlimit cutmask = data0 > p['DARK_CUTLIMIT'] data0c = np.where(cutmask, np.zeros_like(data0), data0) # write image and add header keys (via hdict) p = spirouImage.WriteImage(p, darkfits, data0c, hdict) # ---------------------------------------------------------------------- # Save bad pixel mask # ---------------------------------------------------------------------- # construct bad pixel file name badpixelfits, tag = spirouConfig.Constants.DARK_BADPIX_FILE(p) badpixelfitsname = os.path.split(badpixelfits)[-1] # log that we are saving bad pixel map in dir WLOG(p, '', 'Saving Bad Pixel Map in ' + badpixelfitsname) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # define new keys to add hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) hdict['DACUT'] = (p['DARK_CUTLIMIT'], 'Threshold of dark level retain [ADU/s]') # write to file datacutmask = np.array(datacutmask, dtype=float) p = spirouImage.WriteImage(p, badpixelfits, datacutmask, hdict, dtype='float64') # ---------------------------------------------------------------------- # Move to calibDB and update calibDB # ---------------------------------------------------------------------- if p['QC']: # set dark key if p['DPRTYPE'] == 'DARK_DARK': keydb = 'DARK' elif p['USE_SKYDARK_CORRECTION']: keydb = 'SKYDARK' else: emsg = 'Error: Currently {0} only supports DARK_DARK and OBJ_DARK' WLOG(p, 'error', emsg.format(__NAME__)) # copy dark fits file to the calibDB folder spirouDB.PutCalibFile(p, darkfits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, darkfitsname, hdr) # # set badpix key # keydb = 'BADPIX_OLD' # # copy badpix fits file to calibDB folder # spirouDB.PutCalibFile(p, badpixelfits) # # update the master calib DB file with new key # spirouDB.UpdateCalibMaster(p, keydb, badpixelfitsname, hdr) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(runfile=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, require_night_name=False) # deal with run file p, runtable = spirouReprocess.RunFile(p, runfile) # reset sys.argv so it doesn't mess with recipes sys.argv = [__NAME__] # send email if configured spirouReprocess.SendEmail(p, kind='start') # ---------------------------------------------------------------------- # Deal with reset options # ---------------------------------------------------------------------- spirouReprocess.ResetFiles(p) # ---------------------------------------------------------------------- # find all files # ---------------------------------------------------------------------- WLOG(p, 'info', 'Finding all raw files') rawtable, rawpath = spirouReprocess.FindRawFiles(p) # WLOG(p, 'info', 'Finding all pp files') # tmptable, tmppath = spirouReprocess.FindTmpFiles(p) # WLOG(p, 'info', 'Finding all reduced files') # redtable, redpath = spirouReprocess.FindRedFiles(p) # store in lists tables = [rawtable] # , tmptable, redtable] paths = [rawpath] # , tmppath, redpath] # ---------------------------------------------------------------------- # Generate run list # ---------------------------------------------------------------------- runlist = spirouReprocess.GenerateRunList(p, tables, paths, runtable) # ---------------------------------------------------------------------- # Process run list # ---------------------------------------------------------------------- outlist = spirouReprocess.ProcessRunList(p, runlist) # ---------------------------------------------------------------------- # Print timing # ---------------------------------------------------------------------- # get header header = spirouConfig.Constants.HEADER() WLOG(p, '', '') WLOG(p, '', header) WLOG(p, '', 'Timings:') WLOG(p, '', header) WLOG(p, '', '') # loop around timings (non-errors only) for key in outlist: cond1 = len(outlist[key]['ERROR']) == 0 cond2 = outlist[key]['TIMING'] is not None if cond1 and cond2: wmsg = 'ID={0:05d} Time = {1}' WLOG(p, '', wmsg.format(key, outlist[key]['TIMING'])) WLOG(p, 'warning', '\t{0}'.format(outlist[key]['RUNSTRING']), wrap=False) # ---------------------------------------------------------------------- # Print out any errors # ---------------------------------------------------------------------- # get header header = spirouConfig.Constants.HEADER() WLOG(p, '', '') WLOG(p, '', header) WLOG(p, '', 'Errors:') WLOG(p, '', header) WLOG(p, '', '') # loop around each entry of outlist and print any errors for key in outlist: if len(outlist[key]['ERROR']) > 0: WLOG(p, '', '', colour='red') WLOG(p, '', header, colour='red') WLOG(p, 'warning', 'Error found for ID={0:05d}'.format(key), colour='red', wrap=False) WLOG(p, 'warning', '\t{0}'.format(outlist[key]['RUNSTRING']), colour='red', wrap=False) WLOG(p, '', header, colour='red') WLOG(p, '', '', colour='red') WLOG(p, 'warning', outlist[key]['ERROR'], colour='red', wrap=False) WLOG(p, '', '', colour='red') WLOG.printmessage(p, outlist[key]['TRACEBACK'], colour='red') WLOG(p, '', '', colour='red') WLOG(p, '', header, colour='red') # send email if configured spirouReprocess.SendEmail(p, kind='end') # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # get parameters from configuration files and run time arguments customargs = spirouStartup.GetCustomFromRuntime(p, [0], [str], ['reffile']) p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='reffile', mainfitsdir='reduced') # setup files and get fiber p = spirouStartup.InitialFileSetup(p, calibdb=True) # set the fiber type p['FIB_TYP'] = [p['FIBER']] # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data gfkwargs = dict(path=p['REDUCED_DIR'], filename=p['REFFILE']) p['REFFILENAME'] = spirouStartup.GetFile(p, **gfkwargs) p.set_source('REFFILENAME', __NAME__ + '/main()') # get the fiber type p['FIBER'] = 'AB' e2ds, hdr, nx, ny = spirouImage.ReadImage(p) # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get wave image _, wave, _ = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True, fiber=wave_fiber) blaze = spirouImage.ReadBlazeFile(p) # ---------------------------------------------------------------------- # Get lamp params # ---------------------------------------------------------------------- # get lamp parameters p = spirouTHORCA.GetLampParams(p, hdr) # ---------------------------------------------------------------------- # Get catalogue and fitted line list # ---------------------------------------------------------------------- # load line file (from p['IC_LL_LINE_FILE']) ll_line_cat, ampl_line_cat = spirouImage.ReadLineList(p) # construct fitted lines table filename wavelltbl = spirouConfig.Constants.WAVE_LINE_FILE(p) WLOG(p, '', wavelltbl) # read fitted lines ll_ord, ll_line_fit, ampl_line_fit = np.genfromtxt(wavelltbl, skip_header=4, skip_footer=2, unpack=True, usecols=(0, 1, 3)) # ---------------------------------------------------------------------- # Plots # ---------------------------------------------------------------------- # define line colours col = ['magenta', 'purple'] # get order parity ll_ord_par = np.mod(ll_ord, 2) print(ll_ord_par) col2 = [col[int(x)] for x in ll_ord_par] # start interactive plot sPlt.start_interactive_session(p) plt.figure() for order_num in np.arange(nx): plt.plot(wave[order_num], e2ds[order_num]) # get heights heights = [] for line in range(len(ll_line_cat)): heights.append(200000 + np.max([np.min(e2ds), ampl_line_cat[line]])) # plot ll_line_cat plt.vlines(ll_line_cat, 0, heights, colors='darkgreen', linestyles='dashed') # get heights heights = [] for line in range(len(ll_line_fit)): heights.append(200000 + np.max([np.min(e2ds), ampl_line_fit[line]])) # plot ll_line_fit plt.vlines(ll_line_fit, 0, heights, colors=col2, linestyles='dashdot') plt.xlabel('Wavelength [nm]') plt.ylabel('Flux e-') plt.title(p['REFFILENAME']) # end interactive session # sPlt.end_interactive_session() # old code: # plt.ion() # plt.figure() # # for order_num in np.arange(nx): # plt.plot(wave[order_num], e2ds[order_num]) # # for line in range(len(ll_line_cat)): # plt.vlines(ll_line_cat[line], 0, 200000 + # max(np.min(e2ds), ampl_line_cat[line]), # colors='darkgreen', linestyles='dashed') # # for line in range(len(ll_line_fit)): # plt.vlines(ll_line_fit[line], 0, 200000 + # max(np.min(e2ds), ampl_line_fit[line]), # colors='magenta', linestyles='dashdot') # # plt.xlabel('Wavelength [nm]') # plt.ylabel('Flux e-') # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p, outputs=None) # return a copy of locally defined variables in the memory return dict(locals())
def main(filetype='DARK_DARK'): """ cal_DARK_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_DARK_spirou.py [night_directory] [fitsfilename] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # set up function name main_name = __NAME__ + '.main()' # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # now get custom arguments pos, fmt = [0], [str] names, call = ['filetype'], [filetype] customargs = spirouStartup.GetCustomFromRuntime(p, pos, fmt, names, calls=call, require_night_name=False) p = spirouStartup.LoadArguments(p, None, customargs=customargs, mainfitsdir='tmp', require_night_name=False) # set the DRS type (for file indexing) p['DRS_TYPE'] = 'RAW' p.set_source('DRS_TYPE', __NAME__ + '.main()') # ------------------------------------------------------------------------- # find all "filetype" objects filenames = spirouImage.FindFiles(p, filetype=p['FILETYPE'], allowedtypes=p['ALLOWED_DARK_TYPES']) # convert filenames to a numpy array filenames = np.array(filenames) # ------------------------------------------------------------------------- # julian date to know which file we need to # process together dark_time = np.zeros(len(filenames)) dark_exp, dark_pp_version, dark_wt_temp = [], [], [] basenames, nightnames, dark_cass_temp, dark_humidity = [], [], [], [] # log progress WLOG(p, '', 'Reading all dark file headers') # looping through the file headers for it in range(len(filenames)): # get night name night_name = os.path.dirname(filenames[it]).split(p['TMP_DIR'])[-1] # get header hdr = spirouImage.ReadHeader(p, filepath=filenames[it]) # add MJDATE to dark times dark_time[it] = float(hdr[p['KW_ACQTIME'][0]]) # add other keys (for tabular output) basenames.append(os.path.basename(filenames[it])) nightnames.append(night_name) dark_exp.append(float(hdr[p['KW_EXPTIME'][0]])) dark_pp_version.append(hdr[p['KW_PPVERSION'][0]]) dark_wt_temp.append(float(hdr[p['KW_WEATHER_TOWER_TEMP'][0]])) dark_cass_temp.append(float(hdr[p['KW_CASS_TEMP'][0]])) dark_humidity.append(float(hdr[p['KW_HUMIDITY'][0]])) # ------------------------------------------------------------------------- # match files by date # ------------------------------------------------------------------------- # log progress wmsg = 'Matching dark files by observation time (+/- {0} hrs)' WLOG(p, '', wmsg.format(p['DARK_MASTER_MATCH_TIME'])) # get the time threshold time_thres = p['DARK_MASTER_MATCH_TIME'] # get items grouped by time matched_id = spirouImage.GroupFilesByTime(p, dark_time, time_thres) # ------------------------------------------------------------------------- # get the most recent position lastpos = np.argmax(dark_time) # load up the most recent dark rout = spirouImage.ReadImage(p, filenames[lastpos], log=False) data_ref, hdr_ref, nx, ny = rout # set the night name and update the reduced directory p['ARG_NIGHT_NAME'] = nightnames[lastpos] p.set_source('ARG_NIGHT_NAME', __NAME__ + '.main()') p['REDUCED_DIR'] = spirouConfig.Constants.REDUCED_DIR(p) p.set_source('REDUCED_DIR', __NAME__ + '.main()') # ------------------------------------------------------------------------- # Read individual files and sum groups # ------------------------------------------------------------------------- # log process WLOG(p, '', 'Reading Dark files and combining groups') # Find all unique groups u_groups = np.unique(matched_id) # currently number of bins == number of groups num_bins = len(u_groups) # storage of dark cube dark_cube = np.zeros([num_bins, ny, nx]) bin_cube = np.zeros(num_bins) # loop through groups for g_it, group_num in enumerate(u_groups): # log progress WLOG(p, '', '\tGroup {0} of {1}'.format(g_it + 1, len(u_groups))) # find all files for this group dark_ids = filenames[matched_id == group_num] # load this groups files into a cube cube = [] for filename in dark_ids: # read data data_it, _, _, _ = spirouImage.ReadImage(p, filename, log=False) # add to cube cube.append(data_it) # median dark cube groupdark = np.nanmedian(cube, axis=0) # sum within each bin dark_cube[g_it % num_bins] += groupdark # record the number of cubes that are going into this bin bin_cube[g_it % num_bins] += 1 # need to normalize if we have more than 1 cube per bin for bin_it in range(num_bins): dark_cube[bin_it] /= bin_cube[bin_it] # ------------------------------------------------------------------------- # we perform a median filter over a +/- "med_size" pixel box # ------------------------------------------------------------------------- # log process WLOG(p, '', 'Performing median filter for {0} bins'.format(num_bins)) # get med_size from p med_size = p['DARK_MASTER_MED_SIZE'] # storage of output dark cube dark_cube1 = np.zeros([num_bins, ny, nx]) # loop around the bins for bin_it in range(num_bins): # get the dark for this bin bindark = dark_cube[bin_it] # performing a median filter of the image with [-med_size, med_size] # box in x and 1 pixel wide in y. Skips the pixel considered, # so this is equivalent of a 2*med_size boxcar tmp = [] for jt in range(-med_size, med_size + 1): if jt != 0: tmp.append(np.roll(bindark, [0, jt])) # low frequency image lf_dark = np.nanmedian(tmp, axis=0) # high frequency image dark_cube1[bin_it] = bindark - lf_dark # ------------------------------------------------------------------------- # median the dark cube to create the master dark master_dark = np.nanmedian(dark_cube1, axis=0) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # add to qc header lists qc_values.append('None') qc_names.append('None') qc_logic.append('None') qc_pass.append(1) # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ---------------------------------------------------------------------- # Save master dark to file # ---------------------------------------------------------------------- # set reference filename reffile = filenames[lastpos] # construct folder and filename darkmasterfits, tag = spirouConfig.Constants.DARK_FILE_MASTER(p, reffile) darkmasterfitsname = os.path.basename(darkmasterfits) # log writing of file WLOG(p, '', 'Saving master dark to {0}'.format(darkmasterfitsname)) # construct big dark table colnames = ['FILENAME', 'NIGHT', 'MJDATE', 'EXPTIME', 'WEATHER_TEMP', 'CASS_TEMP', 'RELHUMID', 'PVERSION', 'GROUPID'] values = [basenames, nightnames, dark_time, dark_exp, dark_wt_temp, dark_cass_temp, dark_humidity, dark_pp_version, matched_id] darktable = spirouImage.MakeTable(p, colnames, values) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr_ref) # define new keys to add hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # write master dark + dark table to file p = spirouImage.WriteImageTable(p, darkmasterfits, image=master_dark, table=darktable, hdict=hdict) # ---------------------------------------------------------------------- # Move to calibDB and update calibDB # ---------------------------------------------------------------------- if p['QC']: # set dark master key keydb = 'DARKM' # copy dark fits file to the calibDB folder spirouDB.PutCalibFile(p, darkmasterfits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, darkmasterfitsname, hdr_ref) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files, mainfitsdir='reduced') p = spirouStartup.InitialFileSetup(p, calibdb=True) # set up function name main_name = __NAME__ + '.main()' # ------------------------------------------------------------------ # Load first file # ------------------------------------------------------------------ loc = ParamDict() rd = spirouImage.ReadImage(p, p['FITSFILENAME']) loc['DATA'], loc['DATAHDR'], loc['YDIM'], loc['XDIM'] = rd loc.set_sources(['DATA', 'DATAHDR', 'XDIM', 'YDIM'], main_name) # ------------------------------------------------------------------ # Get the wave solution # ------------------------------------------------------------------ masterwavefile = spirouDB.GetDatabaseMasterWave(p) # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # read master wave map wout = spirouImage.GetWaveSolution(p, filename=masterwavefile, return_wavemap=True, quiet=True, return_header=True, fiber=wave_fiber) _, loc['WAVE'], loc['WAVEFILE'], _ = wout loc.set_sources(['WAVE', 'WAVEFILE'], main_name) # get the wave keys loc = spirouImage.GetWaveKeys(p, loc, loc['DATAHDR']) # ------------------------------------------------------------------ # Construct convolution kernels (used in GetMolecularTellLines) # ------------------------------------------------------------------ loc = spirouTelluric.ConstructConvKernel1(p, loc) # ------------------------------------------------------------------ # Get molecular telluric lines # ------------------------------------------------------------------ loc = spirouTelluric.GetMolecularTellLines(p, loc) # if TAPAS FNAME is not None we generated a new file so should add to tellDB if loc['TAPAS_FNAME'] is not None: # add to the telluric database spirouDB.UpdateDatabaseTellConv(p, loc['TAPAS_FNAME'], loc['DATAHDR']) # put file in telluDB spirouDB.PutTelluFile(p, loc['TAPAS_ABSNAME']) # ---------------------------------------------------------------------- # load the expected atmospheric transmission # ---------------------------------------------------------------------- # read filename from telluDB tapas_file_names = spirouDB.GetDatabaseTellConv(p) tapas_file_name = tapas_file_names[-1] # load atmospheric transmission sp_tapas = np.load(tapas_file_name) loc['TAPAS_ALL_SPECIES'] = sp_tapas # extract the water and other line-of-sight optical depths loc['TAPAS_WATER'] = sp_tapas[1, :] loc['TAPAS_OTHERS'] = np.prod(sp_tapas[2:, :], axis=0) loc.set_sources(['TAPAS_ALL_SPECIES', 'TAPAS_WATER', 'TAPAS_OTHERS'], main_name) # ------------------------------------------------------------------ # Get master wave solution map # ------------------------------------------------------------------ # get master wave map masterwavefile = spirouDB.GetDatabaseMasterWave(p) # log process wmsg1 = 'Shifting transmission map on to master wavelength grid' wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile)) WLOG(p, '', [wmsg1, wmsg2]) # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # read master wave map mout = spirouImage.GetWaveSolution(p, filename=masterwavefile, return_wavemap=True, quiet=True, return_header=True, fiber=wave_fiber) masterwavep, masterwave, masterwaveheader, mwsource = mout # get wave acqtimes master_acqtimes = spirouDB.GetTimes(p, masterwaveheader) # ------------------------------------------------------------------ # Loop around the files # ------------------------------------------------------------------ # construct extension tellu_ext = '{0}_{1}.fits' # get current telluric maps from telluDB # tellu_db_data = spirouDB.GetDatabaseTellMap(p, required=False) # tellu_db_files = tellu_db_data[0] # storage for valid output files loc['OUTPUTFILES'] = [] # loop around the files for basefilename in p['ARG_FILE_NAMES']: # ------------------------------------------------------------------ # Get absolute path of filename # ------------------------------------------------------------------ filename = os.path.join(p['ARG_FILE_DIR'], basefilename) # ------------------------------------------------------------------ # Read obj telluric file and correct blaze (per order) # ------------------------------------------------------------------ # get image sp, shdr, _, _ = spirouImage.ReadImage(p, filename) # ------------------------------------------------------------------ # check that file has valid DPRTYPE # ------------------------------------------------------------------ # get FP_FP DPRTYPE p = spirouImage.ReadParam(p, shdr, 'KW_DPRTYPE', 'DPRTYPE', dtype=str) # if dprtype is incorrect skip if p['DPRTYPE'] not in p['ALLOWED_TELLURIC_DPRTYPES']: wmsg1 = 'Skipping file (DPRTYPE incorrect)' wmsg2 = '\t DPRTYPE = {0}'.format(p['DPRTYPE']) WLOG(p, 'warning', [wmsg1, wmsg2]) continue # get blaze p, blaze = spirouImage.ReadBlazeFile(p, shdr) # get the blaze percentile blaze_p = p['MKTELLU_BLAZE_PERCENTILE'] # loop through blaze orders, normalize blaze by its peak amplitude for order_num in range(sp.shape[0]): # normalize the spectrum spo, bzo = sp[order_num], blaze[order_num] sp[order_num] = spo / np.nanpercentile(spo, blaze_p) # normalize the blaze blaze[order_num] = bzo / np.nanpercentile(bzo, blaze_p) # find where the blaze is bad with warnings.catch_warnings(record=True) as _: badblaze = blaze < p['MKTELLU_CUT_BLAZE_NORM'] # set bad blaze to NaN blaze[badblaze] = np.nan # set to NaN values where spectrum is zero zeromask = sp == 0 sp[zeromask] = np.nan # divide spectrum by blaze with warnings.catch_warnings(record=True) as _: sp = sp / blaze # add sp to loc loc['SP'] = sp loc.set_source('SP', main_name) # ---------------------------------------------------------------------- # Get object name, airmass and berv # ---------------------------------------------------------------------- # Get object name loc['OBJNAME'] = spirouImage.GetObjName(p, shdr) # Get the airmass loc['AIRMASS'] = spirouImage.GetAirmass(p, shdr) # Get the Barycentric correction from header p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, shdr) # set sources source = main_name + '+ spirouImage.ReadParams()' loc.set_sources(['OBJNAME', 'AIRMASS'], source) # ------------------------------------------------------------------ # get output transmission filename outfile, tag1 = spirouConfig.Constants.TELLU_TRANS_MAP_FILE( p, filename) outfilename = os.path.basename(outfile) loc['OUTPUTFILES'].append(outfile) # ---------------------------------------------------------------------- # Load template (if available) # ---------------------------------------------------------------------- # read filename from telluDB template_file = spirouDB.GetDatabaseObjTemp(p, loc['OBJNAME'], required=False) # if we don't have a template flag it if template_file is None: loc['FLAG_TEMPLATE'] = False loc['TEMPLATE'] = None # construct progres string pstring = 'No template found.' else: loc['FLAG_TEMPLATE'] = True # load template template, _, _, _ = spirouImage.ReadImage(p, template_file) # add to loc loc['TEMPLATE'] = template # construct progres string template_bfile = os.path.basename(template_file) pstring = 'Using template {0}'.format(template_bfile) # set the source for flag and template loc.set_sources(['FLAG_TEMPLATE', 'TEMPLATE'], main_name) # ------------------------------------------------------------------ # log processing file wmsg = 'Processing file {0}. {1}' WLOG(p, '', [wmsg.format(outfilename, pstring)]) # ------------------------------------------------------------------ # Check that basefile is not in blacklist # ------------------------------------------------------------------ blacklist_check = spirouTelluric.CheckBlackList(loc['OBJNAME']) if blacklist_check: # log black list file found wmsg = 'File {0} is blacklisted (OBJNAME={1}). Skipping' wargs = [basefilename, loc['OBJNAME']] WLOG(p, 'warning', wmsg.format(*wargs)) # skip this file continue # ------------------------------------------------------------------ # deal with applying template to spectrum # ------------------------------------------------------------------ # Requires from loc: # TEMPLATE (None or template loaded from file) # FLAG_TEMPLATE # WAVE # SP # BERV # # Returns: # SP (modified if template was used) # TEMPLATE # WCONV loc = spirouTelluric.ApplyTemplate(p, loc) # ------------------------------------------------------------------ # calcullate telluric absorption (with a sigma clip loop) # ------------------------------------------------------------------ # Requires from loc: # AIRMASS # WAVE # SP # WCONV # Returns: # PASSED [Bool] True or False # SP_OUT # SED_OUT # RECOV_AIRMASS # RECOV_WATER loc = spirouTelluric.CalcTelluAbsorption(p, loc) # calculate tranmission map from sp and sed transmission_map = loc['SP_OUT'] / loc['SED_OUT'] # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # if array is completely NaNs it shouldn't pass if np.sum(np.isfinite(transmission_map)) == 0: fail_msg.append('transmission map is all NaNs') passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append('NaN') qc_names.append('image') qc_logic.append('image is all NaN') # ---------------------------------------------------------------------- # get SNR for each order from header nbo = loc['DATA'].shape[0] snr_order = p['QC_MK_TELLU_SNR_ORDER'] snr = spirouImage.Read1Dkey(p, shdr, p['kw_E2DS_SNR'][0], nbo) # check that SNR is high enough if snr[snr_order] < p['QC_MK_TELLU_SNR_MIN']: fmsg = 'low SNR in order {0}: ({1:.2f} < {2:.2f})' fargs = [snr_order, snr[snr_order], p['QC_MK_TELLU_SNR_MIN']] fail_msg.append(fmsg.format(*fargs)) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(snr[snr_order]) qc_name_str = 'SNR[{0}]'.format(snr_order) qc_names.append(qc_name_str) qc_logic.append('{0} < {1:.2f}'.format(qc_name_str, p['QC_MK_TELLU_SNR_ORDER'])) # ---------------------------------------------------------------------- # check that the file passed the CalcTelluAbsorption sigma clip loop if not loc['PASSED']: fmsg = 'File {0} did not converge on a solution in function: {1}' fargs = [basefilename, 'spirouTelluric.CalcTelluAbsorption()'] fail_msg.append(fmsg.format(*fargs)) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(basefilename) qc_names.append('FILE') qc_logic.append('FILE did not converge') # ---------------------------------------------------------------------- # check that the airmass is not too different from input airmass airmass_diff = np.abs(loc['RECOV_AIRMASS'] - loc['AIRMASS']) fargs = [ loc['RECOV_AIRMASS'], loc['AIRMASS'], p['QC_MKTELLU_AIRMASS_DIFF'] ] if airmass_diff > p['QC_MKTELLU_AIRMASS_DIFF']: fmsg = ('Recovered airmass to de-similar than input airmass.' 'Recovered: {0:.3f}. Input: {1:.3f}. QC limit = {2}') fail_msg.append(fmsg.format(*fargs)) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(airmass_diff) qc_names.append('airmass_diff') qc_logic.append('airmass_diff > {0:.2f}' ''.format(p['QC_MKTELLU_AIRMASS_DIFF'])) # ---------------------------------------------------------------------- # check that the water vapor is within limits water_cond1 = loc['RECOV_WATER'] < p['MKTELLU_TRANS_MIN_WATERCOL'] water_cond2 = loc['RECOV_WATER'] > p['MKTELLU_TRANS_MAX_WATERCOL'] fargs = [ p['MKTELLU_TRANS_MIN_WATERCOL'], p['MKTELLU_TRANS_MAX_WATERCOL'] ] if water_cond1 or water_cond2: fmsg = ('Recovered water vapor optical depth not between {0:.3f} ' 'and {1:.3f}') fail_msg.append(fmsg.format(*fargs)) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(loc['RECOV_WATER']) qc_names.append('RECOV_WATER') qc_logic.append('RECOV_WATER not between {0:.3f} and {1:.3f}' ''.format(*fargs)) # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') continue # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Save transmission map to file # ------------------------------------------------------------------ # get raw file name raw_in_file = os.path.basename(p['FITSFILENAME']) # copy original keys hdict = spirouImage.CopyOriginalKeys(loc['DATAHDR']) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=os.path.basename(masterwavefile)) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=mwsource) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # add wave solution date hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME1'], value=master_acqtimes[0]) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME2'], value=master_acqtimes[1]) # add wave solution number of orders hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_ORD_N'], value=masterwavep.shape[0]) # add wave solution degree of fit hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_LL_DEG'], value=masterwavep.shape[1] - 1) # add wave solution coefficients hdict = spirouImage.AddKey2DList(p, hdict, p['KW_WAVE_PARAM'], values=masterwavep) # add telluric keys hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_AIRMASS'], value=loc['RECOV_AIRMASS']) hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_WATER'], value=loc['RECOV_WATER']) # write to file p = spirouImage.WriteImage(p, outfile, transmission_map, hdict) # ------------------------------------------------------------------ # Add transmission map to telluDB # ------------------------------------------------------------------ if p['QC']: # copy tellu file to the telluDB folder spirouDB.PutTelluFile(p, outfile) # update the master tellu DB file with transmission map targs = [ p, outfilename, loc['OBJNAME'], loc['RECOV_AIRMASS'], loc['RECOV_WATER'] ] spirouDB.UpdateDatabaseTellMap(*targs) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, reffile=None): """ cal_DRIFT_E2DS_spirou.py main function, if arguments are None uses arguments from run time i.e.: cal_DRIFT_E2DS_spirou.py [night_directory] [reffile] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param reffile: string, the reference file to use :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with reference file being None (i.e. get from sys.argv) if reffile is None: customargs = spirouStartup.GetCustomFromRuntime( p, [0], [str], ['reffile']) else: customargs = dict(reffile=reffile) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='reffile', mainfitsdir='reduced') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, reffilename = spirouStartup.SingleFileSetup(p, filename=p['REFFILE']) p['REFFILENAME'] = reffilename p.set_source('REFFILENAME', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data speref, hdr, nbo, nx = spirouImage.ReadData(p, reffilename) # add to loc loc = ParamDict() loc['SPEREF'] = speref loc['NUMBER_ORDERS'] = nbo loc.set_sources(['speref', 'number_orders'], __NAME__ + '/main()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # get acquisition time p = spirouImage.GetAcqTime(p, hdr, name='acqtime', kind='julian') bjdref = p['ACQTIME'] # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # manually set OBJNAME to FP p['OBJNAME'] = 'FP' # ---------------------------------------------------------------------- # Earth Velocity calculation # ---------------------------------------------------------------------- if p['IC_IMAGE_TYPE'] == 'H4RG': p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, hdr) # ---------------------------------------------------------------------- # Read wavelength solution # ---------------------------------------------------------------------- # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # # get wave image # wout = spirouImage.GetWaveSolution(p, hdr=hdr, fiber=wave_fiber, # return_wavemap=True) # _, loc['WAVE'] = wout # loc.set_source('WAVE', __NAME__+'/main() + /spirouImage.GetWaveSolution') # get wave image wout = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True, return_filename=True, fiber=wave_fiber) param_ll, wave_ll, wavefile, wsource = wout # save to storage loc['PARAM_LL'], loc['WAVE_LL'], loc['WAVEFILE'], loc['WSOURCE'] = wout source = __NAME__ + '/main() + spirouTHORCA.GetWaveSolution()' loc.set_sources(['WAVE_LL', 'PARAM_LL', 'WAVEFILE', 'WSOURCE'], source) # ---------------------------------------------------------------------- # Read Flat file # ---------------------------------------------------------------------- # get flat p, loc['FLAT'] = spirouImage.ReadFlatFile(p, hdr) loc.set_source('FLAT', __NAME__ + '/main() + /spirouImage.ReadFlatFile') # get all values in flat that are zero to 1 loc['FLAT'] = np.where(loc['FLAT'] == 0, 1.0, loc['FLAT']) # ---------------------------------------------------------------------- # Background correction # ---------------------------------------------------------------------- # log that we are performing background correction if p['IC_DRIFT_BACK_CORR']: WLOG(p, '', 'Perform background correction') # get the box size from constants bsize = p['DRIFT_PEAK_MINMAX_BOXSIZE'] # Loop around the orders for order_num in range(loc['NUMBER_ORDERS']): miny, maxy = spirouBACK.MeasureMinMax(loc['SPEREF'][order_num], bsize) loc['SPEREF'][order_num] = loc['SPEREF'][order_num] - miny # ---------------------------------------------------------------------- # Preliminary set up = no flat, no blaze # ---------------------------------------------------------------------- # reset flat to all ones # loc['FLAT'] = np.ones((nbo, nx)) # set blaze to all ones (if not bug in correlbin !!! # TODO Check why Blaze makes bugs in correlbin loc['BLAZE'] = np.ones((nbo, nx)) # set sources # loc.set_sources(['flat', 'blaze'], __NAME__ + '/main()') loc.set_sources(['blaze'], __NAME__ + '/main()') # ------------------------------------------------------------------ # Compute photon noise uncertainty for reference file # ------------------------------------------------------------------ # set up the arguments for DeltaVrms2D dargs = [loc['SPEREF'], loc['WAVE_LL']] dkwargs = dict(sigdet=p['IC_DRIFT_NOISE'], size=p['IC_DRIFT_BOXSIZE'], threshold=p['IC_DRIFT_MAXFLUX']) # run DeltaVrms2D dvrmsref, wmeanref = spirouRV.DeltaVrms2D(*dargs, **dkwargs) # save to loc loc['DVRMSREF'], loc['WMEANREF'] = dvrmsref, wmeanref loc.set_sources(['dvrmsref', 'wmeanref'], __NAME__ + '/main()()') # log the estimated RV uncertainty wmsg = 'On fiber {0} estimated RV uncertainty on spectrum is {1:.3f} m/s' WLOG(p, 'info', wmsg.format(p['FIBER'], wmeanref)) # ------------------------------------------------------------------ # Reference plots # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot FP spectral order # sPlt.drift_plot_selected_wave_ref(p, loc) # plot photon noise uncertainty sPlt.drift_plot_photon_uncertainty(p, loc) # ---------------------------------------------------------------------- # Get template RV (from ccf_mask) # ---------------------------------------------------------------------- # Use CCF Mask function with drift constants p['CCF_MASK'] = p['DRIFT_CCF_MASK'] p['TARGET_RV'] = p['DRIFT_TARGET_RV'] p['CCF_WIDTH'] = p['DRIFT_CCF_WIDTH'] p['CCF_STEP'] = p['DRIFT_CCF_STEP'] # get the CCF mask from file (check location of mask) loc = spirouRV.GetCCFMask(p, loc) # check and deal with mask in microns (should be in nm) if np.mean(loc['LL_MASK_CTR']) < 2.0: loc['LL_MASK_CTR'] *= 1000.0 loc['LL_MASK_D'] *= 1000.0 # ---------------------------------------------------------------------- # Do correlation # ---------------------------------------------------------------------- # calculate and fit the CCF loc['E2DSFF'] = np.array(loc['SPEREF']) loc.set_source('E2DSFF', __NAME__ + '/main()') p['CCF_FIT_TYPE'] = 1 # run the RV coravelation function with these parameters loc = spirouRV.Coravelation(p, loc) # ---------------------------------------------------------------------- # Correlation stats # ---------------------------------------------------------------------- # get the maximum number of orders to use nbmax = p['CCF_NUM_ORDERS_MAX'] # get the average ccf loc['AVERAGE_CCF'] = np.nansum(loc['CCF'][:nbmax], axis=0) # normalize the average ccf normalized_ccf = loc['AVERAGE_CCF'] / np.nanmax(loc['AVERAGE_CCF']) # get the fit for the normalized average ccf ccf_res, ccf_fit = spirouRV.FitCCF(p, loc['RV_CCF'], normalized_ccf, fit_type=1) loc['CCF_RES'] = ccf_res loc['CCF_FIT'] = ccf_fit # get the max cpp loc['MAXCPP'] = np.nansum(loc['CCF_MAX']) / np.nansum( loc['PIX_PASSED_ALL']) # get the RV value from the normalised average ccf fit center location loc['RV'] = float(ccf_res[1]) # get the contrast (ccf fit amplitude) loc['CONTRAST'] = np.abs(100 * ccf_res[0]) # get the FWHM value loc['FWHM'] = ccf_res[2] * spirouCore.spirouMath.fwhm() # ---------------------------------------------------------------------- # set the source keys = [ 'average_ccf', 'maxcpp', 'rv', 'contrast', 'fwhm', 'ccf_res', 'ccf_fit' ] loc.set_sources(keys, __NAME__ + '/main()') # ---------------------------------------------------------------------- # log the stats wmsg = ('Correlation: C={0:.1f}[%] RV={1:.5f}[km/s] ' 'FWHM={2:.4f}[km/s] maxcpp={3:.1f}') wargs = [loc['CONTRAST'], loc['RV'], loc['FWHM'], loc['MAXCPP']] WLOG(p, 'info', wmsg.format(*wargs)) # get the reference RV in m/s rvref = loc['RV'] * 1000. # ---------------------------------------------------------------------- # rv ccf plot # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # Plot rv vs ccf (and rv vs ccf_fit) sPlt.ccf_rv_ccf_plot(p, loc['RV_CCF'], normalized_ccf, ccf_fit) # ------------------------------------------------------------------ # Get all other files that match kw_OUTPUT and kw_EXT_TYPE from # ref file # ------------------------------------------------------------------ # get files listfiles, listtypes = spirouImage.GetSimilarDriftFiles(p, hdr) # get the number of files nfiles = len(listfiles) # Log the number of files found wmsgs = [ 'Number of files found on directory = {0}'.format(nfiles), '\tExtensions allowed:' ] for listtype in listtypes: wmsgs.append('\t\t - {0}'.format(listtype)) WLOG(p, 'info', wmsgs) # ------------------------------------------------------------------ # Set up Extract storage for all files # ------------------------------------------------------------------ # decide whether we need to skip (for large number of files) if len(listfiles) >= p['DRIFT_NLARGE']: skip = p['DRIFT_E2DS_FILE_SKIP'] nfiles = int(nfiles / skip) else: skip = 1 # set up storage loc['MDRIFT'] = np.zeros(nfiles) loc['MERRDRIFT'] = np.zeros(nfiles) loc['DELTATIME'] = np.zeros(nfiles) loc['FLUXRATIO'] = np.zeros(nfiles) # set loc sources keys = ['mdrift', 'merrdrift', 'deltatime'] loc.set_sources(keys, __NAME__ + '/main()()') # ------------------------------------------------------------------ # Loop around all files: correct for dark, reshape, extract and # calculate dvrms and meanpond # ------------------------------------------------------------------ wref = 1 for i_it in range(nfiles): # get file for this iteration fpfile = listfiles[::skip][i_it] # Log the file we are reading wmsg = 'Reading file {0}' WLOG(p, '', wmsg.format(os.path.split(fpfile)[-1])) # ------------------------------------------------------------------ # read e2ds files and get timestamp # ------------------------------------------------------------------ # read data rout = spirouImage.ReadData(p, filename=fpfile, log=False) loc['SPE'], hdri, nxi, nyi = rout # get acqtime bjdspe = spirouImage.GetAcqTime(p, hdri, name='acqtime', return_value=1, kind='julian') # test whether we want to subtract background if p['IC_DRIFT_BACK_CORR']: # Loop around the orders for order_num in range(loc['NUMBER_ORDERS']): # get the box size from constants bsize = p['DRIFT_PEAK_MINMAX_BOXSIZE'] # Measurethe min and max flux miny, maxy = spirouBACK.MeasureMinMax(loc['SPE'][order_num], bsize) # subtract off the background (miny) loc['SPE'][order_num] = loc['SPE'][order_num] - miny # ------------------------------------------------------------------ # calculate flux ratio # ------------------------------------------------------------------ sorder = p['IC_DRIFT_ORDER_PLOT'] fratio = np.nansum(loc['SPE'][sorder]) / np.nansum( loc['SPEREF'][sorder]) loc['FLUXRATIO'][i_it] = fratio # ------------------------------------------------------------------ # Compute photon noise uncertainty for reference file # ------------------------------------------------------------------ # set up the arguments for DeltaVrms2D dargs = [loc['SPE'], loc['WAVE_LL']] dkwargs = dict(sigdet=p['IC_DRIFT_NOISE'], size=p['IC_DRIFT_BOXSIZE'], threshold=p['IC_DRIFT_MAXFLUX']) # run DeltaVrms2D dvrmsspe, wmeanspe = spirouRV.DeltaVrms2D(*dargs, **dkwargs) # ---------------------------------------------------------------------- # Do correlation # ---------------------------------------------------------------------- # calculate and fit the CCF loc['E2DSFF'] = loc['SPE'] * 1. loc.set_source('E2DSFF', __NAME__ + '/main()') loc = spirouRV.Coravelation(p, loc) # ---------------------------------------------------------------------- # Correlation stats # ---------------------------------------------------------------------- # get the maximum number of orders to use nbmax = p['CCF_NUM_ORDERS_MAX'] # get the average ccf loc['AVERAGE_CCF'] = np.nansum(loc['CCF'][:nbmax], axis=0) # normalize the average ccf normalized_ccf = loc['AVERAGE_CCF'] / np.nanmax(loc['AVERAGE_CCF']) # get the fit for the normalized average ccf ccf_res, ccf_fit = spirouRV.FitCCF(p, loc['RV_CCF'], normalized_ccf, fit_type=1) # calculate the mean RV meanrv = ccf_res[1] * 1000. - rvref # ------------------------------------------------------------------ # Calculate delta time # ------------------------------------------------------------------ # calculate the time from reference (in hours) deltatime = (bjdspe - bjdref) * 24 err_meanrv = np.sqrt(dvrmsref + dvrmsspe) merr = 1. / np.sqrt(np.nansum((1. / err_meanrv)**2)) # Log the RV properties wmsg = ('Time from ref= {0:.2f} h ' '- Flux Ratio= {1:.2f} ' '- Drift mean= {2:.2f} +- ' '{3:.2f} m/s') wargs = [deltatime, loc['FLUXRATIO'][i_it], meanrv, merr] WLOG(p, '', wmsg.format(*wargs)) # add this iteration to storage loc['MDRIFT'][i_it] = meanrv loc['MERRDRIFT'][i_it] = merr loc['DELTATIME'][i_it] = deltatime # ------------------------------------------------------------------ # set source loc.set_sources(['mdrift', 'merrdrift'], __NAME__ + '/main()()') # ------------------------------------------------------------------ # peak to peak drift driftptp = np.max(loc['MDRIFT']) - np.min(loc['MDRIFT']) driftrms = np.std(loc['MDRIFT']) # log th etotal drift peak-to-peak and rms wmsg = ('Total drift Peak-to-Peak={0:.3f} m/s RMS={1:.3f} m/s in ' '{2:.2f} hour') wargs = [driftptp, driftrms, np.max(loc['DELTATIME'])] WLOG(p, '', wmsg.format(*wargs)) # ------------------------------------------------------------------ # Plot of mean drift # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot delta time against median drift sPlt.drift_plot_dtime_against_mdrift(p, loc, kind='e2ds') # ------------------------------------------------------------------ # Save drift values to file # ------------------------------------------------------------------ # # get raw input file name # raw_infile = os.path.basename(p['REFFILE']) # # construct filename # driftfits, tag = spirouConfig.Constants.DRIFTCCF_E2DS_FITS_FILE(p) # driftfitsname = os.path.split(driftfits)[-1] # # log that we are saving drift values # wmsg = 'Saving drift values of Fiber {0} in {1}' # WLOG(p, '', wmsg.format(p['FIBER'], driftfitsname)) # # add keys from original header file # hdict = spirouImage.CopyOriginalKeys(hdr) # # add the reference RV # hdict = spirouImage.AddKey(p, hdict, p['KW_REF_RV'], value=rvref) # # # set the version # hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) # hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # # set the input files # hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFLAT'], value=p['FLATFILE']) # hdict = spirouImage.AddKey(p, hdict, p['KW_REFFILE'], value=raw_infile) # # save drift values # p = spirouImage.WriteImage(p, driftfits, loc['DRIFT'], hdict) # ------------------------------------------------------------------ # print .tbl result # ------------------------------------------------------------------ # construct filename drifttbl = spirouConfig.Constants.DRIFTCCF_E2DS_TBL_FILE(p) drifttblname = os.path.split(drifttbl)[-1] # construct and write table columnnames = ['time', 'drift', 'drifterr'] columnformats = ['7.4f', '6.2f', '6.3f'] columnvalues = [loc['DELTATIME'], loc['MDRIFT'], loc['MERRDRIFT']] table = spirouImage.MakeTable(p, columns=columnnames, values=columnvalues, formats=columnformats) # write table wmsg = 'Average Drift saved in {0} Saved ' WLOG(p, '', wmsg.format(drifttblname)) spirouImage.WriteTable(p, table, drifttbl, fmt='ascii.rst') # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files, mainfitsdir='reduced') p = spirouStartup.InitialFileSetup(p, calibdb=True) # set up function name main_name = __NAME__ + '.main()' # ------------------------------------------------------------------ # Load first file # ------------------------------------------------------------------ loc = ParamDict() rd = spirouImage.ReadImage(p, p['FITSFILENAME']) loc['DATA'], loc['DATAHDR'], loc['YDIM'], loc['XDIM'] = rd loc.set_sources(['DATA', 'DATAHDR', 'XDIM', 'YDIM'], main_name) # ------------------------------------------------------------------ # Get the wave solution # ------------------------------------------------------------------ wout = spirouImage.GetWaveSolution(p, image=loc['DATA'], hdr=loc['DATAHDR'], return_wavemap=True, return_filename=True) _, loc['WAVE'], loc['WAVEFILE'], _ = wout loc.set_sources(['WAVE', 'WAVEFILE'], main_name) # get the wave keys loc = spirouImage.GetWaveKeys(p, loc, loc['DATAHDR']) # ------------------------------------------------------------------ # Get and Normalise the blaze # ------------------------------------------------------------------ p, loc = spirouTelluric.GetNormalizedBlaze(p, loc, loc['DATAHDR']) # ------------------------------------------------------------------ # Construct convolution kernels # ------------------------------------------------------------------ loc = spirouTelluric.ConstructConvKernel1(p, loc) loc = spirouTelluric.ConstructConvKernel2(p, loc, vsini=p['TELLU_VSINI']) # ------------------------------------------------------------------ # Get molecular telluric lines # ------------------------------------------------------------------ loc = spirouTelluric.GetMolecularTellLines(p, loc) # if TAPAS FNAME is not None we generated a new file so should add to tellDB if loc['TAPAS_FNAME'] is not None: # add to the telluric database spirouDB.UpdateDatabaseTellConv(p, loc['TAPAS_FNAME'], loc['DATAHDR']) # put file in telluDB spirouDB.PutTelluFile(p, loc['TAPAS_ABSNAME']) # ------------------------------------------------------------------ # Get master wave solution map # ------------------------------------------------------------------ # get master wave map masterwavefile = spirouDB.GetDatabaseMasterWave(p) # log process wmsg1 = 'Shifting transmission map on to master wavelength grid' wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile)) WLOG(p, '', [wmsg1, wmsg2]) # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # read master wave map mout = spirouImage.GetWaveSolution(p, filename=masterwavefile, return_wavemap=True, quiet=True, return_header=True, fiber=wave_fiber) masterwavep, masterwave, masterwaveheader, mwsource = mout # get wave acqtimes master_acqtimes = spirouDB.GetTimes(p, masterwaveheader) # ------------------------------------------------------------------ # Loop around the files # ------------------------------------------------------------------ # construct extension tellu_ext = '{0}_{1}.fits' # get current telluric maps from telluDB tellu_db_data = spirouDB.GetDatabaseTellMap(p, required=False) tellu_db_files = tellu_db_data[0] # storage for valid output files loc['OUTPUTFILES'] = [] # loop around the files for basefilename in p['ARG_FILE_NAMES']: # ------------------------------------------------------------------ # Get absolute path of filename # ------------------------------------------------------------------ filename = os.path.join(p['ARG_FILE_DIR'], basefilename) # ------------------------------------------------------------------ # Read obj telluric file and correct for blaze # ------------------------------------------------------------------ # get image sp, shdr, _, _ = spirouImage.ReadImage(p, filename) # divide my blaze sp = sp / loc['BLAZE'] # ------------------------------------------------------------------ # Get the wave solution # ------------------------------------------------------------------ wout = spirouImage.GetWaveSolution(p, image=sp, hdr=shdr, return_wavemap=True, return_filename=True) _, loc['WAVE_IT'], loc['WAVEFILE_IT'], _ = wout loc.set_sources(['WAVE_IT', 'WAVEFILE_IT'], main_name) # ------------------------------------------------------------------ # Shift data to master wave file # ------------------------------------------------------------------ # shift map wargs = [p, sp, loc['WAVE_IT'], masterwave] sp = spirouTelluric.Wave2Wave(*wargs) loc['SP'] = np.array(sp) loc.set_source('SP', main_name) # ------------------------------------------------------------------ # get output transmission filename outfile, tag1 = spirouConfig.Constants.TELLU_TRANS_MAP_FILE(p, filename) outfilename = os.path.basename(outfile) loc['OUTPUTFILES'].append(outfile) # if we already have the file skip it if outfile in tellu_db_files: wmsg = 'File {0} exists in telluDB, skipping' WLOG(p, '', wmsg.format(outfilename)) continue else: # log processing file wmsg = 'Processing file {0}' WLOG(p, '', wmsg.format(outfilename)) # Get object name and airmass loc['OBJNAME'] = spirouImage.GetObjName(p, shdr) loc['AIRMASS'] = spirouImage.GetAirmass(p, shdr) # set source source = main_name + '+ spirouImage.ReadParams()' loc.set_sources(['OBJNAME', 'AIRMASS'], source) # ------------------------------------------------------------------ # Check that basefile is not in blacklist # ------------------------------------------------------------------ blacklist_check = spirouTelluric.CheckBlackList(loc['OBJNAME']) if blacklist_check: # log black list file found wmsg = 'File {0} is blacklisted (OBJNAME={1}). Skipping' wargs = [basefilename, loc['OBJNAME']] WLOG(p, 'warning', wmsg.format(*wargs)) # skip this file continue # ------------------------------------------------------------------ # loop around the orders # ------------------------------------------------------------------ # define storage for the transmission map transmission_map = np.zeros_like(loc['DATA']) # define storage for measured rms within expected clean domains exp_clean_rms = np.zeros(loc['DATA'].shape[0]) # loop around the orders for order_num in range(loc['DATA'].shape[0]): # start and end start = order_num * loc['XDIM'] end = (order_num * loc['XDIM']) + loc['XDIM'] # get this orders combined tapas transmission trans = loc['TAPAS_ALL_SPECIES'][0, start:end] # keep track of the pixels that are considered valid for the SED # determination mask1 = trans > p['TRANSMISSION_CUT'] mask1 &= np.isfinite(loc['NBLAZE'][order_num, :]) # normalise the spectrum sp[order_num, :] /= np.nanmedian(sp[order_num, :]) # create a float mask fmask = np.array(mask1, dtype=float) # set up an SED to fill sed = np.ones(loc['XDIM']) # sigma clip until limit ww = None for it in range(p['N_ITER_SED_HOTSTAR']): # copy the spectrum sp2 = np.array(sp[order_num, :]) # flag Nans nanmask = ~np.isfinite(sp2) # set all NaNs to zero so that it does not propagate when # we convlve by KER2 - must set sp2[bad] to zero as # NaN * 0.0 = NaN and we want 0.0! sp2[nanmask] = 0.0 # trace the invalid points fmask[nanmask] = 0.0 # multiple by the float mask sp2 *= fmask # convolve with the second kernel sp2b = np.convolve(sp2 / sed, loc['KER2'], mode='same') # convolve with mask to get weights ww = np.convolve(fmask, loc['KER2'], mode='same') # normalise the spectrum by the weights with warnings.catch_warnings(record=True) as w: sp2bw = sp2b / ww # set zero pixels to 1 sp2bw[sp2b == 0] = 1 # recalculate the mask using the deviation from original with warnings.catch_warnings(record=True) as _: dev = (sp2bw - sp[order_num, :] / sed) dev /= np.nanmedian(np.abs(dev)) mask = mask1 * (np.abs(dev) < p['TELLU_SIGMA_DEV']) # update the SED with the corrected spectrum sed *= sp2bw # identify bad pixels with warnings.catch_warnings(record=True) as _: bad = (sp[order_num, :] / sed[:] > 1.2) sed[bad] = np.nan # debug plot if p['DRS_PLOT'] and (p['DRS_DEBUG'] > 1) and FORCE_PLOT_ON: # start non-interactive plot sPlt.plt.ioff() # plot the transmission map plot pargs = [order_num, mask1, sed, trans, sp, ww, outfilename] sPlt.tellu_trans_map_plot(p, loc, *pargs) # show and close sPlt.plt.show() sPlt.plt.close() # set all values below a threshold to NaN sed[ww < p['TELLU_NAN_THRESHOLD']] = np.nan # save the spectrum (normalised by the SED) to the tranmission map transmission_map[order_num, :] = sp[order_num, :] / sed # get expected clean rms fmaskb = np.array(fmask).astype(bool) with warnings.catch_warnings(record=True): zerotrans = np.abs(transmission_map[order_num, fmaskb]-1) ec_rms = np.nanmedian(zerotrans) exp_clean_rms[order_num] = ec_rms # log the rms wmsg = 'Order {0}: Fractional RMS in telluric free domain = {1:.3f}' wargs = [order_num, ec_rms] WLOG(p, '', wmsg.format(*wargs)) # --------------------------------------------------------------------- # Quality control # --------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # get SNR for each order from header nbo = loc['DATA'].shape[0] snr_order = p['QC_MK_TELLU_SNR_ORDER'] snr = spirouImage.Read1Dkey(p, shdr, p['kw_E2DS_SNR'][0], nbo) # check that SNR is high enough if snr[snr_order] < p['QC_MK_TELLU_SNR_MIN']: fmsg = 'low SNR in order {0}: ({1:.2f} < {2:.2f})' fargs = [snr_order, snr[snr_order], p['QC_MK_TELLU_SNR_MIN']] fail_msg.append(fmsg.format(*fargs)) passed = False # add to qc header lists qc_values.append(snr[snr_order]) qc_name_str = 'SNR[{0}]'.format(snr_order) qc_names.append(qc_name_str) qc_logic.append('{0} < {1:.2f}'.format(qc_name_str, p['QC_MK_TELLU_SNR_ORDER'])) qc_pass.append(0) else: qc_pass.append(1) # ---------------------------------------------------------------------- # check that the RMS is not too low if exp_clean_rms[snr_order] > p['QC_TELLU_CLEAN_RMS_MAX']: fmsg = ('Expected clean RMS is too high in order {0} ' '({1:.3f} > {2:.3f})') fargs = [snr_order, exp_clean_rms[snr_order], p['QC_TELLU_CLEAN_RMS_MAX']] fail_msg.append(fmsg.format(*fargs)) passed = False # add to qc header lists qc_values.append(exp_clean_rms[snr_order]) qc_name_str = 'exp_clean_rms[{0}]'.format(snr_order) qc_names.append(qc_name_str) qc_logic.append('{0} > {1:.2f}'.format(qc_name_str, p['QC_TELLU_CLEAN_RMS_MAX'])) qc_pass.append(0) else: qc_pass.append(1) # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Save transmission map to file # ------------------------------------------------------------------ # get raw file name raw_in_file = os.path.basename(p['FITSFILENAME']) # copy original keys hdict = spirouImage.CopyOriginalKeys(loc['DATAHDR']) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=os.path.basename(masterwavefile)) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=mwsource) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # add wave solution date hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME1'], value=master_acqtimes[0]) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME2'], value=master_acqtimes[1]) # add wave solution number of orders hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_ORD_N'], value=masterwavep.shape[0]) # add wave solution degree of fit hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_LL_DEG'], value=masterwavep.shape[1] - 1) # add wave solution coefficients hdict = spirouImage.AddKey2DList(p, hdict, p['KW_WAVE_PARAM'], values=masterwavep) # write to file p = spirouImage.WriteImage(p, outfile, transmission_map, hdict) # ------------------------------------------------------------------ # Generate the absorption map # ------------------------------------------------------------------ # set up storage for the absorption abso = np.array(transmission_map) # set values less than low threshold to low threshold # set values higher than high threshold to 1 low, high = p['TELLU_ABSO_LOW_THRES'], p['TELLU_ABSO_HIGH_THRES'] with warnings.catch_warnings(record=True) as w: abso[abso < low] = low abso[abso > high] = 1.0 # write to loc loc['RECON_ABSO'] = abso.reshape(np.product(loc['DATA'].shape)) loc.set_source('RECON_ABSO', main_name) # ------------------------------------------------------------------ # Get molecular absorption # ------------------------------------------------------------------ loc = spirouTelluric.CalcMolecularAbsorption(p, loc) # add molecular absorption to file for it, molecule in enumerate(p['TELLU_ABSORBERS'][1:]): # get molecule keyword store and key molkey = '{0}_{1}'.format(p['KW_TELLU_ABSO'][0], molecule.upper()) # add water col if molecule == 'h2o': loc['WATERCOL'] = loc[molkey] # set source loc.set_source('WATERCOL', main_name) # ------------------------------------------------------------------ # Add transmission map to telluDB # ------------------------------------------------------------------ if p['QC']: # copy tellu file to the telluDB folder spirouDB.PutTelluFile(p, outfile) # update the master tellu DB file with transmission map targs = [p, outfilename, loc['OBJNAME'], loc['AIRMASS'], loc['WATERCOL']] spirouDB.UpdateDatabaseTellMap(*targs) # ---------------------------------------------------------------------- # Optional Absorption maps # ---------------------------------------------------------------------- if p['TELLU_ABSO_MAPS']: # ------------------------------------------------------------------ # Generate the absorption map # ------------------------------------------------------------------ # get number of files nfiles = len(p['OUTPUTFILES']) # set up storage for the absorption abso = np.zeros([nfiles, np.product(loc['DATA'].shape)]) # loop around outputfiles and add them to abso for it, filename in enumerate(p['OUTPUTFILES']): # push data into array data_it, _, _, _ = spirouImage.ReadImage(p, filename) abso[it, :] = data_it.reshape(np.product(loc['DATA'].shape)) # set values less than low threshold to low threshold # set values higher than high threshold to 1 low, high = p['TELLU_ABSO_LOW_THRES'], p['TELLU_ABSO_HIGH_THRES'] abso[abso < low] = low abso[abso > high] = 1.0 # set values less than TELLU_CUT_BLAZE_NORM threshold to NaN abso[loc['NBLAZE'] < p['TELLU_CUT_BLAZE_NORM']] = np.nan # reshape data (back to E2DS) abso_e2ds = abso.reshape(nfiles, loc['YDIM'], loc['XDIM']) # get file name abso_map_file, tag2 = spirouConfig.Constants.TELLU_ABSO_MAP_FILE(p) # get raw file name raw_in_file = os.path.basename(p['FITSFILENAME']) # write the map to file hdict = spirouImage.CopyOriginalKeys(loc['DATAHDR']) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) # write to file p = spirouImage.WriteImage(p, abso_map_file, abso_e2ds, hdict) # ------------------------------------------------------------------ # Generate the median and normalized absorption maps # ------------------------------------------------------------------ # copy the absorption cube abso2 = np.array(abso) # log the absorption cube log_abso = np.log(abso) # get the threshold from p threshold = p['TELLU_ABSO_SIG_THRESH'] # calculate the abso_med abso_med = np.nanmedian(log_abso, axis=0) # sigma clip around the median for it in range(p['TELLU_ABSO_SIG_N_ITER']): # recalculate the abso_med abso_med = np.nanmedian(log_abso, axis=0) # loop around each file for jt in range(nfiles): # get this iterations row rowvalue = log_abso[jt, :] # get the mask of those values above threshold goodpix = (rowvalue > threshold) & (abso_med > threshold) # apply the mask of good pixels to work out ratio part1 = np.nansum(rowvalue[goodpix] * abso_med[goodpix]) part2 = np.nansum(abso_med[goodpix] ** 2) ratio = part1 / part2 # store normalised absol back on to log_abso log_abso[jt, :] = log_abso[jt, :] / ratio # unlog log_abso abso_out = np.exp(log_abso) # calculate the median of the log_abso abso_med_out = np.exp(np.nanmedian(log_abso, axis=0)) # reshape the median abso_map_n = abso_med_out.reshape(loc['DATA'].shape) # save the median absorption map to file abso_med_file, tag3 = spirouConfig.Constants.TELLU_ABSO_MEDIAN_FILE(p) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3) p = spirouImage.WriteImage(p, abso_med_file, abso_med_out, hdict) # save the normalized absorption map to file abso_map_file, tag4 = spirouConfig.Constants.TELLU_ABSO_NORM_MAP_FILE(p) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) p = spirouImage.WriteImage(p, abso_map_file, abso_map_n, hdict) # ------------------------------------------------------------------ # calculate dv statistic # ------------------------------------------------------------------ # get the order for dv calculation dvselect = p['TELLU_ABSO_DV_ORDER'] size = p['TELLU_ABSO_DV_SIZE'] threshold2 = p['TELLU_ABSO_DV_GOOD_THRES'] fitdeg = p['TELLU_ABSO_FIT_DEG'] ydim, xdim = loc['DATA'].shape # get the start and end points of this order start, end = xdim * dvselect + size, xdim * dvselect - size + xdim # get the median for selected order abso_med2 = np.exp(abso_med[start:end]) # get the dv pixels to extract dvpixels = np.arange(-np.floor(size / 2), np.ceil(size / 2), 1) # loop around files for it, filename in enumerate(p['OUTPUTFILES']): # storage for the extracted abso ratios for this file cc = np.zeros(size) # loop around a box of size="size" for jt, dv in dvpixels: # get the start and end position start = xdim * dvselect + size + dv end = xdim * dvselect + xdim - size + dv # get the log abso for this iteration rowvalue = np.exp(log_abso[it, start:end]) # find the good pixels goodpix = (rowvalue > threshold2) & (abso_med2 > threshold2) # get the ratio part1 = np.nansum(rowvalue[goodpix] * abso_med2[goodpix]) part2 = np.nansum(abso_med2[goodpix] ** 2) cc[jt] = part1 / part2 # fit the ratio across the points cfit = nanpolyfit(dvpixels, cc, fitdeg) # work out the dv pix dvpix = -0.5 * (cfit[1] / cfit[0]) # log stats wmsg = 'File: "{0}", dv={1}' WLOG(p, '', wmsg.format(filename, dvpix)) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): """ cal_SLIT_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_SLIT_spirou.py [night_directory] [files] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) p = spirouStartup.InitialFileSetup(p, calibdb=True) # set the fiber type p['FIB_TYP'] = 'AB' p.set_source('FIB_TYP', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add') # ---------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- data = spirouImage.FixNonPreProcess(p, data) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # ---------------------------------------------------------------------- # Correction of DARK # ---------------------------------------------------------------------- p, datac = spirouImage.CorrectForDark(p, data, hdr) # ---------------------------------------------------------------------- # Resize image # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p) # convert NaN to zeros data0 = np.where(~np.isfinite(data), np.zeros_like(data), data) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) data2 = spirouImage.ResizeImage(p, data0, **bkwargs) # log change in data size WLOG(p, '', ('Image format changed to ' '{0}x{1}').format(*data2.shape)) # ---------------------------------------------------------------------- # Correct for the BADPIX mask (set all bad pixels to zero) # ---------------------------------------------------------------------- p, data2 = spirouImage.CorrectForBadPix(p, data2, hdr) # ---------------------------------------------------------------------- # Background computation # ---------------------------------------------------------------------- if p['IC_DO_BKGR_SUBTRACTION']: # log that we are doing background measurement WLOG(p, '', 'Doing background measurement on raw frame') # get the bkgr measurement bargs = [p, data2, hdr] # background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs) p, background = spirouBACK.MeasureBackgroundMap(*bargs) else: background = np.zeros_like(data2) p['BKGRDFILE'] = 'None' p.set_source('BKGRDFILE', __NAME__ + '.main()') # correct data2 with background data2 = data2 - background # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- # get the number of bad pixels n_bad_pix = np.nansum(~np.isfinite(data2)) n_bad_pix_frac = n_bad_pix * 100 / np.product(data2.shape) # Log number wmsg = 'Nb dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- loc = ParamDict() # ---------------------------------------------------------------------- # Loop around fiber types # ---------------------------------------------------------------------- # set fiber p['FIBER'] = p['FIB_TYP'] # ------------------------------------------------------------------ # Get localisation coefficients # ------------------------------------------------------------------ # original there is a loop but it is not used --> removed p = spirouImage.FiberParams(p, p['FIBER'], merge=True) # get localisation fit coefficients p, loc = spirouLOCOR.GetCoeffs(p, hdr, loc) # ------------------------------------------------------------------ # Calculating the tilt # ------------------------------------------------------------------ # get the tilt by extracting the AB fibers and correlating them loc = spirouImage.GetTilt(p, loc, data2) # fit the tilt with a polynomial loc = spirouImage.FitTilt(p, loc) # log the tilt dispersion wmsg = 'Tilt dispersion = {0:.3f} deg' WLOG(p, 'info', wmsg.format(loc['RMS_TILT'])) # ------------------------------------------------------------------ # Plotting # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # plots setup: start interactive plot sPlt.start_interactive_session(p) # plot image with selected order shown sPlt.slit_sorder_plot(p, loc, data2) # plot slit tilt angle and fit sPlt.slit_tilt_angle_and_fit_plot(p, loc) # end interactive section sPlt.end_interactive_session(p) # ------------------------------------------------------------------ # Replace tilt by the global fit # ------------------------------------------------------------------ loc['TILT'] = loc['YFIT_TILT'] oldsource = loc.get_source('tilt') loc.set_source('TILT', oldsource + '+{0}/main()'.format(__NAME__)) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # check that tilt rms is below required if loc['RMS_TILT'] > p['QC_SLIT_RMS']: # add failed message to fail message list fmsg = 'abnormal RMS of SLIT angle ({0:.2f} > {1:.2f} deg)' fail_msg.append(fmsg.format(loc['RMS_TILT'], p['QC_SLIT_RMS'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(loc['RMS_TILT']) qc_names.append('RMS_TILT') qc_logic.append('RMS_TILT > {0:.2f}'.format(p['QC_SLIT_RMS'])) # ---------------------------------------------------------------------- # check that tilt is less than max tilt required max_tilt = np.max(loc['TILT']) if max_tilt > p['QC_SLIT_MAX']: # add failed message to fail message list fmsg = 'abnormal SLIT angle ({0:.2f} > {1:.2f} deg)' fail_msg.append(fmsg.format(max_tilt, p['QC_SLIT_MAX'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(max_tilt) qc_names.append('max_tilt') qc_logic.append('max_tilt > {0:.2f}'.format(p['QC_SLIT_MAX'])) # ---------------------------------------------------------------------- # check that tilt is greater than min tilt required min_tilt = np.min(loc['TILT']) if min_tilt < p['QC_SLIT_MIN']: # add failed message to fail message list fmsg = 'abnormal SLIT angle ({0:.2f} < {1:.2f} deg)' fail_msg.append(fmsg.format(max_tilt, p['QC_SLIT_MIN'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(min_tilt) qc_names.append('min_tilt') qc_logic.append('min_tilt > {0:.2f}'.format(p['QC_SLIT_MIN'])) # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ---------------------------------------------------------------------- # Save and record of tilt table # ---------------------------------------------------------------------- # copy the tilt along the orders tiltima = np.ones((int(loc['NUMBER_ORDERS']/2), data2.shape[1])) tiltima *= loc['TILT'][:, None] # get the raw tilt file name raw_tilt_file = os.path.basename(p['FITSFILENAME']) # construct file name and path tiltfits, tag = spirouConfig.Constants.SLIT_TILT_FILE(p) tiltfitsname = os.path.basename(tiltfits) # Log that we are saving tilt file wmsg = 'Saving tilt information in file: {0}' WLOG(p, '', wmsg.format(tiltfitsname)) # Copy keys from fits file hdict = spirouImage.CopyOriginalKeys(hdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBACK'], value=p['BKGRDFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # add qc parameters # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # add tilt parameters as 1d list hdict = spirouImage.AddKey1DList(p, hdict, p['KW_TILT'], values=loc['TILT']) # write tilt file to file p = spirouImage.WriteImage(p, tiltfits, tiltima, hdict) # ---------------------------------------------------------------------- # Update the calibration data base # ---------------------------------------------------------------------- if p['QC']: keydb = 'TILT' # copy localisation file to the calibDB folder spirouDB.PutCalibFile(p, tiltfits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, tiltfitsname, hdr) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, e2dsfiles=None): """ cal_CCF_E2DS_spirou.py main function, if arguments are None uses arguments from run time i.e.: cal_CCF_E2DS_spirou.py [night_directory] [E2DSfilename] [mask] [RV] [width] [step] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param e2dsfiles: list of string, the E2DS files to use :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # need custom args (to accept full path or wild card if e2dsfiles is None: names, types = ['e2dsfiles'], [str] customargs = spirouStartup.GetCustomFromRuntime(p, [0], types, names, last_multi=True) else: customargs = dict(e2dsfiles=e2dsfiles) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsdir='reduced') # ---------------------------------------------------------------------- # Process files (including wildcards) # ---------------------------------------------------------------------- try: e2dsfiles = spirouFile.Paths(p['E2DSFILES'], root=p['ARG_FILE_DIR']).abs_paths except PathException as e: WLOG(p, 'error', e) # loop around files for it, e2dsfile in enumerate(e2dsfiles): # get the base file name e2dsfilename = os.path.basename(e2dsfile) # log the file process wargs = [e2dsfilename, it + 1, len(e2dsfiles)] wmsg = ' * Processing file {0} ({1} of {2})'.format(*wargs) WLOG(p, '', spirouStartup.spirouStartup.HEADER) WLOG(p, '', wmsg) WLOG(p, '', spirouStartup.spirouStartup.HEADER) # ------------------------------------------------------------------ # Check that we can process file # ------------------------------------------------------------------ # check if ufile exists if not os.path.exists(e2dsfile): wmsg = 'File {0} does not exist... skipping' WLOG(p, 'warning', wmsg.format(e2dsfilename)) continue elif ('e2ds' not in e2dsfilename) and ('e2dsff' not in e2dsfilename): wmsg = 'File {0} not a valid E2DS or E2DSFF file' WLOG(p, 'warning', wmsg.format(e2dsfilename)) continue elif '.fits' not in e2dsfilename: wmsg = 'File {0} not a fits file... skipping' WLOG(p, 'warning', wmsg.format(e2dsfilename)) continue # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data e2ds, hdr, nbo, nx = spirouImage.ReadData(p, e2dsfile) # add to loc loc = ParamDict() loc['E2DS'] = e2ds loc['NUMBER_ORDERS'] = nbo loc.set_sources(['E2DS', 'number_orders'], __NAME__ + '/main()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # get acquisition time p = spirouImage.GetAcqTime(p, hdr, name='acqtime', kind='julian') # ---------------------------------------------------------------------- # Read star parameters # ---------------------------------------------------------------------- p = spirouImage.ReadParam(p, hdr, 'KW_OBJRA', dtype=str) p = spirouImage.ReadParam(p, hdr, 'KW_OBJDEC', dtype=str) p = spirouImage.ReadParam(p, hdr, 'KW_OBJEQUIN') p = spirouImage.ReadParam(p, hdr, 'KW_OBJRAPM') p = spirouImage.ReadParam(p, hdr, 'KW_OBJDECPM') p = spirouImage.ReadParam(p, hdr, 'KW_DATE_OBS', dtype=str) p = spirouImage.ReadParam(p, hdr, 'KW_UTC_OBS', dtype=str) # ----------------------------------------------------------------------- # Earth Velocity calculation # ----------------------------------------------------------------------- if p['IC_IMAGE_TYPE'] == 'H4RG': loc = spirouImage.EarthVelocityCorrection(p, loc, method=p['CCF_BERVMODE']) else: loc['BERV'], loc['BJD'] = 0.0, 0.0 loc['BERV_MAX'], loc['BERV_SOURCE'] = 0.0, 'None' loc.set_sources(['BERV', 'BJD', 'BERV_MAX'], __NAME__ + '.main()') # ---------------------------------------------------------------------- # archive ccf to fits file # ---------------------------------------------------------------------- outfilename = str(e2dsfile) # add keys hdict = spirouImage.CopyOriginalKeys(hdr) # add berv values hdict = spirouImage.AddKey(p, hdict, p['KW_BERV'], value=loc['BERV']) hdict = spirouImage.AddKey(p, hdict, p['KW_BJD'], value=loc['BJD']) hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_MAX'], value=loc['BERV_MAX']) hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_SOURCE'], value=loc['BERV_SOURCE']) # write image and add header keys (via hdict) p = spirouImage.WriteImage(p, outfilename, e2ds, hdict) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files, mainfitsdir='reduced') p = spirouStartup.InitialFileSetup(p, calibdb=True) # set up function name main_name = __NAME__ + '.main()' # ---------------------------------------------------------------------- # load up first file # ---------------------------------------------------------------------- # construct loc parameter dictionary for storing data loc = ParamDict() # read first file and assign values rdata = spirouImage.ReadImage(p, p['FITSFILENAME']) loc['DATA'], loc['DATAHDR'] = rdata[:2] # Get object name and airmass loc['OBJNAME'] = spirouImage.GetObjName(p, loc['DATAHDR']) loc.set_source('OBJNAME', main_name) loc['AIRMASS'] = spirouImage.GetAirmass(p, loc['DATAHDR']) # set source source = main_name + '+ spirouImage.ReadParams()' loc.set_sources(['OBJNAME', 'AIRMASS'], source) # ---------------------------------------------------------------------- # Get and Normalise the blaze # ---------------------------------------------------------------------- p, loc = spirouTelluric.GetNormalizedBlaze(p, loc, loc['DATAHDR']) # ---------------------------------------------------------------------- # Get database files # ---------------------------------------------------------------------- # get current telluric maps from telluDB tellu_db_data = spirouDB.GetDatabaseTellObj(p, required=False) tellu_db_files, tellu_db_names = tellu_db_data[0], tellu_db_data[1] # sort files by name # sortmask = spirouImage.SortByName(tellu_db_files) # tellu_db_files = np.array(tellu_db_files)[sortmask] # tellu_db_names = np.array(tellu_db_names)[sortmask] # filter by object name (only keep OBJNAME objects) and only keep # unique filenames tell_files, tell_names = [], [] for it in range(len(tellu_db_files)): # check that objname is correct cond1 = loc['OBJNAME'] in tellu_db_names[it] # check that filename is not already used cond2 = tellu_db_files[it] not in tell_files # append to file list if criteria correct if cond1 and cond2: tell_files.append(tellu_db_files[it]) tell_names.append(tellu_db_names[it]) # log if we have no files if len(tell_files) == 0: wmsg = 'No "TELL_OBJ" files found for object ="{0}" skipping' WLOG(p, 'warning', wmsg.format(loc['OBJNAME'])) # End Message wmsg = 'Recipe {0} has been successfully completed' WLOG(p, 'info', wmsg.format(p['PROGRAM'])) # return a copy of locally defined variables in the memory return dict(locals()) else: # log how many found wmsg = 'N={0} "TELL_OBJ" files found for object ="{1}"' WLOG(p, 'info', wmsg.format(len(tell_files), loc['OBJNAME'])) # ---------------------------------------------------------------------- # Set up storage for cubes (NaN arrays) # ---------------------------------------------------------------------- # set up flat size dims = [loc['DATA'].shape[0], loc['DATA'].shape[1], len(tell_files)] flatsize = np.product(dims) # create NaN filled storage big_cube = np.repeat([np.nan], flatsize).reshape(*dims) big_cube0 = np.repeat([np.nan], flatsize).reshape(*dims) # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get master wave map loc['MASTERWAVEFILE'] = spirouDB.GetDatabaseMasterWave(p) # read master wave map mout = spirouImage.GetWaveSolution(p, filename=loc['MASTERWAVEFILE'], return_wavemap=True, quiet=True, fiber=wave_fiber) loc['MASTERWAVEPARAMS'], loc['MASTERWAVE'], loc['WSOURCE'] = mout keys = ['MASTERWAVEPARAMS', 'MASTERWAVE', 'MASTERWAVEFILE', 'WSOURCE'] loc.set_sources(keys, main_name) # ---------------------------------------------------------------------- # Compile a median SNR for rejection of bad files # ---------------------------------------------------------------------- snr_all = [] # choose snr to check snr_order = p['QC_FIT_TELLU_SNR_ORDER'] # loop through files for it, filename in enumerate(tell_files): header = spirouImage.ReadHeader(p, filename) # get the SNR from header nbo = spirouImage.ReadParam(p, header, 'KW_WAVE_ORD_N', dtype=int, return_value=True) snr = spirouImage.Read1Dkey(p, header, p['kw_E2DS_SNR'][0], nbo) # append snr_all snr_all.append(snr[snr_order]) # work our bad snr (less than half the median SNR) snr_thres = np.nanmedian(snr_all) / 2.0 bad_snr_objects = np.where(snr_all < snr_thres)[0] # ---------------------------------------------------------------------- # Loop through input files # ---------------------------------------------------------------------- # empty lists for storage loc['BASE_ROWNUM'], loc['BASE_FILELIST'] = [], [] loc['BASE_OBJNAME'], loc['BASE_OBJECT'] = [], [] loc['BASE_BERVLIST'], loc['BASE_WAVELIST'] = [], [] loc['BASE_SNRLIST_{0}'.format(snr_order)] = [] loc['BASE_DATELIST'], loc['BASE_VERSION'] = [], [] loc['BASE_DARKFILE'], loc['BASE_BADFILE'] = [], [] loc['BASE_LOCOFILE'], loc['BASE_BLAZFILE'] = [], [] loc['BASE_FLATFILE'], loc['BASE_SHAPEFILE'] = [], [] # loop through files for it, filename in enumerate(tell_files): # get base filenmae basefilename = os.path.basename(filename) # ------------------------------------------------------------------ # skip if in bad snr objects if it in bad_snr_objects: wargs = [it + 1, len(tell_files), snr_all[it], snr_thres] wmsg1 = ('Skipping file {0} of {1} due to bad SNR ({2:.3f} < ' '{3:.3f})'.format(*wargs)) wmsg2 = '\tFile = {0}'.format(basefilename) WLOG(p, 'warning', [wmsg1, wmsg2]) continue # ------------------------------------------------------------------ # append basename to file list loc['BASE_FILELIST'].append(basefilename) # ------------------------------------------------------------------ # create image for storage image = np.repeat([np.nan], np.product(loc['DATA'].shape)) image = image.reshape(loc['DATA'].shape) # ------------------------------------------------------------------ # Load the data for this file tdata0, thdr, _, _ = spirouImage.ReadImage(p, filename) nbo, npix = tdata0.shape # Correct for the blaze tdata = tdata0 / loc['NBLAZE'] # get berv and add to list if p['KW_BERV'][0] in thdr: loc['BASE_BERVLIST'].append('{0}'.format(thdr[p['KW_BERV'][0]])) else: loc['BASE_BERVLIST'].append('UNKNOWN') # ------------------------------------------------------------------ # Get parameters from header snr = snr_all[it] dateobs = spirouImage.ReadParam(p, thdr, 'KW_DATE_OBS', dtype=str, return_value=True) utcobs = spirouImage.ReadParam(p, thdr, 'KW_UTC_OBS', dtype=str, return_value=True) tobjname = spirouImage.ReadParam(p, thdr, 'KW_OBJNAME', dtype=str, return_value=True) tobject = spirouImage.ReadParam(p, thdr, 'KW_OBJECT', dtype=str, return_value=True) tversion = spirouImage.ReadParam(p, thdr, 'KW_version', dtype=str, return_value=True) tdarkfile = spirouImage.ReadParam(p, thdr, 'KW_CDBDARK', dtype=str, return_value=True) tbadfile = spirouImage.ReadParam(p, thdr, 'KW_CDBBAD', dtype=str, return_value=True) tlocofile = spirouImage.ReadParam(p, thdr, 'KW_CDBLOCO', dtype=str, return_value=True) tblazfile = spirouImage.ReadParam(p, thdr, 'KW_CDBBLAZE', dtype=str, return_value=True) tflatfile = spirouImage.ReadParam(p, thdr, 'KW_CDBFLAT', dtype=str, return_value=True) tshapfile = spirouImage.ReadParam(p, thdr, 'KW_CDBSHAPE', dtype=str, return_value=True) # append to lists loc['BASE_ROWNUM'].append(it) loc['BASE_SNRLIST_{0}'.format(snr_order)].append(snr_all[it]) loc['BASE_DATELIST'].append('{0}_{1}'.format(dateobs, utcobs)) loc['BASE_OBJNAME'].append(tobjname) loc['BASE_OBJECT'].append(tobject) loc['BASE_VERSION'].append(tversion) loc['BASE_DARKFILE'].append(tdarkfile) loc['BASE_BADFILE'].append(tbadfile) loc['BASE_LOCOFILE'].append(tlocofile) loc['BASE_BLAZFILE'].append(tblazfile) loc['BASE_FLATFILE'].append(tflatfile) loc['BASE_SHAPEFILE'].append(tshapfile) # ------------------------------------------------------------------ # Get the wave solution for this file # ------------------------------------------------------------------ # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # need to reload calibDB # as we have custom arguments need to load the calibration database calib_db = dict(p['CALIBDB']) del p['CALIBDB'] p = spirouStartup.LoadCalibDB(p, header=thdr) # get wave solution wout = spirouImage.GetWaveSolution(p, image=tdata, hdr=thdr, return_wavemap=True, fiber=wave_fiber, return_filename=True) _, loc['WAVE'], loc['WAVEFILE'], _ = wout loc.set_sources(['WAVE', 'WAVEFILE'], main_name) # add wave to wave list loc['BASE_WAVELIST'].append(loc['WAVEFILE']) # readd original calibDB to p p['CALIBDB'] = dict(calib_db) # ------------------------------------------------------------------ # Get the Barycentric correction from header dv, _, _ = spirouImage.GetBERV(p, thdr) # ------------------------------------------------------------------ # log stats wmsg = 'Processing file {0} of {1} file={2} dv={3}' wargs = [it + 1, len(tell_files), basefilename, dv] WLOG(p, 'info', wmsg.format(*wargs)) # ------------------------------------------------------------------ # shift to correct berv dvshift = spirouMath.relativistic_waveshift(dv, units='km/s') image = spirouTelluric.Wave2Wave(p, tdata, loc['WAVE'] * dvshift, loc['MASTERWAVE']) # ------------------------------------------------------------------ # loop around orders for order_num in range(loc['DATA'].shape[0]): # normalise the tdata tdata[order_num, :] /= np.nanmedian(tdata[order_num, :]) image[order_num, :] /= np.nanmedian(image[order_num, :]) # ------------------------------------------------------------------ # add to cube storage big_cube[:, :, it] = image big_cube0[:, :, it] = tdata # ---------------------------------------------------------------------- # log if we have no files if len(loc['BASE_FILELIST']) == 0: wmsg = 'No good files found for object ="{0}" skipping' WLOG(p, 'warning', wmsg.format(loc['OBJNAME'])) # End Message wmsg = 'Recipe {0} has been successfully completed' WLOG(p, 'info', wmsg.format(p['PROGRAM'])) # return a copy of locally defined variables in the memory return dict(locals()) # ---------------------------------------------------------------------- # make median image with warnings.catch_warnings(record=True) as _: big_cube_med = np.nanmedian(big_cube, axis=2) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # TODO: Needs doing # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # add to qc header lists qc_values.append('None') qc_names.append('None') qc_logic.append('None') qc_pass.append(1) # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ---------------------------------------------------------------------- # Write Cube median (the template) to file # ---------------------------------------------------------------------- # get raw file name raw_in_file = os.path.basename(p['FITSFILENAME']) # construct filename outfile, tag = spirouConfig.Constants.OBJTELLU_TEMPLATE_FILE(p, loc) outfilename = os.path.basename(outfile) # hdict is first file keys hdict = spirouImage.CopyOriginalKeys(loc['DATAHDR']) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['MASTERWAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # add wave solution coefficients hdict = spirouImage.AddKey2DList(p, hdict, p['KW_WAVE_PARAM'], values=loc['MASTERWAVEPARAMS']) # write to file p = spirouImage.WriteImage(p, outfile, big_cube_med, hdict) # ---------------------------------------------------------------------- # Update the telluric database with the template # ---------------------------------------------------------------------- spirouDB.UpdateDatabaseObjTemp(p, outfilename, loc['OBJNAME'], loc['DATAHDR']) # put file in telluDB spirouDB.PutTelluFile(p, outfile) # ---------------------------------------------------------------------- # Save cubes to file # ---------------------------------------------------------------------- # make big cube table big_table = spirouTelluric.ConstructBigTable(p, loc) # construct file names outfile1, tag1 = spirouConfig.Constants.OBJTELLU_TEMPLATE_CUBE_FILE1( p, loc) outfile2, tag2 = spirouConfig.Constants.OBJTELLU_TEMPLATE_CUBE_FILE2( p, loc) # log big cube 1 wmsg1 = 'Saving bigcube to file {0}'.format(os.path.basename(outfile1)) # save big cube 1 hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) big_cube_s = np.swapaxes(big_cube, 1, 2) p = spirouImage.WriteImageTable(p, outfile1, image=big_cube_s, table=big_table, hdict=hdict) # log big cube 0 wmsg = 'Saving bigcube0 to file {0}'.format(os.path.basename(outfile2)) # save big cube 0 hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) big_cube_s0 = np.swapaxes(big_cube0, 1, 2) p = spirouImage.WriteImageTable(p, outfile2, image=big_cube_s0, table=big_table, hdict=hdict) # # mega plot # nfiles = big_cube_s0.shape[1] # ncols = int(np.ceil(np.sqrt(nfiles))) # nrows = int(np.ceil(nfiles/ncols)) # fig, frames = plt.subplots(ncols=ncols, nrows=nrows) # for it in range(big_cube_s0.shape[1]): # jt, kt = it // ncols, it % ncols # frame = frames[jt][kt] # frame.imshow(big_cube_s0[:, it, :], origin='lower') # frame.set(xlim=(2030, 2060)) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(cores=1, filetype='EXT_E2DS_FF_AB'): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # set up function name main_name = __NAME__ + '.main()' # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # now get custom arguments pos, fmt = [0, 1], [int, str] names, call = ['cores', 'filetype'], [cores, filetype] customargs = spirouStartup.GetCustomFromRuntime(p, pos, fmt, names, calls=call, require_night_name=False) p = spirouStartup.LoadArguments(p, None, customargs=customargs, mainfitsdir='reduced', require_night_name=False) # ------------------------------------------------------------------------- # find all telluric stars telluric_stars = spirouTelluric.FindTelluricStars(p) # print number found nfound = len(telluric_stars) nstar = 0 for tell_star in list(telluric_stars.keys()): nstar += len(telluric_stars[tell_star]) wmsg = 'Found {0} Telluric stars ({1} observations total)' WLOG(p, 'info', wmsg.format(nfound, nstar)) # # ------------------------------------------------------------------------- # # Step 0: Reset telluric database # # ------------------------------------------------------------------------- # #reset = spirouTools.drs_reset.reset_confirmation(p, 'TelluDB') # reset = True # if reset: # spirouTools.drs_reset.reset_telludb(p, False) # keep track of errors errors = [] # ------------------------------------------------------------------------- # Step 1: Run mk_tellu on all telluric stars # ------------------------------------------------------------------------- for t_it, tell_star in enumerate(list(telluric_stars.keys())): # loop around object filenames for o_it, objfilename in enumerate(telluric_stars[tell_star]): # Log progress pargs = [ p, 'Make Tellurics I', tell_star, t_it, nfound, o_it, len(telluric_stars[tell_star]) ] spirouTelluric.UpdateProcessDB(*pargs) # get arguments from filename args = spirouTelluric.GetDBarguments(p, objfilename) # run obj_mk_tellu try: ll = obj_mk_tellu.main(**args) except SystemExit as e: errors.append([pargs[1], objfilename, e]) # force close all plots sPlt.closeall() # ------------------------------------------------------------------------- # Step 2: Run fit tellu on all telluric stars # ------------------------------------------------------------------------- for t_it, tell_star in enumerate(list(telluric_stars.keys())): # loop around object filenames for o_it, objfilename in enumerate(telluric_stars[tell_star]): # Log progress pargs = [ p, 'Fit Tellurics', tell_star, t_it, nfound, o_it, len(telluric_stars[tell_star]) ] spirouTelluric.UpdateProcessDB(*pargs) # get arguments from filename args = spirouTelluric.GetDBarguments(p, objfilename) # run obj_mk_tellu try: obj_fit_tellu.main(**args) except SystemExit as e: errors.append([pargs[1], objfilename, e]) # force close all plots sPlt.closeall() # ------------------------------------------------------------------------- # step 3: Run mk_obj_template on each telluric star obj name # ------------------------------------------------------------------------- for t_it, tell_star in enumerate(list(telluric_stars.keys())): # log progress (big) pmsg = 'Make Telluric Template: Processing object = {0} ({1}/{2}' wmsgs = [ '', '=' * 60, '', pmsg.format(tell_star, t_it + 1, nfound), '', '=' * 60, '' ] WLOG(p, 'info', wmsgs, wrap=False) # get last object objfilename = telluric_stars[tell_star][-1] # get arguments from filename args = spirouTelluric.GetDBarguments(p, objfilename) # run obj_mk_obj_template try: obj_mk_obj_template.main(**args) except SystemExit as e: errors.append(['Telluric Template', tell_star, e]) # force close all plots sPlt.closeall() # ------------------------------------------------------------------------- # step 4: Run mk_tellu on all telluric stars # ------------------------------------------------------------------------- for t_it, tell_star in enumerate(list(telluric_stars.keys())): # loop around object filenames for o_it, objfilename in enumerate(telluric_stars[tell_star]): # Log progress pargs = [ p, 'Make Tellurics II', tell_star, t_it, nfound, o_it, len(telluric_stars[tell_star]) ] spirouTelluric.UpdateProcessDB(*pargs) # get arguments from filename args = spirouTelluric.GetDBarguments(p, objfilename) # run obj_mk_tellu try: obj_mk_tellu.main(**args) except SystemExit as e: errors.append([pargs[1], objfilename, e]) # force close all plots sPlt.closeall() # ---------------------------------------------------------------------- # Print all errors # ---------------------------------------------------------------------- if len(errors) > 0: emsgs = ['', '=' * 50, 'Errors were as follows: '] # loop around errors for error in errors: emsgs.append('') emsgs.append('{0}: Object = {1}'.format(error[0], error[1])) emsgs.append('\t{0}'.format(error[2])) emsgs.append('') WLOG(p, 'error', emsgs) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): """ cal_loc_RAW_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_loc_RAW_spirou.py [night_name] [files] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) p = spirouStartup.InitialFileSetup(p, calibdb=True) # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add') # ---------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- data = spirouImage.FixNonPreProcess(p, data) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # ---------------------------------------------------------------------- # Correction of DARK # ---------------------------------------------------------------------- p, datac = spirouImage.CorrectForDark(p, data, hdr) # ---------------------------------------------------------------------- # Interpolation over bad regions (to fill in the holes) # ---------------------------------------------------------------------- # log process # wmsg = 'Interpolating over bad regions' # WLOG(p, '', wmsg) # run interpolation # datac = spirouImage.InterpolateBadRegions(p, datac) # ---------------------------------------------------------------------- # Resize image # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p) # convert NaN to zeros data0 = np.where(~np.isfinite(data), np.zeros_like(data), data) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) data2 = spirouImage.ResizeImage(p, data0, **bkwargs) # log change in data size WLOG(p, '', ('Image format changed to ' '{0}x{1}').format(*data2.shape)) # ---------------------------------------------------------------------- # Correct for the BADPIX mask (set all bad pixels to zero) # ---------------------------------------------------------------------- p, data2 = spirouImage.CorrectForBadPix(p, data2, hdr) # ---------------------------------------------------------------------- # Background computation # ---------------------------------------------------------------------- if p['IC_DO_BKGR_SUBTRACTION']: # log that we are doing background measurement WLOG(p, '', 'Doing background measurement on raw frame') # get the bkgr measurement bargs = [p, data2, hdr] # background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs) p, background = spirouBACK.MeasureBackgroundMap(*bargs) else: background = np.zeros_like(data2) p['BKGRDFILE'] = 'None' p.set_source('BKGRDFILE', __NAME__ + '.main()') # apply background correction to data data2 = data2 - background # ---------------------------------------------------------------------- # Construct image order_profile # ---------------------------------------------------------------------- # log that we are doing background measurement WLOG(p, '', 'Creating Order Profile') order_profile = spirouLOCOR.BoxSmoothedImage(data2, p['LOC_BOX_SIZE']) # data 2 is now set to the order profile data2o = data2.copy() data2 = order_profile.copy() # ---------------------------------------------------------------------- # Write image order_profile to file # ---------------------------------------------------------------------- # Construct folder and filename rawfits, tag1 = spirouConfig.Constants.LOC_ORDER_PROFILE_FILE(p) rawfitsname = os.path.split(rawfits)[-1] # log saving order profile wmsg = 'Saving processed raw frame in {0}' WLOG(p, '', wmsg.format(rawfitsname)) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) # write to file p = spirouImage.WriteImage(p, rawfits, order_profile, hdict) # ---------------------------------------------------------------------- # Move order_profile to calibDB and update calibDB # ---------------------------------------------------------------------- # set key for calibDB keydb = 'ORDER_PROFILE_{0}'.format(p['FIBER']) # copy dark fits file to the calibDB folder spirouDB.PutCalibFile(p, rawfits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, rawfitsname, hdr) # ###################################################################### # Localization of orders on central column # ###################################################################### # storage dictionary for localization parameters loc = ParamDict() # Plots setup: start interactive plot if p['DRS_PLOT'] > 0: sPlt.start_interactive_session(p) # ---------------------------------------------------------------------- # Measurement and correction of background on the central column # ---------------------------------------------------------------------- loc = spirouBACK.MeasureBkgrdGetCentPixs(p, loc, data2) # ---------------------------------------------------------------------- # Search for order center on the central column - quick estimation # ---------------------------------------------------------------------- # log progress WLOG(p, '', 'Searching order center on central column') # plot the minimum of ycc and ic_locseuil if in debug and plot mode if p['DRS_DEBUG'] > 0 and p['DRS_PLOT']: sPlt.debug_locplot_min_ycc_loc_threshold(p, loc['YCC']) # find the central positions of the orders in the central posc_all = spirouLOCOR.FindPosCentCol(loc['YCC'], p['IC_LOCSEUIL']) # depending on the fiber type we may need to skip some pixels and also # we need to add back on the ic_offset applied start = p['IC_FIRST_ORDER_JUMP'] posc = posc_all[start:] + p['IC_OFFSET'] # work out the number of orders to use (minimum of ic_locnbmaxo and number # of orders found in 'LocateCentralOrderPositions') number_of_orders = np.min([p['IC_LOCNBMAXO'], len(posc)]) # log the number of orders than have been detected wargs = [p['FIBER'], int(number_of_orders / p['NBFIB']), p['NBFIB']] WLOG(p, 'info', ('On fiber {0} {1} orders have been detected ' 'on {2} fiber(s)').format(*wargs)) # ---------------------------------------------------------------------- # Search for order center and profile on specific columns # ---------------------------------------------------------------------- # get fit polynomial orders for position and width fitpos, fitwid = p['IC_LOCDFITC'], p['IC_LOCDFITW'] # Create arrays to store position and width of order for each order loc['CTRO'] = np.zeros((number_of_orders, data2.shape[1]), dtype=float) loc['SIGO'] = np.zeros((number_of_orders, data2.shape[1]), dtype=float) # Create arrays to store coefficients for position and width loc['ACC'] = np.zeros((number_of_orders, fitpos + 1)) loc['ASS'] = np.zeros((number_of_orders, fitpos + 1)) # Create arrays to store rms values for position and width loc['RMS_CENTER'] = np.zeros(number_of_orders) loc['RMS_FWHM'] = np.zeros(number_of_orders) # Create arrays to store point to point max value for position and width loc['MAX_PTP_CENTER'] = np.zeros(number_of_orders) loc['MAX_PTP_FRACCENTER'] = np.zeros(number_of_orders) loc['MAX_PTP_FWHM'] = np.zeros(number_of_orders) loc['MAX_PTP_FRACFWHM'] = np.zeros(number_of_orders) # Create arrays to store rejected points loc['MAX_RMPTS_POS'] = np.zeros(number_of_orders) loc['MAX_RMPTS_WID'] = np.zeros(number_of_orders) # set the central col centers in the cpos_orders array loc['CTRO'][:, p['IC_CENT_COL']] = posc[0:number_of_orders] # set source for all locs loc.set_all_sources(__NAME__ + '/main()') # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # storage for plotting loc['XPLOT'], loc['YPLOT'] = [], [] # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # loop around each order rorder_num = 0 for order_num in range(number_of_orders): # find the row centers of the columns loc = spirouLOCOR.FindOrderCtrs(p, data2, loc, order_num) # only keep the orders with non-zero width mask = loc['SIGO'][order_num, :] != 0 loc['X'] = np.arange(data2.shape[1])[mask] loc.set_source('X', __NAME__ + '/main()') # check that we have enough data points to fit data if len(loc['X']) > (fitpos + 1): # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # initial fit params iofargs = [p, loc, mask, rorder_num] # initial fit for center positions for this order loc, cf_data = spirouLOCOR.InitialOrderFit(*iofargs, kind='center') # initial fit for widths for this order loc, wf_data = spirouLOCOR.InitialOrderFit(*iofargs, kind='fwhm') # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # Log order number and fit at central pixel and width and rms wargs = [ rorder_num, cf_data['cfitval'], wf_data['cfitval'], cf_data['rms'] ] WLOG(p, '', ('ORDER: {0} center at pixel {1:.1f} width ' '{2:.1f} rms {3:.3f}').format(*wargs)) # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # sigma fit params sigfargs = [p, loc, cf_data, mask, order_num, rorder_num] # sigma clip fit for center positions for this order cf_data = spirouLOCOR.SigClipOrderFit(*sigfargs, kind='center') # load results into storage arrags for this order loc['ACC'][rorder_num] = cf_data['a'] loc['RMS_CENTER'][rorder_num] = cf_data['rms'] loc['MAX_PTP_CENTER'][rorder_num] = cf_data['max_ptp'] loc['MAX_PTP_FRACCENTER'][rorder_num] = cf_data['max_ptp_frac'] loc['MAX_RMPTS_POS'][rorder_num] = cf_data['max_rmpts'] # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # sigma fit params sigfargs = [p, loc, wf_data, mask, order_num, rorder_num] # sigma clip fit for width positions for this order wf_data = spirouLOCOR.SigClipOrderFit(*sigfargs, kind='fwhm') # load results into storage arrags for this order loc['ASS'][rorder_num] = wf_data['a'] loc['RMS_FWHM'][rorder_num] = wf_data['rms'] loc['MAX_PTP_FWHM'][rorder_num] = wf_data['max_ptp'] loc['MAX_PTP_FRACFWHM'][rorder_num] = wf_data['max_ptp_frac'] loc['MAX_RMPTS_WID'][rorder_num] = wf_data['max_rmpts'] # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - # increase the roder_num iterator rorder_num += 1 # else log that the order is unusable else: WLOG(p, '', 'Order found too much incomplete, discarded') # ---------------------------------------------------------------------- # Plot the image (ready for fit points to be overplotted later) if p['DRS_PLOT'] > 0: # get saturation threshold satseuil = p['IC_SATSEUIL'] * p['GAIN'] * p['NBFRAMES'] # plot image above saturation threshold sPlt.locplot_im_sat_threshold(p, loc, data2, satseuil) # ---------------------------------------------------------------------- # Log that order geometry has been measured WLOG(p, 'info', ('On fiber {0} {1} orders geometry have been ' 'measured').format(p['FIBER'], rorder_num)) # Get mean rms mean_rms_center = np.nansum( loc['RMS_CENTER'][:rorder_num]) * 1000 / rorder_num mean_rms_fwhm = np.nansum(loc['RMS_FWHM'][:rorder_num]) * 1000 / rorder_num # Log mean rms values wmsg = 'Average uncertainty on {0}: {1:.2f} [mpix]' WLOG(p, 'info', wmsg.format('position', mean_rms_center)) WLOG(p, 'info', wmsg.format('width', mean_rms_fwhm)) # ---------------------------------------------------------------------- # Plot of RMS for positions and widths # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: sPlt.locplot_order_number_against_rms(p, loc, rorder_num) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # check that max number of points rejected in center fit is below threshold if np.nansum(loc['MAX_RMPTS_POS']) > p['QC_LOC_MAXLOCFIT_REMOVED_CTR']: fmsg = 'abnormal points rejection during ctr fit ({0:.2f} > {1:.2f})' fail_msg.append( fmsg.format(np.nansum(loc['MAX_RMPTS_POS']), p['QC_LOC_MAXLOCFIT_REMOVED_CTR'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(np.nansum(loc['MAX_RMPTS_POS'])) qc_names.append('sum(MAX_RMPTS_POS)') qc_logic.append('sum(MAX_RMPTS_POS) > {0:.2f}' ''.format(p['QC_LOC_MAXLOCFIT_REMOVED_CTR'])) # ---------------------------------------------------------------------- # check that max number of points rejected in width fit is below threshold if np.nansum(loc['MAX_RMPTS_WID']) > p['QC_LOC_MAXLOCFIT_REMOVED_WID']: fmsg = 'abnormal points rejection during width fit ({0:.2f} > {1:.2f})' fail_msg.append( fmsg.format(np.nansum(loc['MAX_RMPTS_WID']), p['QC_LOC_MAXLOCFIT_REMOVED_WID'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(np.nansum(loc['MAX_RMPTS_WID'])) qc_names.append('sum(MAX_RMPTS_WID)') qc_logic.append('sum(MAX_RMPTS_WID) > {0:.2f}' ''.format(p['QC_LOC_MAXLOCFIT_REMOVED_WID'])) # ---------------------------------------------------------------------- # check that the rms in center fit is lower than qc threshold if mean_rms_center > p['QC_LOC_RMSMAX_CENTER']: fmsg = 'too high rms on center fitting ({0:.2f} > {1:.2f})' fail_msg.append(fmsg.format(mean_rms_center, p['QC_LOC_RMSMAX_CENTER'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(mean_rms_center) qc_names.append('mean_rms_center') qc_logic.append('mean_rms_center > {0:.2f}' ''.format(p['QC_LOC_RMSMAX_CENTER'])) # ---------------------------------------------------------------------- # check that the rms in center fit is lower than qc threshold if mean_rms_fwhm > p['QC_LOC_RMSMAX_FWHM']: fmsg = 'too high rms on profile fwhm fitting ({0:.2f} > {1:.2f})' fail_msg.append(fmsg.format(mean_rms_fwhm, p['QC_LOC_RMSMAX_CENTER'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(mean_rms_fwhm) qc_names.append('mean_rms_fwhm') qc_logic.append('mean_rms_fwhm > {0:.2f}' ''.format(p['QC_LOC_RMSMAX_CENTER'])) # ---------------------------------------------------------------------- # check for abnormal number of identified orders if rorder_num != p['QC_LOC_NBO']: fmsg = ('abnormal number of identified orders (found {0:.2f} ' 'expected {1:.2f})') fail_msg.append(fmsg.format(rorder_num, p['QC_LOC_NBO'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(rorder_num) qc_names.append('rorder_num') qc_logic.append('rorder_num != {0:.2f}'.format(p['QC_LOC_NBO'])) # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ---------------------------------------------------------------------- # Save and record of image of localization with order center and keywords # ---------------------------------------------------------------------- raw_loco_file = os.path.basename(p['FITSFILENAME']) # construct filename locofits, tag2 = spirouConfig.Constants.LOC_LOCO_FILE(p) locofitsname = os.path.split(locofits)[-1] # log that we are saving localization file WLOG(p, '', ('Saving localization information ' 'in file: {0}').format(locofitsname)) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # define new keys to add hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=raw_loco_file) hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_SIGDET']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_CONAD']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_BCKGRD'], value=loc['MEAN_BACKGRD']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_NBO'], value=rorder_num) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_C'], value=p['IC_LOCDFITC']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_W'], value=p['IC_LOCDFITW']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_E']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DELTA']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_MAXFLX'], value=float(loc['MAX_SIGNAL'])) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_SMAXPTS_CTR'], value=np.nansum(loc['MAX_RMPTS_POS'])) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_SMAXPTS_WID'], value=np.nansum(loc['MAX_RMPTS_WID'])) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_RMS_CTR'], value=mean_rms_center) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_RMS_WID'], value=mean_rms_fwhm) # write 2D list of position fit coefficients hdict = spirouImage.AddKey2DList(p, hdict, p['KW_LOCO_CTR_COEFF'], values=loc['ACC'][0:rorder_num]) # write 2D list of width fit coefficients hdict = spirouImage.AddKey2DList(p, hdict, p['KW_LOCO_FWHM_COEFF'], values=loc['ASS'][0:rorder_num]) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # write center fits and add header keys (via hdict) center_fits = spirouLOCOR.CalcLocoFits(loc['ACC'], data2.shape[1]) p = spirouImage.WriteImage(p, locofits, center_fits, hdict) # ---------------------------------------------------------------------- # Save and record of image of sigma # ---------------------------------------------------------------------- # construct filename locofits2, tag3 = spirouConfig.Constants.LOC_LOCO_FILE2(p) locofits2name = os.path.split(locofits2)[-1] # log that we are saving localization file wmsg = 'Saving FWHM information in file: {0}' WLOG(p, '', wmsg.format(locofits2name)) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # define new keys to add hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBACK'], value=p['BKGRDFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # add outputs hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_SIGDET']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_CONAD']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_NBO'], value=rorder_num) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_C'], value=p['IC_LOCDFITC']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_W'], value=p['IC_LOCDFITW']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_E']) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_MAXFLX'], value=float(loc['MAX_SIGNAL'])) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_SMAXPTS_CTR'], value=np.nansum(loc['MAX_RMPTS_POS'])) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_SMAXPTS_WID'], value=np.nansum(loc['MAX_RMPTS_WID'])) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_RMS_CTR'], value=mean_rms_center) hdict = spirouImage.AddKey(p, hdict, p['KW_LOC_RMS_WID'], value=mean_rms_fwhm) # write 2D list of position fit coefficients hdict = spirouImage.AddKey2DList(p, hdict, p['KW_LOCO_CTR_COEFF'], values=loc['ACC'][0:rorder_num]) # write 2D list of width fit coefficients hdict = spirouImage.AddKey2DList(p, hdict, p['KW_LOCO_FWHM_COEFF'], values=loc['ASS'][0:rorder_num]) # add quality control hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) # write image and add header keys (via hdict) width_fits = spirouLOCOR.CalcLocoFits(loc['ASS'], data2.shape[1]) p = spirouImage.WriteImage(p, locofits2, width_fits, hdict) # ---------------------------------------------------------------------- # Save and Record of image of localization # ---------------------------------------------------------------------- if p['IC_LOCOPT1']: # construct filename locofits3, tag4 = spirouConfig.Constants.LOC_LOCO_FILE3(p) locofits3name = os.path.split(locofits3)[-1] # log that we are saving localization file wmsg1 = 'Saving localization image with superposition of orders in ' wmsg2 = 'file: {0}'.format(locofits3name) WLOG(p, '', [wmsg1, wmsg2]) # superpose zeros over the fit in the image data4 = spirouLOCOR.ImageLocSuperimp(data2o, loc['ACC'][0:rorder_num]) # save this image to file hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_FIBER'], value=p['FIBER']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) p = spirouImage.WriteImage(p, locofits3, data4, hdict) # ---------------------------------------------------------------------- # Update the calibration database # ---------------------------------------------------------------------- if p['QC'] == 1: keydb = 'LOC_' + p['FIBER'] # copy localisation file to the calibDB folder spirouDB.PutCalibFile(p, locofits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, locofitsname, hdr) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) # force plotting to 1 p['DRS_PLOT'] = 1 # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data data, hdr, nx, ny = spirouImage.ReadImage(p) # ---------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- data = spirouImage.FixNonPreProcess(p, data, filename=p['FITSFILENAME']) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # now change the value of sigdet if require if p['IC_EXT_SIGDET'] > 0: p['SIGDET'] = float(p['IC_EXT_SIGDET']) # ---------------------------------------------------------------------- # Resize image # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- data = spirouImage.ConvertToADU(spirouImage.FlipImage(p, data), p=p) # convert NaN to zeros data2 = np.where(~np.isfinite(data), np.zeros_like(data), data) # resize image # bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], # ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], # getshape=False) # data2 = spirouImage.ResizeImage(data0, **bkwargs) # log change in data size # WLOG(p, '', ('Image format changed to ' # '{0}x{1}').format(*data2.shape[::-1])) # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- # get the number of bad pixels n_bad_pix = np.nansum(data2 == 0) n_bad_pix_frac = n_bad_pix * 100 / np.product(data2.shape) # Log number wmsg = 'Nb dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) satseuil = 64536. col = 2100 seuil = 10000 slice0 = 5 plt.ion() plt.clf() plt.imshow(data2, origin='lower', clim=(1., seuil)) plt.colorbar() plt.axis([0, nx, 0, ny]) plt.plot(np.ones(4096) * col, np.arange(4096), c='red') plt.figure() plt.clf() centpart = data2[:, col - slice0:col + slice0] # centpart = data2[col - slice:col + slice,:] # weights = np.where((centpart < satseuil) & (centpart > 0), 1, 0.0001) # y = np.average(centpart, axis=1, weights=weights) ## weighted average y = np.nanmedian(centpart, axis=1) # y=average(centpart,axis=1,weights=where((centpart>0),1,0.0001)) # ## weighted average plt.plot(np.arange(ny), y) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p, outputs=None) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, hcfile=None, fpfiles=None): """ cal_SLIT_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_SLIT_spirou.py [night_directory] [files] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) if hcfile is None or fpfiles is None: names, types = ['hcfile', 'fpfiles'], [str, str] customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1], types, names, last_multi=True) else: customargs = dict(hcfile=hcfile, fpfiles=fpfiles) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='fpfiles') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, hcfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['HCFILE']) p, fpfilenames = spirouStartup.MultiFileSetup(p, files=p['FPFILES']) # set fiber (it doesn't matter with the 2D image but we need this to get # the lamp type for FPFILES and HCFILES, AB == C p['FIBER'] = 'AB' p['FIB_TYP'] = [p['FIBER']] fsource = __NAME__ + '/main()' p.set_sources(['FIBER', 'FIB_TYP'], fsource) # set the hcfilename to the first hcfilenames fpfitsfilename = fpfilenames[0] # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # add a force plot off p['PLOT_PER_ORDER'] = PLOT_PER_ORDER p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Read FP and HC files # ---------------------------------------------------------------------- # read and combine all FP files except the first (fpfitsfilename) rargs = [p, 'add', fpfitsfilename, fpfilenames[1:]] p, fpdata, fphdr = spirouImage.ReadImageAndCombine(*rargs) # read first file (hcfitsfilename) hcdata, hchdr, _, _ = spirouImage.ReadImage(p, hcfitsfilename) # add data and hdr to loc loc = ParamDict() loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr # set the source sources = ['HCDATA', 'HCHDR'] loc.set_sources(sources, 'spirouImage.ReadImageAndCombine()') sources = ['FPDATA', 'FPHDR'] loc.set_sources(sources, 'spirouImage.ReadImage()') # --------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- hcdata = spirouImage.FixNonPreProcess(p, hcdata) fpdata = spirouImage.FixNonPreProcess(p, fpdata) # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, fphdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, fphdr, name='exptime') # get gain p = spirouImage.GetGain(p, fphdr, name='gain') # get lamp parameters p = spirouTHORCA.GetLampParams(p, hchdr) # ---------------------------------------------------------------------- # Correction of DARK # ---------------------------------------------------------------------- # p, hcdatac = spirouImage.CorrectForDark(p, hcdata, hchdr) hcdatac = hcdata p['DARKFILE'] = 'None' # p, fpdatac = spirouImage.CorrectForDark(p, fpdata, fphdr) fpdatac = fpdata # ---------------------------------------------------------------------- # Resize hc data # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- hcdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, hcdatac), p=p) # convert NaN to zeros hcdata0 = np.where(~np.isfinite(hcdata), np.zeros_like(hcdata), hcdata) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) hcdata2 = spirouImage.ResizeImage(p, hcdata0, **bkwargs) # log change in data size WLOG(p, '', ('HC Image format changed to ' '{0}x{1}').format(*hcdata2.shape)) # ---------------------------------------------------------------------- # Resize fp data # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- fpdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, fpdatac), p=p) # convert NaN to zeros fpdata0 = np.where(~np.isfinite(fpdata), np.zeros_like(fpdata), fpdata) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) fpdata2 = spirouImage.ResizeImage(p, fpdata0, **bkwargs) # log change in data size WLOG(p, '', ('FP Image format changed to ' '{0}x{1}').format(*fpdata2.shape)) # ---------------------------------------------------------------------- # Correct for the BADPIX mask (set all bad pixels to zero) # ---------------------------------------------------------------------- # p, hcdata2 = spirouImage.CorrectForBadPix(p, hcdata2, hchdr) # p, fpdata2 = spirouImage.CorrectForBadPix(p, fpdata2, fphdr) p['BADPFILE'] = 'None' # save data to loc loc['HCDATA'] = hcdata2 loc.set_source('HCDATA', __NAME__ + '/main()') # save data to loc loc['FPDATA'] = fpdata2 loc.set_source('FPDATA', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- # get the number of bad pixels n_bad_pix = np.nansum(hcdata2 <= 0) n_bad_pix_frac = n_bad_pix * 100 / np.product(hcdata2.shape) # Log number wmsg = 'Nb HC dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- # get the number of bad pixels n_bad_pix = np.nansum(fpdata2 <= 0) n_bad_pix_frac = n_bad_pix * 100 / np.product(fpdata2.shape) # Log number wmsg = 'Nb FP dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ------------------------------------------------------------------ # Get localisation coefficients # ------------------------------------------------------------------ # original there is a loop but it is not used --> removed p = spirouImage.FiberParams(p, p['FIBER'], merge=True) # get localisation fit coefficients p, loc = spirouLOCOR.GetCoeffs(p, fphdr, loc) # ------------------------------------------------------------------ # Get master wave solution map # ------------------------------------------------------------------ # get master wave map masterwavefile = spirouDB.GetDatabaseMasterWave(p) # log process wmsg1 = 'Getting master wavelength grid' wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile)) WLOG(p, '', [wmsg1, wmsg2]) # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # read master wave map wout = spirouImage.GetWaveSolution(p, filename=masterwavefile, return_wavemap=True, quiet=True, return_header=True, fiber=wave_fiber) loc['MASTERWAVEP'], loc['MASTERWAVE'] = wout[:2] loc['MASTERWAVEHDR'], loc['WSOURCE'] = wout[2:] # set sources wsource = ['MASTERWAVEP', 'MASTERWAVE', 'MASTERWAVEHDR'] loc.set_sources(wsource, 'spirouImage.GetWaveSolution()') # ---------------------------------------------------------------------- # Read UNe solution # ---------------------------------------------------------------------- wave_u_ne, amp_u_ne = spirouImage.ReadLineList(p) loc['LL_LINE'], loc['AMPL_LINE'] = wave_u_ne, amp_u_ne source = __NAME__ + '.main() + spirouImage.ReadLineList()' loc.set_sources(['LL_LINE', 'AMPL_LINE'], source) # ---------------------------------------------------------------------- # Read cavity length file # ---------------------------------------------------------------------- loc['CAVITY_LEN_COEFFS'] = spirouImage.ReadCavityLength(p) source = __NAME__ + '.main() + spirouImage.ReadCavityLength()' loc.set_source('CAVITY_LEN_COEFFS', source) # ------------------------------------------------------------------ # Calculate shape map # ------------------------------------------------------------------ loc = spirouImage.GetShapeMap(p, loc) # ------------------------------------------------------------------ # Plotting # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # plots setup: start interactive plot sPlt.start_interactive_session(p) # plot the shape process for one order sPlt.slit_shape_angle_plot(p, loc) # end interactive section sPlt.end_interactive_session(p) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # TODO: Decide on some quality control criteria? # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # add to qc header lists qc_values.append('None') qc_names.append('None') qc_logic.append('None') qc_pass.append(1) # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Writing DXMAP to file # ------------------------------------------------------------------ # get the raw tilt file name raw_shape_file = os.path.basename(p['FITSFILENAME']) # construct file name and path shapefits, tag = spirouConfig.Constants.SLIT_XSHAPE_FILE(p) shapefitsname = os.path.basename(shapefits) # Log that we are saving tilt file wmsg = 'Saving shape information in file: {0}' WLOG(p, '', wmsg.format(shapefitsname)) # Copy keys from fits file hdict = spirouImage.CopyOriginalKeys(fphdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='hcfile', values=p['HCFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE2'], dim1name='fpfile', values=p['FPFILES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # write tilt file to file p = spirouImage.WriteImage(p, shapefits, loc['DXMAP'], hdict) # ------------------------------------------------------------------ # Writing sanity check files # ------------------------------------------------------------------ if p['SHAPE_DEBUG_OUTPUTS']: # log WLOG(p, '', 'Saving debug sanity check files') # construct file names input_fp_file, tag1 = spirouConfig.Constants.SLIT_SHAPE_IN_FP_FILE(p) output_fp_file, tag2 = spirouConfig.Constants.SLIT_SHAPE_OUT_FP_FILE(p) input_hc_file, tag3 = spirouConfig.Constants.SLIT_SHAPE_IN_HC_FILE(p) output_hc_file, tag4 = spirouConfig.Constants.SLIT_SHAPE_OUT_HC_FILE(p) overlap_file, tag5 = spirouConfig.Constants.SLIT_SHAPE_OVERLAP_FILE(p) # write input fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) p = spirouImage.WriteImage(p, input_fp_file, loc['FPDATA'], hdict) # write output fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) p = spirouImage.WriteImage(p, output_fp_file, loc['FPDATA2'], hdict) # write input fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3) p = spirouImage.WriteImage(p, input_hc_file, loc['HCDATA'], hdict) # write output fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) p = spirouImage.WriteImage(p, output_hc_file, loc['HCDATA2'], hdict) # write overlap file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag5) p = spirouImage.WriteImage(p, overlap_file, loc['ORDER_OVERLAP'], hdict) # ---------------------------------------------------------------------- # Move to calibDB and update calibDB # ---------------------------------------------------------------------- if p['QC']: keydb = 'SHAPE' # copy shape file to the calibDB folder spirouDB.PutCalibFile(p, shapefits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, shapefitsname, fphdr) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, reffile=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) customargs = spirouStartup.GetCustomFromRuntime(p, [0], [str], ['reffile'], [True], [reffile]) p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='reffile', mainfitsdir='reduced') # load the calibDB p = spirouStartup.LoadCalibDB(p) # force plotting to 1 p['DRS_PLOT'] = 1 # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data p, fpfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['REFFILE']) # get the fiber type fiber1 = str(p['FIBER']) e2ds, hdr, nx, ny = spirouImage.ReadImage(p) p, blaze = spirouImage.ReadBlazeFile(p) # set source of wave file wsource = __NAME__ + '/main() + /spirouImage.GetWaveSolution' # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get wave image _, wave, _ = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True, fiber=wave_fiber) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- plt.ion() plt.figure() for i in np.arange(nx): plt.plot(wave[i], e2ds[i]) plt.xlabel('Wavelength [nm]') plt.ylabel('Flux e-') plt.title('Extracted spectra') plt.figure() for i in np.arange(nx): plt.plot(wave[i], e2ds[i] / blaze[i]) plt.xlabel('Wavelength [nm]') plt.ylabel('Relative Flux e-') plt.title('Blaze corrected Extracted spectra') # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p, outputs=None) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, fpfile=None, hcfiles=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) if hcfiles is None or fpfile is None: names, types = ['fpfile', 'hcfiles'], [str, str] customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1], types, names, last_multi=True) else: customargs = dict(hcfiles=hcfiles, fpfile=fpfile) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsdir='reduced', mainfitsfile='hcfiles') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, fpfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['FPFILE']) fiber1 = str(p['FIBER']) p, hcfilenames = spirouStartup.MultiFileSetup(p, files=p['HCFILES']) fiber2 = str(p['FIBER']) # set the hcfilename to the first hcfilenames hcfitsfilename = hcfilenames[0] # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Have to check that the fibers match # ---------------------------------------------------------------------- if fiber1 == fiber2: p['FIBER'] = fiber1 fsource = __NAME__ + '/main() & spirouStartup.GetFiberType()' p.set_source('FIBER', fsource) else: emsg = 'Fiber not matching for {0} and {1}, should be the same' eargs = [hcfitsfilename, fpfitsfilename] WLOG(p, 'error', emsg.format(*eargs)) # set the fiber type p['FIB_TYP'] = [p['FIBER']] p.set_source('FIB_TYP', __NAME__ + '/main()') # set find line mode find_lines_mode = p['HC_FIND_LINES_MODE'] # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read and combine all HC files except the first (fpfitsfilename) rargs = [p, 'add', hcfitsfilename, hcfilenames[1:]] p, hcdata, hchdr = spirouImage.ReadImageAndCombine(*rargs) # read first file (fpfitsfilename) fpdata, fphdr, _, _ = spirouImage.ReadImage(p, fpfitsfilename) # add data and hdr to loc loc = ParamDict() loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr # set the source sources = ['HCDATA', 'HCHDR'] loc.set_sources(sources, 'spirouImage.ReadImageAndCombine()') sources = ['FPDATA', 'FPHDR'] loc.set_sources(sources, 'spirouImage.ReadImage()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hchdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hchdr, name='exptime') # get gain p = spirouImage.GetGain(p, hchdr, name='gain') # get acquisition time p = spirouImage.GetAcqTime(p, hchdr, name='acqtime', kind='julian') bjdref = p['ACQTIME'] # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # get lamp parameters p = spirouTHORCA.GetLampParams(p, hchdr) # ---------------------------------------------------------------------- # Obtain the flat # ---------------------------------------------------------------------- # get the flat p, loc = spirouFLAT.GetFlat(p, loc, hchdr) # ---------------------------------------------------------------------- # Read blaze # ---------------------------------------------------------------------- # get tilts p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hchdr) loc.set_source('BLAZE', __NAME__ + '/main() + /spirouImage.ReadBlazeFile') # correct the data with the flat # TODO: Should this be used? # log # WLOG(p, '', 'Applying flat correction') # loc['HCDATA'] = loc['HCDATA']/loc['FLAT'] # loc['FPDATA'] = loc['FPDATA']/loc['FLAT'] # ---------------------------------------------------------------------- # Start plotting session # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # start interactive plot sPlt.start_interactive_session(p) # ---------------------------------------------------------------------- # loop around fiber type # ---------------------------------------------------------------------- for fiber in p['FIB_TYP']: # set fiber type for inside loop p['FIBER'] = fiber # ------------------------------------------------------------------ # Instrumental drift computation (if previous solution exists) # ------------------------------------------------------------------ # get key keydb = 'HCREF_{0}'.format(p['FIBER']) # check for key in calibDB if keydb in p['CALIBDB'].keys(): # log process wmsg = ('Doing Instrumental drift computation from previous ' 'calibration') WLOG(p, '', wmsg) # calculate instrument drift loc = spirouTHORCA.CalcInstrumentDrift(p, loc) # ------------------------------------------------------------------ # Wave solution # ------------------------------------------------------------------ # log message for loop wmsg = 'Processing Wavelength Calibration for Fiber {0}' WLOG(p, 'info', wmsg.format(p['FIBER'])) # ------------------------------------------------------------------ # Part 1 of cal_HC # ------------------------------------------------------------------ p, loc = cal_HC_E2DS_spirou.part1(p, loc, mode=find_lines_mode) # ------------------------------------------------------------------ # FP solution # ------------------------------------------------------------------ # log message wmsg = 'Calculating FP wave solution' WLOG(p, '', wmsg) # calculate FP wave solution # spirouTHORCA.FPWaveSolution(p, loc, mode=find_lines_mode) spirouTHORCA.FPWaveSolutionNew(p, loc) # ------------------------------------------------------------------ # FP solution plots # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # Plot the FP extracted spectrum against wavelength solution sPlt.wave_plot_final_fp_order(p, loc, iteration=1) # Plot the measured FP cavity width offset against line number sPlt.wave_local_width_offset_plot(p, loc) # Plot the FP line wavelength residuals sPlt.wave_fp_wavelength_residuals(p, loc) # ------------------------------------------------------------------ # Part 2 of cal_HC # ------------------------------------------------------------------ # set params for part2 p['QC_RMS_LITTROW_MAX'] = p['QC_WAVE_RMS_LITTROW_MAX'] p['QC_DEV_LITTROW_MAX'] = p['QC_WAVE_DEV_LITTROW_MAX'] p['IC_HC_N_ORD_START_2'] = min(p['IC_HC_N_ORD_START_2'], p['IC_FP_N_ORD_START']) p['IC_HC_N_ORD_FINAL_2'] = max(p['IC_HC_N_ORD_FINAL_2'], p['IC_FP_N_ORD_FINAL']) # run part 2 # p, loc = part2test(p, loc) p, loc = cal_HC_E2DS_spirou.part2(p, loc) # ---------------------------------------------------------------------- # End plotting session # ---------------------------------------------------------------------- # end interactive session sPlt.end_interactive_session(p) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(preview=1): # ---------------------------------------------------------------------- # get values from config file p = spirouStartup.Begin(recipe=__NAME__, quiet=True) customargs = spirouStartup.GetCustomFromRuntime(p, [0], [int], ['preview'], calls=[preview], required=[False], require_night_name=False) p = spirouStartup.LoadArguments(p, None, customargs=customargs, require_night_name=False) # ---------------------------------------------------------------------- # if in preview mode tell user if p['PREVIEW']: WLOG(p, 'info', 'Running in preview mode.') # ---------------------------------------------------------------------- # read and ask for new version WLOG(p, '', 'Reading DRS version') # set new version version = ask_for_new_version() # add tag of version if version is not None: # tag head with version git_remove_tag(version) git_tag_head(version) new = True else: version = str(__version__) new = False # ---------------------------------------------------------------------- # update DRS files if not p['PREVIEW']: update_version_file(VERSIONFILE, version) update_py_version(CONSTFILE, version) # ---------------------------------------------------------------------- # create new changelog WLOG(p, '', 'Updating changelog') if not p['PREVIEW']: git_change_log(FILENAME) else: git_change_log('tmp.txt') preview_log('tmp.txt') if new: git_remove_tag(version) # ---------------------------------------------------------------------- # if we are in preview mode should we keep these changes and update version if p['PREVIEW']: uinput = input('Keep changes? [Y]es [N]o:\t') if 'Y' in uinput.upper(): # redo tagging git_remove_tag(version) git_tag_head(version) # update version file and python version file update_version_file(VERSIONFILE, version) update_py_version(CONSTFILE, version) # move the tmp.txt to change log shutil.move('tmp.txt', FILENAME) else: os.remove('tmp.txt') # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p, outputs=None) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None, fiber_type=None, **kwargs): """ cal_DRIFT_E2DS_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_DRIFT_E2DS_spirou.py [night_directory] [files] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :param fiber_type: string, if None does all fiber types (defined in constants_SPIROU FIBER_TYPES (default is AB, A, B, C if defined then only does this fiber type (but must be in FIBER_TYPES) :param kwargs: any keyword to overwrite constant in param dict "p" :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) p = spirouStartup.InitialFileSetup(p, calibdb=True) # deal with fiber type if fiber_type is None: fiber_type = p['FIBER_TYPES'] if type(fiber_type) == str: if fiber_type.upper() == 'ALL': fiber_type = p['FIBER_TYPES'] elif fiber_type in p['FIBER_TYPES']: fiber_type = [fiber_type] else: emsg = 'fiber_type="{0}" not understood' WLOG(p, 'error', emsg.format(fiber_type)) # set fiber type p['FIB_TYPE'] = fiber_type p.set_source('FIB_TYPE', __NAME__ + '__main__()') # Overwrite keys from source for kwarg in kwargs: p[kwarg] = kwargs[kwarg] # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add') # ---------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- data = spirouImage.FixNonPreProcess(p, data) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # now change the value of sigdet if require if p['IC_EXT_SIGDET'] > 0: p['SIGDET'] = float(p['IC_EXT_SIGDET']) # get DPRTYPE from header (Will have it if valid) p = spirouImage.ReadParam(p, hdr, 'KW_DPRTYPE', required=False, dtype=str) # check the DPRTYPE is not None if (p['DPRTYPE'] == 'None') or (['DPRTYPE'] is None): emsg1 = 'Error: {0} is not set in header for file {1}' eargs = [p['KW_DPRTYPE'][0], p['FITSFILENAME']] emsg2 = '\tPlease run pre-processing on file.' emsg3 = ('\tIf pre-processing fails or skips file, file is not ' 'currrently as valid DRS fits file.') WLOG(p, 'error', [emsg1.format(*eargs), emsg2, emsg3]) else: p['DPRTYPE'] = p['DPRTYPE'].strip() # ---------------------------------------------------------------------- # Correction of DARK # ---------------------------------------------------------------------- p, datac = spirouImage.CorrectForDark(p, data, hdr) # ---------------------------------------------------------------------- # Resize image # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to ADU data = spirouImage.ConvertToADU(spirouImage.FlipImage(p, datac), p=p) # convert NaN to zeros data0 = np.where(~np.isfinite(data), np.zeros_like(data), data) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) data1 = spirouImage.ResizeImage(p, data0, **bkwargs) # log change in data size wmsg = 'Image format changed to {1}x{0}' WLOG(p, '', wmsg.format(*data1.shape)) # ---------------------------------------------------------------------- # Correct for the BADPIX mask (set all bad pixels to zero) # ---------------------------------------------------------------------- p, data1 = spirouImage.CorrectForBadPix(p, data1, hdr) # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- # get the number of bad pixels n_bad_pix = np.sum(~np.isfinite(data1)) n_bad_pix_frac = n_bad_pix * 100 / np.product(data1.shape) # Log number wmsg = 'Nb dead pixels = {0} / {1:.4f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ---------------------------------------------------------------------- # Get the miny, maxy and max_signal for the central column # ---------------------------------------------------------------------- # get the central column y = data1[p['IC_CENT_COL'], :] # get the min max and max signal using box smoothed approach miny, maxy, max_signal, diff_maxmin = spirouBACK.MeasureMinMaxSignal(p, y) # Log max average flux/pixel wmsg = 'Maximum average flux/pixel in the spectrum: {0:.1f} [ADU]' WLOG(p, 'info', wmsg.format(max_signal / p['NBFRAMES'])) # ---------------------------------------------------------------------- # Background computation # ---------------------------------------------------------------------- if p['IC_DO_BKGR_SUBTRACTION']: # log that we are doing background measurement WLOG(p, '', 'Doing background measurement on raw frame') # get the bkgr measurement bargs = [p, data1, hdr] # background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs) p, background = spirouBACK.MeasureBackgroundMap(*bargs) else: background = np.zeros_like(data1) p['BKGRDFILE'] = 'None' p.set_source('BKGRDFILE', __NAME__ + '.main()') # apply background correction to data (and set to zero where negative) data1 = data1 - background # ---------------------------------------------------------------------- # Read tilt slit angle # ---------------------------------------------------------------------- # define loc storage parameter dictionary loc = ParamDict() # get tilts (if the mode requires it) if p['IC_EXTRACT_TYPE'] not in EXTRACT_SHAPE_TYPES: p, loc['TILT'] = spirouImage.ReadTiltFile(p, hdr) loc.set_source('TILT', __NAME__ + '/main() + /spirouImage.ReadTiltFile') else: loc['TILT'] = None loc.set_source('TILT', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Earth Velocity calculation # ---------------------------------------------------------------------- if p['IC_IMAGE_TYPE'] == 'H4RG': p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, hdr) # ---------------------------------------------------------------------- # Get all fiber data (for all fibers) # ---------------------------------------------------------------------- # TODO: This is temp solution for options 5a and 5b loc_fibers = spirouLOCOR.GetFiberData(p, hdr) # ------------------------------------------------------------------ # Deal with debananafication # ------------------------------------------------------------------ # if mode 4a or 4b we need to straighten in x only if p['IC_EXTRACT_TYPE'] in ['4a', '4b']: # get the shape parameters p, shapem_x = spirouImage.GetShapeX(p, hdr) p, shape_local = spirouImage.GetShapeLocal(p, hdr) # log progress WLOG(p, '', 'Debananafying (straightening) image') # apply shape transforms targs = dict(lin_transform_vect=shape_local, dxmap=shapem_x) data2 = spirouImage.EATransform(data1, **targs) # if mode 5a or 5b we need to straighten in x and y using the # polynomial fits for location elif p['IC_EXTRACT_TYPE'] in ['5a', '5b']: # get the shape parameters p, shapem_x = spirouImage.GetShapeX(p, hdr) p, shapem_y = spirouImage.GetShapeY(p, hdr) p, shape_local = spirouImage.GetShapeLocal(p, hdr) p, fpmaster = spirouImage.GetFPMaster(p, hdr) # get the bad pixel map bkwargs = dict(return_map=True, quiet=True) p, badpix = spirouImage.CorrectForBadPix(p, data1, hdr, **bkwargs) # log progress WLOG(p, '', 'Cleaning image') # clean the image data1 = spirouEXTOR.CleanHotpix(data1, badpix) # log progress WLOG(p, '', 'Debananafying (straightening) image') # apply shape transforms targs = dict(lin_transform_vect=shape_local, dxmap=shapem_x, dymap=shapem_y) data2 = spirouImage.EATransform(data1, **targs) # in any other mode we do not straighten else: data2 = np.array(data1) # ---------------------------------------------------------------------- # Fiber loop # ---------------------------------------------------------------------- # loop around fiber types for fiber in p['FIB_TYPE']: # set fiber p['FIBER'] = fiber p.set_source('FIBER', __NAME__ + '/main()()') # ------------------------------------------------------------------ # Read wavelength solution # ------------------------------------------------------------------ # set source of wave file wsource = __NAME__ + '/main() + /spirouImage.GetWaveSolution' # Force A and B to AB solution if fiber in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = fiber # get wave image wkwargs = dict(hdr=hdr, return_wavemap=True, return_filename=True, return_header=True, fiber=wave_fiber) wout = spirouImage.GetWaveSolution(p, **wkwargs) loc['WAVEPARAMS'], loc['WAVE'], loc['WAVEFILE'] = wout[:3] loc['WAVEHDR'], loc['WSOURCE'] = wout[3:] source_names = ['WAVE', 'WAVEFILE', 'WAVEPARAMS', 'WAVEHDR'] loc.set_sources(source_names, wsource) # get dates loc['WAVE_ACQTIMES'] = spirouDB.GetTimes(p, loc['WAVEHDR']) loc.set_source('WAVE_ACQTIMES', __NAME__ + '.main()') # get the recipe that produced the wave solution if 'WAVECODE' in loc['WAVEHDR']: loc['WAVE_CODE'] = loc['WAVEHDR']['WAVECODE'] else: loc['WAVE_CODE'] = 'UNKNOWN' loc.set_source('WAVE_CODE', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Get WFP keys # ---------------------------------------------------------------------- # Read the WFP keys - if they don't exist set to None and deal # with later p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_DRIFT', name='WFP_DRIFT', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_FWHM', name='WFP_FWHM', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_CONTRAST', name='WFP_CONTRAST', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_MAXCPP', name='WFP_MAXCPP', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_MASK', name='WFP_MASK', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_LINES', name='WFP_LINES', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_TARG_RV', name='WFP_TARG_RV', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_WIDTH', name='WFP_WIDTH', required=False) p = spirouImage.ReadParam(p, loc['WAVEHDR'], 'KW_WFP_STEP', name='WFP_STEP', required=False) # ---------------------------------------------------------------------- # Read Flat file # ---------------------------------------------------------------------- fout = spirouImage.ReadFlatFile(p, hdr, return_header=True) p, loc['FLAT'], flathdr = fout loc.set_source('FLAT', __NAME__ + '/main() + /spirouImage.ReadFlatFile') # get flat extraction mode if p['KW_E2DS_EXTM'][0] in flathdr: flat_ext_mode = flathdr[p['KW_E2DS_EXTM'][0]] else: flat_ext_mode = None # ------------------------------------------------------------------ # Check extraction method is same as flat extraction method # ------------------------------------------------------------------ # get extraction method and function extmethod, extfunc = spirouEXTOR.GetExtMethod(p, p['IC_EXTRACT_TYPE']) if not DEBUG: # compare flat extraction mode to extraction mode spirouEXTOR.CompareExtMethod(p, flat_ext_mode, extmethod, 'FLAT', 'EXTRACTION') # ------------------------------------------------------------------ # Read Blaze file # ------------------------------------------------------------------ p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hdr) blazesource = __NAME__ + '/main() + /spirouImage.ReadBlazeFile' loc.set_source('BLAZE', blazesource) # ------------------------------------------------------------------ # Get fiber specific parameters from loc_fibers # ------------------------------------------------------------------ # get this fibers parameters p = spirouImage.FiberParams(p, p['FIBER'], merge=True) # get localisation parameters for key in loc_fibers[fiber]: loc[key] = loc_fibers[fiber][key] loc.set_source(key, loc_fibers[fiber].sources[key]) # get locofile source p['LOCOFILE'] = loc['LOCOFILE'] p.set_source('LOCOFILE', loc.sources['LOCOFILE']) # get the order_profile order_profile = loc_fibers[fiber]['ORDER_PROFILE'] # ------------------------------------------------------------------ # Set up Extract storage # ------------------------------------------------------------------ # Create array to store extraction (for each order and each pixel # along order) loc['E2DS'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) loc['E2DSFF'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) loc['E2DSLL'] = [] loc['SPE1'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) loc['SPE3'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) loc['SPE4'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) loc['SPE5'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1])) # Create array to store the signal to noise ratios for each order loc['SNR'] = np.zeros(loc['NUMBER_ORDERS']) # ------------------------------------------------------------------ # Extract orders # ------------------------------------------------------------------ # source for parameter dictionary source = __NAME__ + '/main()' # get limits of order extraction valid_orders = spirouEXTOR.GetValidOrders(p, loc) # loop around each order for order_num in valid_orders: # ------------------------------------------------------------- # IC_EXTRACT_TYPE decides the extraction routine # ------------------------------------------------------------- eargs = [p, loc, data2, order_num] ekwargs = dict(mode=p['IC_EXTRACT_TYPE'], order_profile=order_profile) with warnings.catch_warnings(record=True) as w: eout = spirouEXTOR.Extraction(*eargs, **ekwargs) # deal with different return if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES: e2ds, e2dsll, cpt = eout else: e2ds, cpt = eout e2dsll = None # ------------------------------------------------------------- # calculate the noise range1, range2 = p['IC_EXT_RANGE1'], p['IC_EXT_RANGE2'] # set the noise noise = p['SIGDET'] * np.sqrt(range1 + range2) # get window size blaze_win1 = int(data2.shape[0] / 2) - p['IC_EXTFBLAZ'] blaze_win2 = int(data2.shape[0] / 2) + p['IC_EXTFBLAZ'] # get average flux per pixel flux = np.nansum( e2ds[blaze_win1:blaze_win2]) / (2 * p['IC_EXTFBLAZ']) # calculate signal to noise ratio = flux/sqrt(flux + noise^2) snr = flux / np.sqrt(flux + noise**2) # log the SNR RMS wmsg = 'On fiber {0} order {1}: S/N= {2:.1f} Nbcosmic= {3}' wargs = [p['FIBER'], order_num, snr, cpt] WLOG(p, '', wmsg.format(*wargs)) # add calculations to storage loc['E2DS'][order_num] = e2ds loc['E2DSFF'][order_num] = e2ds / loc['FLAT'][order_num] loc['SNR'][order_num] = snr # save the longfile if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES: loc['E2DSLL'].append(e2dsll) # set sources loc.set_sources(['e2ds', 'SNR'], source) # Log if saturation level reached satvalue = (flux / p['GAIN']) / (range1 + range2) if satvalue > (p['QC_LOC_FLUMAX'] * p['NBFRAMES']): wmsg = 'SATURATION LEVEL REACHED on Fiber {0} order={1}' WLOG(p, 'warning', wmsg.format(fiber, order_num)) # ------------------------------------------------------------------ # Thermal correction # ------------------------------------------------------------------ # get fiber type if fiber in ['AB', 'A', 'B']: fibertype = p['DPRTYPE'].split('_')[0] else: fibertype = p['DPRTYPE'].split('_')[1] # apply thermal correction based on fiber type if fibertype in p['THERMAL_CORRECTION_TYPE1']: # log progress wmsg = 'Correcting thermal background for {0}={1} mode={2}' wargs = [fiber, fibertype, 1] WLOG(p, 'info', wmsg.format(*wargs)) # correct E2DS tkwargs = dict(image=loc['E2DS'], mode=1, fiber=fiber, hdr=hdr) p, loc['E2DS'] = spirouBACK.ThermalCorrect(p, **tkwargs) # correct E2DSFF tkwargs = dict(image=loc['E2DSFF'], mode=1, fiber=fiber, hdr=hdr, flat=loc['FLAT']) p, loc['E2DSFF'] = spirouBACK.ThermalCorrect(p, **tkwargs) elif fibertype in p['THERMAL_CORRECTION_TYPE2']: # log progress wmsg = 'Correcting thermal background for {0}={1} mode={2}' wargs = [fiber, fibertype, 2] WLOG(p, 'info', wmsg.format(*wargs)) # correct E2DS tkwargs = dict(image=loc['E2DS'], mode=2, fiber=fiber, hdr=hdr) p, loc['E2DS'] = spirouBACK.ThermalCorrect(p, **tkwargs) # correct E2DSFF tkwargs = dict(image=loc['E2DSFF'], mode=2, fiber=fiber, hdr=hdr, flat=loc['FLAT']) p, loc['E2DSFF'] = spirouBACK.ThermalCorrect(p, **tkwargs) else: # log progress wmsg = 'Not correcting thermal background for {0}={1}' wargs = [fiber, fibertype] WLOG(p, 'info', wmsg.format(*wargs)) # set filename for output outfile = 'THERMALFILE_{0}'.format(fiber) p[outfile] = 'None' p.set_source(outfile, __NAME__ + '.main()') # ------------------------------------------------------------------ # Plots # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot all orders or one order if p['IC_FF_PLOT_ALL_ORDERS']: # plot image with all order fits (slower) sPlt.ext_aorder_fit(p, loc, data1, max_signal / 10.) else: # plot image with selected order fit and edge fit (faster) sPlt.ext_sorder_fit(p, loc, data1, max_signal / 10.) # plot e2ds against wavelength sPlt.ext_spectral_order_plot(p, loc) if p['IC_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES: sPlt.ext_debanana_plot(p, loc, data2, max_signal / 10.) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # if array is completely NaNs it shouldn't pass if np.sum(np.isfinite(loc['E2DS'])) == 0: fail_msg.append('E2DS image is all NaNs') passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append('NaN') qc_names.append('image') qc_logic.append('image is all NaN') # ---------------------------------------------------------------------- # saturation check: check that the max_signal is lower than # qc_max_signal max_qcflux = p['QC_MAX_SIGNAL'] * p['NBFRAMES'] if max_signal > max_qcflux: fmsg = 'Too much flux in the image ({0:.2f} > {1:.2f})' fail_msg.append(fmsg.format(max_signal, max_qcflux)) passed = False # Question: Why is this test ignored? # For some reason this test is ignored in old code passed = True WLOG(p, 'info', fail_msg[-1]) qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(max_signal) qc_names.append('max_signal') qc_logic.append('QC_MAX_SIGNAL > {0:.3f}'.format(max_qcflux)) # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Store extraction in file(s) # ------------------------------------------------------------------ raw_ext_file = os.path.basename(p['FITSFILENAME']) # construct filename e2dsfits, tag1 = spirouConfig.Constants.EXTRACT_E2DS_FILE(p) e2dsfitsname = os.path.split(e2dsfits)[-1] e2dsfffits, tag2 = spirouConfig.Constants.EXTRACT_E2DSFF_FILE(p) e2dsfffitsname = os.path.split(e2dsfffits)[-1] e2dsllfits, tag4 = spirouConfig.Constants.EXTRACT_E2DSLL_FILE(p) e2dsfllitsname = os.path.split(e2dsllfits)[-1] # log that we are saving E2DS spectrum wmsg = 'Saving E2DS spectrum of Fiber {0} in {1}' WLOG(p, '', wmsg.format(p['FIBER'], e2dsfitsname)) wmsg = 'Saving E2DSFF spectrum of Fiber {0} in {1}' WLOG(p, '', wmsg.format(p['FIBER'], e2dsfffitsname)) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_FIBER'], value=p['FIBER']) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBACK'], value=p['BKGRDFILE']) if p['IC_EXTRACT_TYPE'] not in EXTRACT_SHAPE_TYPES: hdict = spirouImage.AddKey(p, hdict, p['KW_CDBTILT'], value=p['TILTFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFLAT'], value=p['FLATFILE']) if p['IC_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES: hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEX'], value=p['SHAPEXFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEY'], value=p['SHAPEYFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPE'], value=p['SHAPEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFPMASTER'], value=p['FPMASTERFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBTHERMAL'], value=p['THERMALFILE_{0}'.format(fiber)]) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # construct loco filename locofile, _ = spirouConfig.Constants.EXTRACT_LOCO_FILE(p) locofilename = os.path.basename(locofile) # add barycentric keys to header hdict = spirouImage.AddKey(p, hdict, p['KW_BERV'], value=loc['BERV']) hdict = spirouImage.AddKey(p, hdict, p['KW_BJD'], value=loc['BJD']) hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_MAX'], value=loc['BERV_MAX']) hdict = spirouImage.AddKey(p, hdict, p['KW_B_OBS_HOUR'], value=loc['BERVHOUR']) # add barycentric estimate keys to header hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_EST'], value=loc['BERV_EST']) hdict = spirouImage.AddKey(p, hdict, p['KW_BJD_EST'], value=loc['BJD_EST']) hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_MAX_EST'], value=loc['BERV_MAX_EST']) hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_SOURCE'], value=loc['BERV_SOURCE']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # copy extraction method and function to header # (for reproducibility) hdict = spirouImage.AddKey(p, hdict, p['KW_E2DS_EXTM'], value=extmethod) hdict = spirouImage.AddKey(p, hdict, p['KW_E2DS_FUNC'], value=extfunc) # add localization file name to header hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_FILE'], value=locofilename) # add wave solution date hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME1'], value=loc['WAVE_ACQTIMES'][0]) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME2'], value=loc['WAVE_ACQTIMES'][1]) # add wave solution number of orders hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_ORD_N'], value=loc['WAVEPARAMS'].shape[0]) # add wave solution degree of fit hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_LL_DEG'], value=loc['WAVEPARAMS'].shape[1] - 1) # ------------------------------------------------------------------------- # add keys of the wave solution FP CCF hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_FILE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_DRIFT'], value=p['WFP_DRIFT']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_FWHM'], value=p['WFP_FWHM']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_CONTRAST'], value=p['WFP_CONTRAST']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_MAXCPP'], value=p['WFP_MAXCPP']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_MASK'], value=p['WFP_MASK']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_LINES'], value=p['WFP_LINES']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_TARG_RV'], value=p['WFP_TARG_RV']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_WIDTH'], value=p['WFP_WIDTH']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_STEP'], value=p['WFP_STEP']) # write 1D list of the SNR hdict = spirouImage.AddKey1DList(p, hdict, p['KW_E2DS_SNR'], values=loc['SNR']) # add localization file keys to header root = p['KW_ROOT_DRS_LOC'][0] hdict = spirouImage.CopyRootKeys(p, hdict, locofile, root=root) # add wave solution coefficients hdict = spirouImage.AddKey2DList(p, hdict, p['KW_WAVE_PARAM'], values=loc['WAVEPARAMS']) # Save E2DS file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) p = spirouImage.WriteImage(p, e2dsfits, loc['E2DS'], hdict) # Save E2DSFF file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) p = spirouImage.WriteImage(p, e2dsfffits, loc['E2DSFF'], hdict) # Save E2DSLL file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES: llstack = np.vstack(loc['E2DSLL']) p = spirouImage.WriteImage(p, e2dsllfits, llstack, hdict) # ------------------------------------------------------------------ # 1-dimension spectral S1D (uniform in wavelength) # ------------------------------------------------------------------ # get arguments for E2DS to S1D e2dsargs = [loc['WAVE'], loc['E2DSFF'], loc['BLAZE']] # get 1D spectrum xs1d1, ys1d1 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='wave') # Plot the 1D spectrum if p['DRS_PLOT'] > 0: sPlt.ext_1d_spectrum_plot(p, xs1d1, ys1d1) # construct file name s1dfile1, tag3 = spirouConfig.Constants.EXTRACT_S1D_FILE1(p) s1dfilename1 = os.path.basename(s1dfile1) # add header keys # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) # log writing to file wmsg = 'Saving 1D spectrum (uniform in wavelength) for Fiber {0} in {1}' WLOG(p, '', wmsg.format(p['FIBER'], s1dfilename1)) # Write to file columns = ['wavelength', 'flux', 'eflux'] values = [xs1d1, ys1d1, np.zeros_like(ys1d1)] units = ['nm', None, None] s1d1 = spirouImage.MakeTable(p, columns, values, units=units) spirouImage.WriteTable(p, s1d1, s1dfile1, header=hdict) # ------------------------------------------------------------------ # 1-dimension spectral S1D (uniform in velocity) # ------------------------------------------------------------------ # get arguments for E2DS to S1D e2dsargs = [loc['WAVE'], loc['E2DSFF'], loc['BLAZE']] # get 1D spectrum xs1d2, ys1d2 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='velocity') # Plot the 1D spectrum if p['DRS_PLOT'] > 0: sPlt.ext_1d_spectrum_plot(p, xs1d2, ys1d2) # construct file name s1dfile2, tag4 = spirouConfig.Constants.EXTRACT_S1D_FILE2(p) s1dfilename2 = os.path.basename(s1dfile2) # add header keys hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) hdict = spirouImage.AddKey(p, hdict, p['KW_EXT_TYPE'], value=p['DPRTYPE']) # log writing to file wmsg = 'Saving 1D spectrum (uniform in velocity) for Fiber {0} in {1}' WLOG(p, '', wmsg.format(p['FIBER'], s1dfilename2)) # Write to file columns = ['wavelength', 'flux', 'eflux'] values = [xs1d2, ys1d2, np.zeros_like(ys1d2)] units = ['nm', None, None] s1d2 = spirouImage.MakeTable(p, columns, values, units=units) spirouImage.WriteTable(p, s1d2, s1dfile2, header=hdict) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, ufiles=None): """ cal_preprocess_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_preprocess_spirou.py [night_directory] [fitsfilename] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param ufiles: string, list or None, the list of files to process Note can include wildcard i.e. "*.fits" (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # need custom args (to accept full path or wild card if ufiles is None: names, types = ['ufiles'], [str] customargs = spirouStartup.GetCustomFromRuntime(p, [0], types, names, last_multi=True) else: customargs = dict(ufiles=ufiles) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsdir='raw') # ---------------------------------------------------------------------- # Get hot pixels for corruption check # ---------------------------------------------------------------------- hotpixels = spirouImage.PPGetHotPixels(p) # ---------------------------------------------------------------------- # Process files (including wildcards) # ---------------------------------------------------------------------- # get raw folder (assume all files are in the root directory) rawdir = spirouConfig.Constants.RAW_DIR(p) try: ufiles = spirouFile.Paths(p['UFILES'], root=rawdir).abs_paths except PathException as e: WLOG(p, 'error', e) # log how many files were found wmsg = '{0} files found' WLOG(p, '', wmsg.format(len(ufiles))) # storage for output files p['OUTPUT_NAMES'] = [] p.set_source('OUTPUT_NAMES', __NAME__ + '.main()') # loop around files for u_it, ufile in enumerate(ufiles): # log the file process wmsg = 'Processing file {0} ({1} of {2})' WLOG(p, '', spirouStartup.spirouStartup.HEADER) bfilename = os.path.basename(ufile) WLOG(p, 'info', wmsg.format(bfilename, u_it+1, len(ufiles))) WLOG(p, '', spirouStartup.spirouStartup.HEADER) # ------------------------------------------------------------------ # Check that we can process file # ------------------------------------------------------------------ # check if ufile exists if not os.path.exists(ufile): wmsg = 'File {0} does not exist... skipping' WLOG(p, 'warning', wmsg.format(ufile)) continue # skip processed files elif p['PROCESSED_SUFFIX'] in bfilename: wmsg = 'File {0} has been processed... skipping' WLOG(p, 'warning', wmsg.format(ufile)) continue # skip non-fits files elif '.fits' not in bfilename: wmsg = 'File {0} not a fits file... skipping' WLOG(p, 'warning', wmsg.format(ufile)) continue # skip index file elif bfilename == spirouConfig.Constants.INDEX_OUTPUT_FILENAME(): wmsg = 'Skipping index fits file' WLOG(p, 'warning', wmsg.format(ufile)) continue # ------------------------------------------------------------------ # Read image file # ------------------------------------------------------------------ # read the image data rout = spirouImage.ReadImage(p, filename=ufile) image, hdr, nx, ny = rout # ------------------------------------------------------------------ # Identify file (and update filename, header and comments) # ------------------------------------------------------------------ ufile, hdr = spirouImage.IdentifyUnProFile(p, ufile, hdr) # ------------------------------------------------------------------ # correct image # ------------------------------------------------------------------ # correct for the top and bottom reference pixels WLOG(p, '', 'Correcting for top and bottom pixels') image = spirouImage.PPCorrectTopBottom(p, image) # correct by a median filter from the dark amplifiers wmsg = 'Correcting by the median filter from dark amplifiers' WLOG(p, '', wmsg) image = spirouImage.PPMedianFilterDarkAmps(p, image) # correct for the 1/f noise wmsg = 'Correcting for the 1/f noise' WLOG(p, '', wmsg) image = spirouImage.PPMedianOneOverfNoise2(p, image) # ------------------------------------------------------------------ # Quality control to check for corrupt files # ------------------------------------------------------------------ # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # get pass condition cout = spirouImage.PPTestForCorruptFile(p, image, hotpixels) snr_hotpix, rms_list = cout # print out SNR hotpix value wmsg = 'Corruption check: SNR Hotpix value = {0:.5e}' WLOG(p, '', wmsg.format(snr_hotpix)) #deal with printing corruption message if snr_hotpix < p['PP_CORRUPT_SNR_HOTPIX']: # add failed message to fail message list fargs = [snr_hotpix, p['PP_CORRUPT_SNR_HOTPIX'],ufile ] fmsg = ('File was found to be corrupted. (SNR_HOTPIX < threshold, ' '{0:.4e} < {1:.4e}). File will not be saved. ' 'File = {2}'.format(*fargs)) fail_msg.append(fmsg) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(snr_hotpix) qc_names.append('snr_hotpix') qc_logic.append('snr_hotpix < {0:.5e}' ''.format(p['PP_CORRUPT_SNR_HOTPIX'])) # ---------------------------------------------------------------------- if np.max(rms_list) > p['PP_CORRUPT_RMS_THRES']: # add failed message to fail message list fargs = [np.max(rms_list), p['PP_CORRUPT_RMS_THRES'], ufile] fmsg = ('File was found to be corrupted. (RMS < threshold, ' '{0:.4e} > {1:.4e}). File will not be saved. ' 'File = {0}'.format(*fargs)) fail_msg.append(fmsg) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(np.max(rms_list)) qc_names.append('max(rms_list)') qc_logic.append('max(rms_list) > {0:.4e}' ''.format(p['PP_CORRUPT_RMS_THRES'])) # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') WLOG(p, 'warning', '\tFile not written') continue # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # rotate image # ------------------------------------------------------------------ # rotation to match HARPS orientation (expected by DRS) image = spirouImage.RotateImage(image, p['RAW_TO_PP_ROTATION']) # ------------------------------------------------------------------ # Save rotated image # ------------------------------------------------------------------ # construct rotated file name outfits = spirouConfig.Constants.PP_FILE(p, bfilename) outfitsname = os.path.basename(outfits) # log that we are saving rotated image WLOG(p, '', 'Saving Rotated Image in ' + outfitsname) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_PPVERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) # set the inputs hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=[os.path.basename(ufile)]) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_DRS_QC_NAME'], values=qc_names) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # set the DRS type (for file indexing) p['DRS_TYPE'] = 'RAW' p.set_source('DRS_TYPE', __NAME__ + '.main()') # write to file p = spirouImage.WriteImage(p, outfits, image, hdict) # index this file p = spirouStartup.End(p, outputs='pp', end=False) # ------------------------------------------------------------------ # append to output storage in p # ------------------------------------------------------------------ p['OUTPUT_NAMES'].append(outfitsname) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p, outputs=None) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, reffile=None): """ cal_DRIFTPEAK_E2DS_spirou.py main function, if arguments are None uses arguments from run time i.e.: cal_DRIFTPEAK_E2DS_spirou.py [night_directory] [reffile] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param reffile: string, the reference file to use :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with reference file being None (i.e. get from sys.argv) if reffile is None: customargs = spirouStartup.GetCustomFromRuntime( p, [0], [str], ['reffile']) else: customargs = dict(reffile=reffile) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='reffile', mainfitsdir='reduced') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, reffilename = spirouStartup.SingleFileSetup(p, filename=p['REFFILE']) p['REFFILENAME'] = reffilename p.set_source('REFFILENAME', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data speref, hdr, nbo, nx = spirouImage.ReadData(p, p['REFFILENAME']) # add to loc loc = ParamDict() loc['SPEREF'] = speref loc['NUMBER_ORDERS'] = nbo loc.set_sources(['SPEREF', 'NUMBER_ORDERS'], __NAME__ + '/main()') # ---------------------------------------------------------------------- # Get lamp type # ---------------------------------------------------------------------- # get lamp type if p['KW_EXT_TYPE'][0] in hdr: ext_type = hdr[p['KW_EXT_TYPE'][0]] drift_types = p['DRIFT_PEAK_ALLOWED_TYPES'].keys() found = False for kind in drift_types: if ext_type == kind: loc['LAMP'] = p['DRIFT_PEAK_ALLOWED_TYPES'][kind] found = True if not found: eargs1 = [p['KW_EXT_TYPE'][0], ' or '.join(drift_types)] emsg1 = ( 'Wrong type of image for Drift, header key "{0}" should be' '{1}'.format(*eargs1)) emsg2 = '\tPlease check DRIFT_PEAK_ALLOWED_TYPES' WLOG(p, 'error', [emsg1, emsg2]) else: emsg = 'Header key = "{0}" missing from file {1}' eargs = [p['KW_EXT_TYPE'][0], p['REFFILENAME']] WLOG(p, 'error', emsg.format(*eargs)) loc.set_source('LAMP', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # get acquisition time p = spirouImage.GetAcqTime(p, hdr, name='acqtime', kind='julian') bjdref = p['ACQTIME'] # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # ---------------------------------------------------------------------- # Read wavelength solution # ---------------------------------------------------------------------- # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get wave image source = __NAME__ + '/main() + /spirouImage.GetWaveSolution' wout = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True, return_filename=True, fiber=wave_fiber) _, loc['WAVE'], loc['WAVEFILE'], loc['WSOURCE'] = wout loc.set_sources(['WAVE', 'WAVEFILE', 'WSOURCE'], source) # ---------------------------------------------------------------------- # Read Flat file # ---------------------------------------------------------------------- # get flat p, loc['FLAT'] = spirouImage.ReadFlatFile(p, hdr) loc.set_source('FLAT', __NAME__ + '/main() + /spirouImage.ReadFlatFile') # get all values in flat that are zero to 1 loc['FLAT'] = np.where(loc['FLAT'] == 0, 1.0, loc['FLAT']) # correct for flat file loc['SPEREF'] = loc['SPEREF'] / loc['FLAT'] # ---------------------------------------------------------------------- # Background correction # ---------------------------------------------------------------------- # test whether we want to subtract background if p['IC_DRIFT_BACK_CORR']: # Loop around the orders for order_num in range(loc['NUMBER_ORDERS']): # get the box size from constants bsize = p['DRIFT_PEAK_MINMAX_BOXSIZE'] # Measurethe min and max flux miny, maxy = spirouBACK.MeasureMinMax(loc['SPEREF'][order_num], bsize) # subtract off the background (miny) loc['SPEREF'][order_num] = loc['SPEREF'][order_num] - miny # ---------------------------------------------------------------------- # Identify FP peaks in reference file # ---------------------------------------------------------------------- # log that we are identifying peaks wmsg = 'Identification of lines in reference file: {0}' WLOG(p, '', wmsg.format(p['REFFILE'])) # get the position of FP peaks from reference file loc = spirouRV.CreateDriftFile(p, loc) # ---------------------------------------------------------------------- # Removal of suspiciously wide FP lines # ---------------------------------------------------------------------- loc = spirouRV.RemoveWidePeaks(p, loc) # ---------------------------------------------------------------------- # Get reference drift # ---------------------------------------------------------------------- # are we using gaussfit? gaussfit = p['DRIFT_PEAK_GETDRIFT_GAUSSFIT'] # get drift loc['XREF'] = spirouRV.GetDrift(p, loc['SPEREF'], loc['ORDPEAK'], loc['XPEAK'], gaussfit=gaussfit) loc.set_source('XREF', __NAME__ + '/main()') # remove any drifts that are zero (i.e. peak not measured loc = spirouRV.RemoveZeroPeaks(p, loc) # ------------------------------------------------------------------ # Reference plots # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot FP spectral order sPlt.drift_plot_selected_wave_ref(p, loc) # ------------------------------------------------------------------ # Get all other files that match kw_OUTPUT and kw_EXT_TYPE from # ref file # ------------------------------------------------------------------ # get files listfiles, listtypes = spirouImage.GetSimilarDriftFiles(p, hdr) # get the number of files nfiles = len(listfiles) # Log the number of files found wmsgs = [ 'Number of files found on directory = {0}'.format(nfiles), '\tExtensions allowed:' ] for listtype in listtypes: wmsgs.append('\t\t - {0}'.format(listtype)) WLOG(p, 'info', wmsgs) # ------------------------------------------------------------------ # Set up Extract storage for all files # ------------------------------------------------------------------ # decide whether we need to skip (for large number of files) if len(listfiles) >= p['DRIFT_NLARGE']: skip = p['DRIFT_PEAK_FILE_SKIP'] nfiles = int(nfiles / skip) else: skip = 1 # set up storage loc['DRIFT'] = np.zeros((nfiles, loc['NUMBER_ORDERS'])) loc['DRIFT_LEFT'] = np.zeros((nfiles, loc['NUMBER_ORDERS'])) loc['DRIFT_RIGHT'] = np.zeros((nfiles, loc['NUMBER_ORDERS'])) loc['ERRDRIFT'] = np.zeros((nfiles, loc['NUMBER_ORDERS'])) loc['DELTATIME'] = np.zeros(nfiles) loc['MEANRV'] = np.zeros(nfiles) loc['MEANRV_LEFT'] = np.zeros(nfiles) loc['MEANRV_RIGHT'] = np.zeros(nfiles) loc['MERRDRIFT'] = np.zeros(nfiles) loc['FLUXRATIO'] = np.zeros(nfiles) # add sources source = __NAME__ + '/main()' keys = [ 'drift', 'drift_left', 'drift_right', 'errdrift', 'deltatime', 'meanrv', 'meanrv_left', 'meanrv_right', 'merrdrift', 'fluxratio' ] loc.set_sources(keys, source) # ------------------------------------------------------------------ # Loop around all files: correct for dark, reshape, extract and # calculate dvrms and meanpond # ------------------------------------------------------------------ # get the maximum number of orders to use nomin = p['IC_DRIFT_PEAK_N_ORDER_MIN'] nomax = p['IC_DRIFT_PEAK_N_ORDER_MAX'] # loop around files for i_it in range(nfiles): # get file for this iteration fpfile = listfiles[::skip][i_it] # Log the file we are reading wmsg = 'Reading file {0}' WLOG(p, '', wmsg.format(os.path.split(fpfile)[-1])) # ------------------------------------------------------------------ # read e2ds files and get timestamp # ------------------------------------------------------------------ # read data rout = spirouImage.ReadData(p, filename=fpfile, log=False) loc['SPE'], hdri, nxi, nyi = rout # apply flat loc['SPE'] = loc['SPE'] / loc['FLAT'] # get acqtime bjdspe = spirouImage.GetAcqTime(p, hdri, name='acqtime', return_value=1, kind='julian') # ---------------------------------------------------------------------- # Background correction # ---------------------------------------------------------------------- # test whether we want to subtract background if p['IC_DRIFT_BACK_CORR']: # Loop around the orders for order_num in range(loc['NUMBER_ORDERS']): # get the box size from constants bsize = p['DRIFT_PEAK_MINMAX_BOXSIZE'] # Measurethe min and max flux miny, maxy = spirouBACK.MeasureMinMax(loc['SPE'][order_num], bsize) # subtract off the background (miny) loc['SPE'][order_num] = loc['SPE'][order_num] - miny # ------------------------------------------------------------------ # calculate flux ratio # ------------------------------------------------------------------ sorder = p['IC_DRIFT_ORDER_PLOT'] fratio = np.nansum(loc['SPE'][sorder]) / np.nansum( loc['SPEREF'][sorder]) loc['FLUXRATIO'][i_it] = fratio # ------------------------------------------------------------------ # Calculate delta time # ------------------------------------------------------------------ # calculate the time from reference (in hours) loc['DELTATIME'][i_it] = (bjdspe - bjdref) * 24 # ------------------------------------------------------------------ # Calculate PearsonR coefficient # ------------------------------------------------------------------ pargs = [loc['NUMBER_ORDERS'], loc['SPE'], loc['SPEREF']] correlation_coeffs = spirouRV.PearsonRtest(*pargs) # ---------------------------------------------------------------------- # Get drift with comparison to the reference image # ---------------------------------------------------------------------- # only calculate drift if the correlation between orders and # reference file is above threshold prcut = p['DRIFT_PEAK_PEARSONR_CUT'] if np.min(correlation_coeffs[nomin:nomax]) > prcut: # get drifts for each order dargs = [p, loc['SPE'], loc['ORDPEAK'], loc['XREF']] x = spirouRV.GetDrift(*dargs, gaussfit=gaussfit) # get delta v loc['DV'] = (x - loc['XREF']) * loc['VRPEAK'] # sigma clip loc = spirouRV.SigmaClip(loc, sigma=p['DRIFT_PEAK_SIGMACLIP']) # work out median drifts per order loc = spirouRV.DriftPerOrder(loc, i_it) # work out mean drift across all orders loc = spirouRV.DriftAllOrders(p, loc, i_it, nomin, nomax) # log the mean drift wmsg = ('Time from ref= {0:.2f} h - Flux Ratio= {1:.3f} ' '- Drift mean= {2:.2f} +- {3:.2f} m/s') wargs = [ loc['DELTATIME'][i_it], loc['FLUXRATIO'][i_it], loc['MEANRV'][i_it], loc['MERRDRIFT'][i_it] ] WLOG(p, 'info', wmsg.format(*wargs)) # else we can't use this extract else: if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.plt.ioff() # plot comparison between spe and ref sPlt.drift_plot_correlation_comp(p, loc, correlation_coeffs, i_it) sPlt.plt.show() sPlt.plt.close() # turn interactive plotting back on sPlt.plt.ion() # log that we cannot use this extraction wmsg1 = 'The correlation of some orders compared to the template is' wmsg2 = ' < {0}, something went wrong in the extract.' WLOG(p, 'warning', wmsg1) WLOG(p, 'warning', wmsg2.format(prcut)) # ------------------------------------------------------------------ # peak to peak drift driftptp = np.max(loc['MEANRV']) - np.min(loc['MEANRV']) driftrms = np.std(loc['MEANRV']) # log th etotal drift peak-to-peak and rms wmsg = ('Total drift Peak-to-Peak={0:.3f} m/s RMS={1:.3f} m/s in ' '{2:.2f} hour') wargs = [driftptp, driftrms, np.max(loc['DELTATIME'])] WLOG(p, '', wmsg.format(*wargs)) # ------------------------------------------------------------------ # Plot of mean drift # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot delta time against median drift sPlt.drift_peak_plot_dtime_against_drift(p, loc) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # TODO: Needs doing # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # add to qc header lists qc_values.append('None') qc_names.append('None') qc_logic.append('None') qc_pass.append(1) # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Save drift values to file # ------------------------------------------------------------------ # get raw input file name raw_infile = os.path.basename(p['REFFILE']) # construct filename driftfits, tag = spirouConfig.Constants.DRIFTPEAK_E2DS_FITS_FILE(p) driftfitsname = os.path.split(driftfits)[-1] # log that we are saving drift values wmsg = 'Saving drift values of Fiber {0} in {1}' WLOG(p, '', wmsg.format(p['FIBER'], driftfitsname)) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFLAT'], value=p['FLATFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_REFFILE'], value=raw_infile) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # save drift values p = spirouImage.WriteImage(p, driftfits, loc['DRIFT'], hdict) # ------------------------------------------------------------------ # print .tbl result # ------------------------------------------------------------------ # construct filename drifttbl = spirouConfig.Constants.DRIFTPEAK_E2DS_TBL_FILE(p) drifttblname = os.path.split(drifttbl)[-1] # construct and write table columnnames = ['time', 'drift', 'drifterr', 'drift_left', 'drift_right'] columnformats = ['7.4f', '6.2f', '6.3f', '6.2f', '6.2f'] columnvalues = [ loc['DELTATIME'], loc['MEANRV'], loc['MERRDRIFT'], loc['MEANRV_LEFT'], loc['MEANRV_RIGHT'] ] table = spirouImage.MakeTable(p, columns=columnnames, values=columnvalues, formats=columnformats) # write table wmsg = 'Average Drift saved in {0} Saved ' WLOG(p, '', wmsg.format(drifttblname)) spirouImage.WriteTable(p, table, drifttbl, fmt='ascii.rst') # ------------------------------------------------------------------ # Plot amp and llpeak # ------------------------------------------------------------------ if p['DRS_PLOT'] and p['DRIFT_PEAK_PLOT_LINE_LOG_AMP']: # start interactive session if needed sPlt.start_interactive_session(p) # plot delta time against median drift sPlt.drift_peak_plot_llpeak_amps(p, loc) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): """ cal_SLIT_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_SLIT_spirou.py [night_directory] [files] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) p = spirouStartup.InitialFileSetup(p, calibdb=True) # set the fiber type p['FIBER'] = 'AB' p.set_source('FIBER', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add') # ---------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- data = spirouImage.FixNonPreProcess(p, data) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # ---------------------------------------------------------------------- # Correction of DARK # ---------------------------------------------------------------------- p, datac = spirouImage.CorrectForDark(p, data, hdr) datac = data # ---------------------------------------------------------------------- # Resize image # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p) # convert NaN to zeros data0 = np.where(~np.isfinite(data), np.zeros_like(data), data) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) data2 = spirouImage.ResizeImage(p, data0, **bkwargs) # log change in data size WLOG(p, '', ('Image format changed to ' '{0}x{1}').format(*data2.shape)) # ---------------------------------------------------------------------- # Correct for the BADPIX mask (set all bad pixels to zero) # ---------------------------------------------------------------------- p, data2 = spirouImage.CorrectForBadPix(p, data2, hdr) p, badpixmap = spirouImage.CorrectForBadPix(p, data2, hdr, return_map=True) # ---------------------------------------------------------------------- # Background computation # ---------------------------------------------------------------------- if p['IC_DO_BKGR_SUBTRACTION']: # log that we are doing background measurement WLOG(p, '', 'Doing background measurement on raw frame') # get the bkgr measurement bargs = [p, data2, hdr, badpixmap] # background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs) p, background = spirouBACK.MeasureBackgroundMap(*bargs) else: background = np.zeros_like(data2) p['BKGRDFILE'] = 'None' p.set_source('BKGRDFILE', __NAME__ + '.main()') # apply background correction to data data2 = data2 - background # save data to loc loc = ParamDict() loc['DATA'] = data2 loc.set_source('DATA', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- # get the number of bad pixels n_bad_pix = np.nansum(data2 <= 0) n_bad_pix_frac = n_bad_pix * 100 / np.product(data2.shape) # Log number wmsg = 'Nb dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ------------------------------------------------------------------ # Get localisation coefficients # ------------------------------------------------------------------ # original there is a loop but it is not used --> removed p = spirouImage.FiberParams(p, p['FIBER'], merge=True) # get localisation fit coefficients p, loc = spirouLOCOR.GetCoeffs(p, hdr, loc) # ------------------------------------------------------------------ # Calculate shape map # ------------------------------------------------------------------ loc = spirouImage.GetShapeMap(p, loc) # ------------------------------------------------------------------ # Plotting # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # plots setup: start interactive plot sPlt.start_interactive_session(p) # plot the shape process for each order sPlt.slit_shape_angle_plot(p, loc) # end interactive section sPlt.end_interactive_session(p) # ------------------------------------------------------------------ # Writing to file # ------------------------------------------------------------------ # get the raw tilt file name raw_shape_file = os.path.basename(p['FITSFILENAME']) # construct file name and path shapefits, tag = spirouConfig.Constants.SLIT_XSHAPE_FILE(p) shapefitsname = os.path.basename(shapefits) # Log that we are saving tilt file wmsg = 'Saving shape information in file: {0}' WLOG(p, '', wmsg.format(shapefitsname)) # Copy keys from fits file # Copy keys from fits file hdict = spirouImage.CopyOriginalKeys(hdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPE'], value=raw_shape_file) # write tilt file to file p = spirouImage.WriteImage(p, shapefits, loc['DXMAP'], hdict) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # TODO: Decide on some quality control criteria? # set passed variable and fail message list passed, fail_msg = True, [] # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Move to calibDB and update calibDB # ---------------------------------------------------------------------- if p['QC']: keydb = 'SHAPE' # copy shape file to the calibDB folder spirouDB.PutCalibFile(p, shapefits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, shapefitsname, hdr) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, e2dsfile=None, mask=None, rv=None, width=None, step=None): """ cal_CCF_E2DS_spirou.py main function, if arguments are None uses arguments from run time i.e.: cal_CCF_E2DS_spirou.py [night_directory] [E2DSfilename] [mask] [RV] [width] [step] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param e2dsfile: string, the E2DS file to use :param mask: string, the mask file to use (i.e. "UrNe.mas") :param rv: float, the target RV to use :param width: float, the CCF width to use :param step: float, the CCF step to use :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with arguments being None (i.e. get from sys.argv) pos = [0, 1, 2, 3, 4] fmt = [str, str, float, float, float] name = ['e2dsfile', 'ccf_mask', 'target_rv', 'ccf_width', 'ccf_step'] lname = ['input_file', 'CCF_mask', 'RV', 'CCF_width', 'CCF_step'] req = [True, True, True, False, False] call = [e2dsfile, mask, rv, width, step] call_priority = [True, True, True, True, True] # now get custom arguments customargs = spirouStartup.GetCustomFromRuntime(p, pos, fmt, name, req, call, call_priority, lname) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='e2dsfile', mainfitsdir='reduced') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, e2dsfilename = spirouStartup.SingleFileSetup(p, filename=p['E2DSFILE']) # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Deal with optional run time arguments # ---------------------------------------------------------------------- # define default arguments (if ccf_width and ccf_step are not defined # in function call or run time arguments if 'ccf_width' not in p: p['CCF_WIDTH'] = p['IC_CCF_WIDTH'] if 'ccf_step' not in p: p['CCF_STEP'] = p['IC_CCF_STEP'] # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data e2ds, hdr, nbo, nx = spirouImage.ReadData(p, e2dsfilename) # add to loc loc = ParamDict() loc['E2DS'] = e2ds loc['NUMBER_ORDERS'] = nbo loc.set_sources(['E2DS', 'number_orders'], __NAME__ + '/main()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # get acquisition time p = spirouImage.GetAcqTime(p, hdr, name='acqtime', kind='julian') # get obj name p = spirouImage.ReadParam(p, hdr, 'KW_OBJNAME', name='OBJNAME', dtype=str) bjdref = p['ACQTIME'] # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # ---------------------------------------------------------------------- # Earth Velocity calculation # ---------------------------------------------------------------------- if p['IC_IMAGE_TYPE'] == 'H4RG': p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, hdr) # ---------------------------------------------------------------------- # Read wavelength solution # ---------------------------------------------------------------------- # log WLOG(p, '', 'Reading wavelength solution ') # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get wave image wout = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True, return_filename=True, fiber=wave_fiber) param_ll, wave_ll, wavefile, wsource = wout # save to storage loc['PARAM_LL'], loc['WAVE_LL'], loc['WAVEFILE'], loc['WSOURCE'] = wout source = __NAME__ + '/main() + spirouTHORCA.GetWaveSolution()' loc.set_sources(['WAVE_LL', 'PARAM_LL', 'WAVEFILE', 'WSOURCE'], source) # ---------------------------------------------------------------------- # Read Flat file # ---------------------------------------------------------------------- # TODO We do not need to correct FLAT # log # WLOG(p, '', 'Reading Flat-Field ') # get flat # loc['FLAT'] = spirouImage.ReadFlatFile(p, hdr) # loc.set_source('FLAT', __NAME__ + '/main() + /spirouImage.ReadFlatFile') # get all values in flat that are zero to 1 # loc['FLAT'] = np.where(loc['FLAT'] == 0, 1.0, loc['FLAT']) # get blaze # p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hdr) p, blaze0 = spirouImage.ReadBlazeFile(p, hdr) # ---------------------------------------------------------------------- # Preliminary set up = no flat, no blaze # ---------------------------------------------------------------------- # reset flat to all ones # loc['FLAT'] = np.ones((nbo, nx)) # set blaze to all ones (if not bug in correlbin !!! # TODO Check why Blaze makes bugs in correlbin loc['BLAZE'] = np.ones((nbo, nx)) # set sources # loc.set_sources(['flat', 'blaze'], __NAME__ + '/main()') loc.set_sources(['blaze'], __NAME__ + '/main()') # Modification of E2DS array with N.A.N if np.isnan(np.sum(e2ds)): WLOG(p, 'warning', 'NaN values found in e2ds, converting process') # First basic approach Replacing N.A.N by zeros # e2ds[np.isnan(e2ds)] = 0 # Second approach replacing N.A.N by the Adjusted Blaze e2dsb = e2ds / blaze0 for i in np.arange(len(e2ds)): with warnings.catch_warnings(record=True) as _: rap = np.mean(e2dsb[i][np.isfinite(e2dsb[i])]) if np.isnan(rap): rap = 0.0 e2ds[i] = np.where(np.isfinite(e2dsb[i]), e2ds[i], blaze0[i] * rap) # ---------------------------------------------------------------------- # correct extracted image for flat # ---------------------------------------------------------------------- # loc['E2DSFF'] = e2ds/loc['FLAT'] # loc['E2DSFF'] = e2ds*1. loc['E2DSFF'] = e2ds loc.set_source('E2DSFF', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Compute photon noise uncertainty for reference file # ---------------------------------------------------------------------- # set up the arguments for DeltaVrms2D dargs = [loc['E2DS'], loc['WAVE_LL']] dkwargs = dict(sigdet=p['IC_DRIFT_NOISE'], size=p['IC_DRIFT_BOXSIZE'], threshold=p['IC_DRIFT_MAXFLUX']) # run DeltaVrms2D dvrmsref, wmeanref = spirouRV.DeltaVrms2D(*dargs, **dkwargs) # save to loc loc['DVRMSREF'], loc['WMEANREF'] = dvrmsref, wmeanref loc.set_sources(['dvrmsref', 'wmeanref'], __NAME__ + '/main()()') # log the estimated RV uncertainty # wmsg = 'On fiber {0} estimated RV uncertainty on spectrum is {1:.3f} m/s' # WLOG(p, 'info', wmsg.format(p['FIBER'], wmeanref)) wmsg = 'On fiber estimated RV uncertainty on spectrum is {0:.3f} m/s' WLOG(p, 'info', wmsg.format(wmeanref)) # TEST N.A.N # loc['E2DSFF'][20:22,2000:3000]=np.nan # e2ds[20:30,1000:3000]=np.nan # ---------------------------------------------------------------------- # Reference plots # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot FP spectral order sPlt.drift_plot_selected_wave_ref(p, loc, x=loc['WAVE_LL'], y=loc['E2DS']) # plot photon noise uncertainty sPlt.drift_plot_photon_uncertainty(p, loc) # ---------------------------------------------------------------------- # Get template RV (from ccf_mask) # ---------------------------------------------------------------------- # get the CCF mask from file (check location of mask) loc = spirouRV.GetCCFMask(p, loc) # check and deal with mask in microns (should be in nm) if np.mean(loc['LL_MASK_CTR']) < 2.0: loc['LL_MASK_CTR'] *= 1000.0 loc['LL_MASK_D'] *= 1000.0 # ---------------------------------------------------------------------- # Do correlation # ---------------------------------------------------------------------- # calculate and fit the CCF loc = spirouRV.Coravelation(p, loc) # ---------------------------------------------------------------------- # Correlation stats # ---------------------------------------------------------------------- # get the maximum number of orders to use nbmax = p['CCF_NUM_ORDERS_MAX'] # get the average ccf loc['AVERAGE_CCF'] = np.nansum(loc['CCF'][:nbmax], axis=0) # normalize the average ccf normalized_ccf = loc['AVERAGE_CCF'] / np.max(loc['AVERAGE_CCF']) # get the fit for the normalized average ccf ccf_res, ccf_fit = spirouRV.FitCCF(p, loc['RV_CCF'], normalized_ccf, fit_type=0) loc['CCF_RES'] = ccf_res loc['CCF_FIT'] = ccf_fit # get the max cpp loc['MAXCPP'] = np.nansum(loc['CCF_MAX']) / np.nansum( loc['PIX_PASSED_ALL']) # get the RV value from the normalised average ccf fit center location loc['RV'] = float(ccf_res[1]) # get the contrast (ccf fit amplitude) loc['CONTRAST'] = np.abs(100 * ccf_res[0]) # get the FWHM value loc['FWHM'] = ccf_res[2] * spirouCore.spirouMath.fwhm() # ---------------------------------------------------------------------- # set the source keys = [ 'average_ccf', 'maxcpp', 'rv', 'contrast', 'fwhm', 'ccf_res', 'ccf_fit' ] loc.set_sources(keys, __NAME__ + '/main()') # ---------------------------------------------------------------------- # log the stats wmsg = ('Correlation: C={0:.1f}[%] RV={1:.5f}[km/s] ' 'FWHM={2:.4f}[km/s] maxcpp={3:.1f}') wargs = [loc['CONTRAST'], loc['RV'], loc['FWHM'], loc['MAXCPP']] WLOG(p, 'info', wmsg.format(*wargs)) # ---------------------------------------------------------------------- # rv ccf plot # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # Plot rv vs ccf (and rv vs ccf_fit) sPlt.ccf_rv_ccf_plot(p, loc['RV_CCF'], normalized_ccf, ccf_fit) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # TODO: Needs doing # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # add to qc header lists qc_values.append('None') qc_names.append('None') qc_logic.append('None') qc_pass.append(1) # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ---------------------------------------------------------------------- # archive ccf to table # ---------------------------------------------------------------------- # construct filename res_table_file = spirouConfig.Constants.CCF_TABLE_FILE(p) # log progress WLOG(p, '', 'Archiving CCF on file {0}'.format(res_table_file)) # define column names columns = ['order', 'maxcpp', 'nlines', 'contrast', 'RV', 'sig'] # define values for each column values = [ loc['ORDERS'], loc['CCF_MAX'] / loc['PIX_PASSED_ALL'], loc['TOT_LINE'], np.abs(100 * loc['CCF_ALL_RESULTS'][:, 0]), loc['CCF_ALL_RESULTS'][:, 1], loc['CCF_ALL_RESULTS'][:, 2] ] # define the format for each column formats = ['2.0f', '5.0f', '4.0f', '4.1f', '9.4f', '7.4f'] # construct astropy table from column names, values and formats table = spirouImage.MakeTable(p, columns, values, formats) # save table to file spirouImage.WriteTable(p, table, res_table_file, fmt='ascii') # ---------------------------------------------------------------------- # archive ccf to fits file # ---------------------------------------------------------------------- raw_infile = os.path.basename(p['E2DSFILE']) # construct folder and filename corfile, tag = spirouConfig.Constants.CCF_FITS_FILE(p) corfilename = os.path.split(corfile)[-1] # log that we are archiving the CCF on file WLOG(p, '', 'Archiving CCF on file {0}'.format(corfilename)) # get constants from p mask = p['CCF_MASK'] # if file exists remove it if os.path.exists(corfile): os.remove(corfile) # add the average ccf to the end of ccf data = np.vstack([loc['CCF'], loc['AVERAGE_CCF']]) # add drs keys hdict = spirouImage.CopyOriginalKeys(hdr) hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['E2DSFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_INCCFMASK'], value=p['CCF_MASK']) hdict = spirouImage.AddKey(p, hdict, p['KW_INRV'], value=p['TARGET_RV']) hdict = spirouImage.AddKey(p, hdict, p['KW_INWIDTH'], value=p['CCF_WIDTH']) hdict = spirouImage.AddKey(p, hdict, p['KW_INSTEP'], value=p['CCF_STEP']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # add CCF keys hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CTYPE'], value='km/s') hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CRVAL'], value=loc['RV_CCF'][0]) # the rv step rvstep = np.abs(loc['RV_CCF'][0] - loc['RV_CCF'][1]) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CDELT'], value=rvstep) # add ccf stats hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_RV'], value=loc['CCF_RES'][1]) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_RVC'], value=loc['RV']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_FWHM'], value=loc['FWHM']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_WMREF'], value=loc['WMEANREF']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CONTRAST'], value=loc['CONTRAST']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_MAXCPP'], value=loc['MAXCPP']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_MASK'], value=p['CCF_MASK']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_LINES'], value=np.nansum(loc['TOT_LINE'])) # add berv values hdict = spirouImage.AddKey(p, hdict, p['KW_BERV'], value=loc['BERV']) hdict = spirouImage.AddKey(p, hdict, p['KW_BJD'], value=loc['BJD']) hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_MAX'], value=loc['BERV_MAX']) # write image and add header keys (via hdict) p = spirouImage.WriteImage(p, corfile, data, hdict) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): """ cal_shape_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_shape_spirou.py [night_directory] [fitsfilename] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) p = spirouStartup.LoadArguments(p, night_name, files) p = spirouStartup.InitialFileSetup(p) # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add') # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Get basic image properties for FP file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # ---------------------------------------------------------------------- # Correction of reference FP # ---------------------------------------------------------------------- # set the number of frames p['NBFRAMES'] = 1 p.set_source('NBFRAMES', __NAME__ + '.main()') # Correction of DARK p, datac = spirouImage.CorrectForDark(p, data, hdr) # Resize hc data # rotate the image and convert from ADU/s to e- data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) data1 = spirouImage.ResizeImage(p, data, **bkwargs) # log change in data size WLOG(p, '', ('FPref Image format changed to {0}x{1}').format(*data1.shape)) # Correct for the BADPIX mask (set all bad pixels to zero) bargs = [p, data1, hdr] p, data1 = spirouImage.CorrectForBadPix(*bargs) p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True) # log progress WLOG(p, '', 'Cleaning FPref hot pixels') # correct hot pixels data1 = spirouEXTOR.CleanHotpix(data1, badpixmask) # Log the number of dead pixels # get the number of bad pixels with warnings.catch_warnings(record=True) as _: n_bad_pix = np.nansum(data1 <= 0) n_bad_pix_frac = n_bad_pix * 100 / np.product(data1.shape) # Log number wmsg = 'Nb FPref dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ---------------------------------------------------------------------- # Get master FP file # ---------------------------------------------------------------------- # log progress WLOG(p, '', 'Getting FP Master from calibDB') # get master fp p, masterfp = spirouImage.GetFPMaster(p, hdr) # ---------------------------------------------------------------------- # Get transform parameters # ---------------------------------------------------------------------- # log progress wargs = [p['ARG_FILE_NAMES'][0], p['FPMASTERFILE']] WLOG(p, 'info', 'Calculating transforming for {0} onto {1}'.format(*wargs)) gout = spirouImage.GetLinearTransformParams(p, masterfp, data1) transform, xres, yres = gout # ------------------------------------------------------------------ # Need to straighten the fp data for debug # ------------------------------------------------------------------ p, shapem_x = spirouImage.GetShapeX(p, hdr) p, shapem_y = spirouImage.GetShapeY(p, hdr) data2 = spirouImage.EATransform(data1, transform, dxmap=shapem_x, dymap=shapem_y) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # if transform is None means the fp image quality was too poor if transform is None: fail_msg.append('FP Image quality too poor (sigma clip failed)') passed = False qc_pass.append(0) else: qc_pass.append(1) qc_values.append('None') qc_names.append('Image Quality') qc_logic.append('Image too poor') # ---------------------------------------------------------------------- # get residual qc parameter qc_res = p['SHAPE_QC_LINEAR_TRANS_RES_THRES'] # assess quality of x residuals if xres > qc_res: fail_msg.append('x-resdiuals too high {0} > {1}'.format(xres, qc_res)) passed = False qc_pass.append(0) else: qc_pass.append(1) qc_values.append(xres) qc_names.append('XRES') qc_logic.append('XRES > {0}'.format(qc_res)) # assess quality of x residuals if yres > qc_res: fail_msg.append('y-resdiuals too high {0} > {1}'.format(yres, qc_res)) passed = False qc_pass.append(0) else: qc_pass.append(1) qc_values.append(yres) qc_names.append('YRES') qc_logic.append('YRES > {0}'.format(qc_res)) # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Writing shape to file # ------------------------------------------------------------------ # get the raw tilt file name raw_shape_file = os.path.basename(p['FITSFILENAME']) # construct file name and path shapefits, tag = spirouConfig.Constants.SLIT_SHAPE_LOCAL_FILE(p) shapefitsname = os.path.basename(shapefits) # Log that we are saving tilt file wmsg = 'Saving shape information in file: {0}' WLOG(p, '', wmsg.format(shapefitsname)) # Copy keys from fits file hdict = spirouImage.CopyOriginalKeys(hdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEX'], value=p['SHAPEXFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEY'], value=p['SHAPEYFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFPMASTER'], value=p['FPMASTERFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # add the transform parameters hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_DX'], value=transform[0]) hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_DY'], value=transform[1]) hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_A'], value=transform[2]) hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_B'], value=transform[3]) hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_C'], value=transform[4]) hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_D'], value=transform[5]) # write tilt file to file p = spirouImage.WriteImage(p, shapefits, [transform], hdict) # ---------------------------------------------------------------------- # Move to calibDB and update calibDB # ---------------------------------------------------------------------- if p['QC']: # add shape keydb = 'SHAPE' # copy shape file to the calibDB folder spirouDB.PutCalibFile(p, shapefits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, shapefitsname, hdr) # ------------------------------------------------------------------ # Writing sanity check files # ------------------------------------------------------------------ if p['SHAPE_DEBUG_OUTPUTS']: # log WLOG(p, '', 'Saving debug sanity check files') # construct file names dargs = [p, p['ARG_FILE_NAMES'][0]] out1 = spirouConfig.Constants.SLIT_SHAPE_IN_FP_FILE(*dargs) input_fp_file, tag1= out1 out2 = spirouConfig.Constants.SLIT_SHAPE_OUT_FP_FILE(*dargs) output_fp_file, tag2 = out2 # write input fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) p = spirouImage.WriteImage(p, input_fp_file, data1, hdict) # write output fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) p = spirouImage.WriteImage(p, output_fp_file, data2, hdict) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())