def main(night_name=None, key=None, filename=None): """ Manually add a file the the calibDB with "key" i.e. adds key night_name filename human-date unix-time to the calibDB and copies "filename" from .../reduced_dir/night_name/ into the calibDB Note filename must be in .../reduced_dir/night_name/ """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with arguments being None (i.e. get from sys.argv) pos = [0, 1] fmt = [str, str] names = ['key', 'filename'] call = [key, filename] # now get custom arguments customargs = spirouStartup.GetCustomFromRuntime(p, pos, fmt, names, calls=call) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='filename', mainfitsdir='reduced') # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data image, hdr, nbo, nx = spirouImage.ReadData(p, p['FITSFILENAME']) # ---------------------------------------------------------------------- # Move to calibDB and update calibDB # ---------------------------------------------------------------------- # set dark key keydb = p['KEY'] # copy dark fits file to the calibDB folder spirouDB.PutCalibFile(p, p['FITSFILENAME']) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, p['FILENAME'], hdr) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, oldfile=None, newfile=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with arguments being None (i.e. get from sys.argv) pos = [0, 1] fmt = [str, str] names = ['oldfile', 'newfile'] call = [oldfile, newfile] # now get custom arguments customargs = spirouStartup.GetCustomFromRuntime(p, pos, fmt, names, calls=call) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='newfile') # ---------------------------------------------------------------------- # Get files # ---------------------------------------------------------------------- # check that path exists emsg = '{0} file = {1} does not exist' if not os.path.exists(p['OLDFILE']): WLOG(p, 'error', emsg.format('old', p['OLDFILE'])) # check that paths exists if not os.path.exists(p['NEWFILE']): WLOG(p, 'error', emsg.format('new', p['NEWFILE'])) # load files data1, hdr1, _, _ = spirouImage.ReadImage(p, filename=oldfile) data2, hdr2, _, _ = spirouImage.ReadImage(p, filename=newfile) # ---------------------------------------------------------------------- # Do difference image # ---------------------------------------------------------------------- diff_image(p, data1, data2, 'old image', 'new image', scale=(1, 99)) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- wmsg = 'Recipe {0} has been successfully completed' WLOG(p, 'info', wmsg.format(p['PROGRAM'])) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, reffile=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with reference file being None (i.e. get from sys.argv) if reffile is None: customargs = spirouStartup.GetCustomFromRuntime(p, [0], [str], ['reffile']) else: customargs = dict(reffile=reffile) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='reffile', mainfitsdir='reduced') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, reffilename = spirouStartup.SingleFileSetup(p, filename=p['REFFILE'], skipcheck=True) p['REFFILENAME'] = reffilename p.set_source('REFFILENAME', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data data, hdr, nbo, nx = spirouImage.ReadData(p, p['REFFILENAME']) # ---------------------------------------------------------------------- # Add keys and save file # ---------------------------------------------------------------------- newfilename = p['REFFILE'].replace('.fits', '_edit.fits') newpath = os.path.join(p['ARG_FILE_DIR'], newfilename) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # set the version hdict = spirouImage.AddKey(p, hdict, HEADER_KEY, value=HEADER_VALUE) # log saving wmsg = 'Writing file {0} to {1}' WLOG(p, '', wmsg.format(newfilename, p['ARG_FILE_DIR'])) # save drift values p = spirouImage.WriteImage(p, newpath, data, hdict) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, ufiles=None): """ cal_preprocess_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_preprocess_spirou.py [night_directory] [fitsfilename] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param ufiles: string, list or None, the list of files to process Note can include wildcard i.e. "*.fits" (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # need custom args (to accept full path or wild card if ufiles is None: names, types = ['ufiles'], [str] customargs = spirouStartup.GetCustomFromRuntime(p, [0], types, names, last_multi=True) else: customargs = dict(ufiles=ufiles) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsdir='raw') # ---------------------------------------------------------------------- # Get hot pixels for corruption check # ---------------------------------------------------------------------- hotpixels = spirouImage.PPGetHotPixels(p) # ---------------------------------------------------------------------- # Process files (including wildcards) # ---------------------------------------------------------------------- # get raw folder (assume all files are in the root directory) rawdir = spirouConfig.Constants.RAW_DIR(p) try: ufiles = spirouFile.Paths(p['UFILES'], root=rawdir).abs_paths except PathException as e: WLOG(p, 'error', e) # log how many files were found wmsg = '{0} files found' WLOG(p, '', wmsg.format(len(ufiles))) # storage for output files p['OUTPUT_NAMES'] = [] p.set_source('OUTPUT_NAMES', __NAME__ + '.main()') # loop around files for u_it, ufile in enumerate(ufiles): # log the file process wmsg = 'Processing file {0} ({1} of {2})' WLOG(p, '', spirouStartup.spirouStartup.HEADER) bfilename = os.path.basename(ufile) WLOG(p, 'info', wmsg.format(bfilename, u_it+1, len(ufiles))) WLOG(p, '', spirouStartup.spirouStartup.HEADER) # ------------------------------------------------------------------ # Check that we can process file # ------------------------------------------------------------------ # check if ufile exists if not os.path.exists(ufile): wmsg = 'File {0} does not exist... skipping' WLOG(p, 'warning', wmsg.format(ufile)) continue # skip processed files elif p['PROCESSED_SUFFIX'] in bfilename: wmsg = 'File {0} has been processed... skipping' WLOG(p, 'warning', wmsg.format(ufile)) continue # skip non-fits files elif '.fits' not in bfilename: wmsg = 'File {0} not a fits file... skipping' WLOG(p, 'warning', wmsg.format(ufile)) continue # skip index file elif bfilename == spirouConfig.Constants.INDEX_OUTPUT_FILENAME(): wmsg = 'Skipping index fits file' WLOG(p, 'warning', wmsg.format(ufile)) continue # ------------------------------------------------------------------ # Read image file # ------------------------------------------------------------------ # read the image data rout = spirouImage.ReadImage(p, filename=ufile) image, hdr, nx, ny = rout # ------------------------------------------------------------------ # Identify file (and update filename, header and comments) # ------------------------------------------------------------------ ufile, hdr = spirouImage.IdentifyUnProFile(p, ufile, hdr) # ------------------------------------------------------------------ # correct image # ------------------------------------------------------------------ # correct for the top and bottom reference pixels WLOG(p, '', 'Correcting for top and bottom pixels') image = spirouImage.PPCorrectTopBottom(p, image) # correct by a median filter from the dark amplifiers wmsg = 'Correcting by the median filter from dark amplifiers' WLOG(p, '', wmsg) image = spirouImage.PPMedianFilterDarkAmps(p, image) # correct for the 1/f noise wmsg = 'Correcting for the 1/f noise' WLOG(p, '', wmsg) image = spirouImage.PPMedianOneOverfNoise2(p, image) # ------------------------------------------------------------------ # Quality control to check for corrupt files # ------------------------------------------------------------------ # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # get pass condition cout = spirouImage.PPTestForCorruptFile(p, image, hotpixels) snr_hotpix, rms_list = cout # print out SNR hotpix value wmsg = 'Corruption check: SNR Hotpix value = {0:.5e}' WLOG(p, '', wmsg.format(snr_hotpix)) #deal with printing corruption message if snr_hotpix < p['PP_CORRUPT_SNR_HOTPIX']: # add failed message to fail message list fargs = [snr_hotpix, p['PP_CORRUPT_SNR_HOTPIX'],ufile ] fmsg = ('File was found to be corrupted. (SNR_HOTPIX < threshold, ' '{0:.4e} < {1:.4e}). File will not be saved. ' 'File = {2}'.format(*fargs)) fail_msg.append(fmsg) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(snr_hotpix) qc_names.append('snr_hotpix') qc_logic.append('snr_hotpix < {0:.5e}' ''.format(p['PP_CORRUPT_SNR_HOTPIX'])) # ---------------------------------------------------------------------- if np.max(rms_list) > p['PP_CORRUPT_RMS_THRES']: # add failed message to fail message list fargs = [np.max(rms_list), p['PP_CORRUPT_RMS_THRES'], ufile] fmsg = ('File was found to be corrupted. (RMS < threshold, ' '{0:.4e} > {1:.4e}). File will not be saved. ' 'File = {0}'.format(*fargs)) fail_msg.append(fmsg) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(np.max(rms_list)) qc_names.append('max(rms_list)') qc_logic.append('max(rms_list) > {0:.4e}' ''.format(p['PP_CORRUPT_RMS_THRES'])) # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') WLOG(p, 'warning', '\tFile not written') continue # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # rotate image # ------------------------------------------------------------------ # rotation to match HARPS orientation (expected by DRS) image = spirouImage.RotateImage(image, p['RAW_TO_PP_ROTATION']) # ------------------------------------------------------------------ # Save rotated image # ------------------------------------------------------------------ # construct rotated file name outfits = spirouConfig.Constants.PP_FILE(p, bfilename) outfitsname = os.path.basename(outfits) # log that we are saving rotated image WLOG(p, '', 'Saving Rotated Image in ' + outfitsname) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_PPVERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) # set the inputs hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=[os.path.basename(ufile)]) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_DRS_QC_NAME'], values=qc_names) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # set the DRS type (for file indexing) p['DRS_TYPE'] = 'RAW' p.set_source('DRS_TYPE', __NAME__ + '.main()') # write to file p = spirouImage.WriteImage(p, outfits, image, hdict) # index this file p = spirouStartup.End(p, outputs='pp', end=False) # ------------------------------------------------------------------ # append to output storage in p # ------------------------------------------------------------------ p['OUTPUT_NAMES'].append(outfitsname) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p, outputs=None) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, flatfile=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with arguments being None (i.e. get from sys.argv) name, lname = ['flatfile'], ['Reference file'] req, call, call_priority = [True], [flatfile], [True] # now get custom arguments customargs = spirouStartup.GetCustomFromRuntime(p, [0], [str], name, req, call, call_priority, lname) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='flatfile') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, reffile = spirouStartup.SingleFileSetup(p, filename=p['FLATFILE']) # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Get the required fiber type from the constants file # ---------------------------------------------------------------------- # get the fiber type (set to AB) p['FIBER'] = p['EM_FIB_TYPE'] p['FIBER_TYPES'] = [p['EM_FIB_TYPE']] # ---------------------------------------------------------------------- # Read flat image file # ---------------------------------------------------------------------- # read the image data (for the header only) image, hdr, ny, nx = spirouImage.ReadData(p, reffile) # ---------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- image = spirouImage.FixNonPreProcess(p, image) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- # create loc loc = ParamDict() # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # ---------------------------------------------------------------------- # Resize flat image # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- image2 = spirouImage.ConvertToE(spirouImage.FlipImage(p, image), p=p) # convert NaN to zeros image2 = np.where(~np.isfinite(image2), np.zeros_like(image2), image2) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) image2 = spirouImage.ResizeImage(p, image2, **bkwargs) # save flat to to loc and set source loc['IMAGE'] = image2 loc.set_sources(['image'], __NAME__ + '/main()') # log change in data size wmsg = 'Image format changed to {0}x{1}' WLOG(p, '', wmsg.format(*image2.shape)) # ---------------------------------------------------------------------- # Read shape or tilt slit angle # ---------------------------------------------------------------------- # set source of tilt file tsource = __NAME__ + '/main() + /spirouImage.ReadTiltFile' if p['IC_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES: # log progress WLOG(p, '', 'Debananafying (straightening) image') # get the shape map p, loc['SHAPE'] = spirouImage.ReadShapeMap(p, hdr) loc.set_source('SHAPE', tsource) else: # get tilts p, loc['TILT'] = spirouImage.ReadTiltFile(p, hdr) loc.set_source('TILT', tsource) # ---------------------------------------------------------------------- # Read blaze # ---------------------------------------------------------------------- # get tilts p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hdr) loc.set_source('BLAZE', __NAME__ + '/main() + /spirouImage.ReadBlazeFile') # set number of orders from blaze file loc['NBO'] = loc['BLAZE'].shape[0] loc.set_source('NBO', __NAME__ + '/main()') # ------------------------------------------------------------------ # Read wavelength solution # ------------------------------------------------------------------ # set source of wave file wsource = __NAME__ + '/main() + /spirouImage.GetWaveSolution' # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get wave image wout = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True, return_filename=True, fiber=wave_fiber) loc['WAVEPARAMS'], loc['WAVE'], loc['WAVEFILE'], loc['WSOURCE'] = wout loc.set_sources(['WAVEPARAMS', 'WAVE', 'WAVEFILE', 'WSOURCE'], wsource) # ------------------------------------------------------------------ # Get localisation coefficients # ------------------------------------------------------------------ # storage for fiber parameters loc['ALL_ACC'] = OrderedDict() loc['ALL_ASS'] = OrderedDict() # get this fibers parameters for fiber in p['FIBER_TYPES']: p = spirouImage.FiberParams(p, fiber, merge=True) # get localisation fit coefficients p, loc = spirouLOCOR.GetCoeffs(p, hdr, loc=loc) # save all fibers loc['ALL_ACC'][fiber] = loc['ACC'] loc['ALL_ASS'][fiber] = loc['ASS'] # ------------------------------------------------------------------ # Get telluric and telluric mask and add to loc # ------------------------------------------------------------------ # log process wmsg = 'Loading telluric model and locating "good" tranmission' WLOG(p, '', wmsg) # load telluric and get mask (add to loc) loc = spirouExM.get_telluric(p, loc) # ------------------------------------------------------------------ # Make 2D map of orders # ------------------------------------------------------------------ # log progress WLOG(p, '', 'Making 2D map of order locations') # make the 2D wave-image loc = spirouExM.order_profile(p, loc) # ------------------------------------------------------------------ # Make 2D map of wavelengths accounting for shape / tilt # ------------------------------------------------------------------ # log progress WLOG(p, '', 'Mapping pixels on to wavelength grid') # make the 2D map of wavelength loc = spirouExM.create_wavelength_image(p, loc) # ------------------------------------------------------------------ # Use spectra wavelength to create 2D image from wave-image # ------------------------------------------------------------------ if p['EM_SAVE_MASK_MAP'] or p['EM_SAVE_TELL_SPEC']: # log progress WLOG(p, '', 'Creating image from wave-image interpolation') # create image from waveimage wkwargs = dict(x=loc['TELL_X'], y=loc['TELL_Y']) loc = spirouExM.create_image_from_waveimage(loc, **wkwargs) else: loc['SPE'] = np.zeros_like(image2).astype(float) # ------------------------------------------------------------------ # Create 2D mask (min to max lambda + transmission threshold) # ------------------------------------------------------------------ if p['EM_SAVE_MASK_MAP']: # log progress WLOG(p, '', 'Creating wavelength/tranmission mask') # create mask loc = spirouExM.create_mask(p, loc) else: loc['TELL_MASK_2D'] = np.zeros_like(image2).astype(bool) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # TODO: Needs doing # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # add to qc header lists qc_values.append('None') qc_names.append('None') qc_logic.append('None') qc_pass.append(1) # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Construct parameters for header # ------------------------------------------------------------------ hdict = OrderedDict() # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) # set the input files if loc['SHAPE'] is not None: hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPE'], value=p['SHAPFILE']) else: hdict = spirouImage.AddKey(p, hdict, p['KW_CDBTILT'], value=p['TILTFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['FLATFILE']) # add name of the TAPAS y data hdict = spirouImage.AddKey(p, hdict, p['KW_EM_TELLY'], value=loc['TELLSPE']) # add name of the localisation fits file used hfile = os.path.basename(loc['LOCO_CTR_FILE']) hdict = spirouImage.AddKey(p, hdict, p['kw_EM_LOCFILE'], value=hfile) # add the max and min wavelength threshold hdict = spirouImage.AddKey(p, hdict, p['kw_EM_MINWAVE'], value=p['EM_MIN_LAMBDA']) hdict = spirouImage.AddKey(p, hdict, p['kw_EM_MAXWAVE'], value=p['EM_MAX_LAMBDA']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # add the transmission cut hdict = spirouImage.AddKey(p, hdict, p['kw_EM_TRASCUT'], value=p['EM_TELL_THRESHOLD']) # ------------------------------------------------------------------ # Deal with output preferences # ------------------------------------------------------------------ # add bad pixel map (if required) if p['EM_COMBINED_BADPIX']: # get bad pix mask (True where bad) badpixmask, bhdr, badfile = spirouImage.GetBadPixMap(p, hdr) goodpixels = badpixmask == 0 # apply mask (multiply) loc['TELL_MASK_2D'] = loc['TELL_MASK_2D'] & goodpixels.astype(bool) else: badfile = 'None' # add to hdict hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=badfile) # convert waveimage mask into float array loc['TELL_MASK_2D'] = loc['TELL_MASK_2D'].astype('float') # check EM_OUTPUT_TYPE and deal with set to "all" if p['EM_OUTPUT_TYPE'] not in ["drs", "raw", "preprocess", "all"]: emsg1 = '"EM_OUTPUT_TYPE" not understood' emsg2 = ' must be either "drs", "raw" or "preprocess"' emsg3 = ' currently EM_OUTPUT_TYPE="{0}"'.format(p['EM_OUTPUT_TYPE']) WLOG(p, 'error', [emsg1, emsg2, emsg3]) outputs = [] elif p['EM_OUTPUT_TYPE'] != 'all': outputs = [str(p['EM_OUTPUT_TYPE'])] else: outputs = ["drs", "raw", "preprocess"] # ---------------------------------------------------------------------- # loop around output types # ---------------------------------------------------------------------- for output in outputs: # log progress WLOG(p, '', 'Processing {0} outputs'.format(output)) # change EM_OUTPUT_TYPE p['EM_OUTPUT_TYPE'] = output # copy arrays out_spe = np.array(loc['SPE']) out_wave = np.array(loc['WAVEIMAGE']) out_mask = np.array(loc['TELL_MASK_2D']) # change image size if needed if output in ["raw", "preprocess"]: kk = dict(xsize=image.shape[1], ysize=image.shape[0]) if p['EM_SAVE_TELL_SPEC']: WLOG(p, '', 'Resizing/Flipping SPE') out_spe = spirouExM.unresize(p, out_spe, **kk) WLOG(p, '', 'Rescaling SPE') out_spe = out_spe / (p['GAIN'] * p['EXPTIME']) if p['EM_SAVE_WAVE_MAP']: WLOG(p, '', 'Resizing/Flipping WAVEIMAGE') out_wave = spirouExM.unresize(p, out_wave, **kk) if p['EM_SAVE_MASK_MAP']: WLOG(p, '', 'Resizing/Flipping TELL_MASK_2D') out_mask = spirouExM.unresize(p, out_mask, **kk) # if raw need to rotate (undo pre-processing) if output == "raw": if p['EM_SAVE_TELL_SPEC']: WLOG(p, '', 'Rotating SPE') out_spe = np.rot90(out_spe, 1) if p['EM_SAVE_WAVE_MAP']: WLOG(p, '', 'Rotating WAVEIMAGE') out_wave = np.rot90(out_wave, 1) if p['EM_SAVE_MASK_MAP']: WLOG(p, '', 'Rotating TELL_MASK_2D') out_mask = np.rot90(out_mask, 1) # ---------------------------------------------------------------------- # save 2D spectrum, wavelength image and mask to file # ---------------------------------------------------------------------- # save telluric spectrum if p['EM_SAVE_TELL_SPEC']: # construct spectrum filename specfitsfile, tag = spirouConfig.Constants.EM_SPE_FILE(p) specfilename = os.path.split(specfitsfile)[-1] # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # log progress wmsg = 'Writing spectrum to file {0}' WLOG(p, '', wmsg.format(specfilename)) # write to file p = spirouImage.WriteImage(p, specfitsfile, out_spe, hdict=hdict) # ---------------------------------------------------------------------- # save wave map if p['EM_SAVE_WAVE_MAP']: # construct waveimage filename wavefitsfile, tag = spirouConfig.Constants.EM_WAVE_FILE(p) wavefilename = os.path.split(wavefitsfile)[-1] # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # log progress wmsg = 'Writing wave image to file {0}' WLOG(p, '', wmsg.format(wavefilename)) # write to file p = spirouImage.WriteImage(p, wavefitsfile, out_wave, hdict=hdict) # ---------------------------------------------------------------------- # save mask file if p['EM_SAVE_MASK_MAP']: # construct tell mask 2D filename maskfitsfile, tag = spirouConfig.Constants.EM_MASK_FILE(p) maskfilename = os.path.split(maskfitsfile)[-1] # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # log progress wmsg = 'Writing telluric mask to file {0}' WLOG(p, '', wmsg.format(maskfilename)) # convert boolean mask to integers writablemask = np.array(out_mask, dtype=float) # write to file p = spirouImage.WriteImage(p, maskfitsfile, writablemask, hdict=hdict) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, ufile=None, xsize=None, ysize=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # need custom args (to accept full path or wild card if ufile is None or xsize is None or ysize is None: pos = [0, 1, 2] names, types = ['ufile', 'xsize', 'ysize'], [str, int, int] customargs = spirouStartup.GetCustomFromRuntime(p, pos, types, names, last_multi=True) else: customargs = dict(ufile=ufile, xsize=xsize, ysize=ysize) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs) # add constants not currently in constants file # ------------------------------------------------------------------ # Read image file # ------------------------------------------------------------------ # read the image data rout = spirouImage.ReadImage(p, filename=p['UFILE']) image, hdr, nx, ny = rout # ---------------------------------------------------------------------- # un-Resize image # ---------------------------------------------------------------------- # create an array of given size size = np.product([p['YSIZE'], p['XSIZE']]) newimage = np.repeat(np.nan, size).reshape(p['YSIZE'], p['XSIZE']) # insert image at given pixels xlow, xhigh = p['IC_CCDX_LOW'], p['IC_CCDX_HIGH'] ylow, yhigh = p['IC_CCDY_LOW'], p['IC_CCDY_HIGH'] newimage[ylow:yhigh, xlow:xhigh] = image # rotate the image newimage = spirouImage.FlipImage(p, newimage) # ------------------------------------------------------------------ # Save rotated image # ------------------------------------------------------------------ # construct rotated file name outfits = ufile.replace('.fits', '_old.fits') outfitsname = os.path.split(outfits)[-1] # log that we are saving rotated image WLOG(p, '', 'Saving Rotated Image in ' + outfitsname) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # write to file p = spirouImage.WriteImage(p, outfits, newimage, hdict) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- wmsg = 'Recipe {0} has been successfully completed' WLOG(p, 'info', wmsg.format(p['PROGRAM'])) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, files=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # get parameters from configuration files and run time arguments customargs = spirouStartup.GetCustomFromRuntime(p, [0], [str], ['reffile']) p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='reffile', mainfitsdir='reduced') # setup files and get fiber p = spirouStartup.InitialFileSetup(p, calibdb=True) # set the fiber type p['FIB_TYP'] = [p['FIBER']] # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data gfkwargs = dict(path=p['REDUCED_DIR'], filename=p['REFFILE']) p['REFFILENAME'] = spirouStartup.GetFile(p, **gfkwargs) p.set_source('REFFILENAME', __NAME__ + '/main()') # get the fiber type p['FIBER'] = 'AB' e2ds, hdr, nx, ny = spirouImage.ReadImage(p) # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get wave image _, wave, _ = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True, fiber=wave_fiber) blaze = spirouImage.ReadBlazeFile(p) # ---------------------------------------------------------------------- # Get lamp params # ---------------------------------------------------------------------- # get lamp parameters p = spirouTHORCA.GetLampParams(p, hdr) # ---------------------------------------------------------------------- # Get catalogue and fitted line list # ---------------------------------------------------------------------- # load line file (from p['IC_LL_LINE_FILE']) ll_line_cat, ampl_line_cat = spirouImage.ReadLineList(p) # construct fitted lines table filename wavelltbl = spirouConfig.Constants.WAVE_LINE_FILE(p) WLOG(p, '', wavelltbl) # read fitted lines ll_ord, ll_line_fit, ampl_line_fit = np.genfromtxt(wavelltbl, skip_header=4, skip_footer=2, unpack=True, usecols=(0, 1, 3)) # ---------------------------------------------------------------------- # Plots # ---------------------------------------------------------------------- # define line colours col = ['magenta', 'purple'] # get order parity ll_ord_par = np.mod(ll_ord, 2) print(ll_ord_par) col2 = [col[int(x)] for x in ll_ord_par] # start interactive plot sPlt.start_interactive_session(p) plt.figure() for order_num in np.arange(nx): plt.plot(wave[order_num], e2ds[order_num]) # get heights heights = [] for line in range(len(ll_line_cat)): heights.append(200000 + np.max([np.min(e2ds), ampl_line_cat[line]])) # plot ll_line_cat plt.vlines(ll_line_cat, 0, heights, colors='darkgreen', linestyles='dashed') # get heights heights = [] for line in range(len(ll_line_fit)): heights.append(200000 + np.max([np.min(e2ds), ampl_line_fit[line]])) # plot ll_line_fit plt.vlines(ll_line_fit, 0, heights, colors=col2, linestyles='dashdot') plt.xlabel('Wavelength [nm]') plt.ylabel('Flux e-') plt.title(p['REFFILENAME']) # end interactive session # sPlt.end_interactive_session() # old code: # plt.ion() # plt.figure() # # for order_num in np.arange(nx): # plt.plot(wave[order_num], e2ds[order_num]) # # for line in range(len(ll_line_cat)): # plt.vlines(ll_line_cat[line], 0, 200000 + # max(np.min(e2ds), ampl_line_cat[line]), # colors='darkgreen', linestyles='dashed') # # for line in range(len(ll_line_fit)): # plt.vlines(ll_line_fit[line], 0, 200000 + # max(np.min(e2ds), ampl_line_fit[line]), # colors='magenta', linestyles='dashdot') # # plt.xlabel('Wavelength [nm]') # plt.ylabel('Flux e-') # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p, outputs=None) # return a copy of locally defined variables in the memory return dict(locals())
def main(runname=None, quiet=False): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__, quiet=True) # now get custom arguments ckwargs = dict(positions=[0], types=[str], names=['RUNNAME'], calls=[runname], require_night_name=False, required=[False]) customargs = spirouStartup.GetCustomFromRuntime(p, **ckwargs) # add custom args straight to p p = spirouStartup.LoadMinimum(p, customargs=customargs, quiet=True) # ---------------------------------------------------------------------- # Read the run file and extract parameters # ---------------------------------------------------------------------- # construct filename rfile = os.path.join(UNIT_TEST_PATH, p['RUNNAME']) # check if RUNNAME is None if p['RUNNAME'] == 'None': exists = False emsgs = ['No unit test run file defined.'] # check that rfile exists elif not os.path.exists(rfile): emsgs = ['Unit test run file "{0}" does not exist'.format(rfile)] exists = False else: exists = True emsgs = [] # deal with file wrong (or no file defined) --> print valid unit tests if not exists: emsgs.append('') emsgs.append('Available units tests are:') for rfile in os.listdir(UNIT_TEST_PATH): emsgs.append('\t{0}'.format(rfile)) emsgs.append('') emsgs.append('Located at {0}'.format(UNIT_TEST_PATH)) WLOG(p, 'error', emsgs) # get the parameters in the run file rparams = spirouConfig.GetConfigParams(p, filename=rfile) # reset the DRS if not quiet: spirouTools.DRS_Reset(log=True, called=True) # ---------------------------------------------------------------------- # Get runs # ---------------------------------------------------------------------- runs = spirouUnitTests.get_runs(p, rparams, rfile) # ---------------------------------------------------------------------- # Run runs # ---------------------------------------------------------------------- # storage for times times = OrderedDict() errors = OrderedDict() # log the start of the unit tests spirouUnitTests.unit_log_title(p) # loop around runs and process each for runn in list(runs.keys()): if p['DRS_DEBUG'] > 0: # do run rargs = [p, runn, runs[runn], times] times = spirouUnitTests.manage_run(*rargs) else: # try to run try: # do run rargs = [p, runn, runs[runn], times] times = spirouUnitTests.manage_run(*rargs) # Manage unexpected errors except Exception as e: wmsgs = ['Run "{0}" had an unexpected error:'.format(runn)] for msg in str(e).split('\n'): wmsgs.append('\t' + msg) WLOG(p, 'warning', wmsgs) errors[runn] = str(e) # Manage expected errors except SystemExit as e: wmsgs = ['Run "{0}" had an expected error:'.format(runn)] for msg in str(e).split('\n'): wmsgs.append('\t' + msg) WLOG(p, 'warning', wmsgs) errors[runn] = str(e) # ---------------------------------------------------------------------- # Make sure all plots are close # ---------------------------------------------------------------------- sPlt.closeall() # ---------------------------------------------------------------------- # Print timings # ---------------------------------------------------------------------- spirouUnitTests.log_timings(p, times) # ---------------------------------------------------------------------- # Print errors # ---------------------------------------------------------------------- spirouUnitTests.log_errors(p, errors) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- wmsg = 'Recipe {0} has been successfully completed' WLOG(p, 'info', wmsg.format(p['PROGRAM'])) # return a copy of locally defined variables in the memory return dict(locals())
WLOG = spirouCore.wlog # Get plotting functions sPlt = spirouCore.sPlt night_name = '18BQ08-Sep21' reffile = '2305120o_pp_e2dsff_AB_tellu_corrected.fits' # ============================================================================= # Start of code # ============================================================================= # Main code here if __name__ == "__main__": # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) customargs = spirouStartup.GetCustomFromRuntime(p, [0], [str], ['reffile'], [True], [reffile]) p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='reffile', mainfitsdir='reduced') p['FIBER'] = 'AB' # load the calibDB p = spirouStartup.LoadCalibDB(p) # load ref spectrum e2ds, hdr, nx, ny = spirouImage.ReadImage(p) # get blaze blaze = spirouImage.ReadBlazeFile(p) # get wave image _, wave, _ = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True)
def main(night_name=None, reffile=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) customargs = spirouStartup.GetCustomFromRuntime(p, [0], [str], ['reffile'], [True], [reffile]) p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='reffile', mainfitsdir='reduced') # load the calibDB p = spirouStartup.LoadCalibDB(p) # force plotting to 1 p['DRS_PLOT'] = 1 # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data p, fpfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['REFFILE']) # get the fiber type fiber1 = str(p['FIBER']) e2ds, hdr, nx, ny = spirouImage.ReadImage(p) p, blaze = spirouImage.ReadBlazeFile(p) # set source of wave file wsource = __NAME__ + '/main() + /spirouImage.GetWaveSolution' # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get wave image _, wave, _ = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True, fiber=wave_fiber) # ---------------------------------------------------------------------- # Get basic image properties # ---------------------------------------------------------------------- plt.ion() plt.figure() for i in np.arange(nx): plt.plot(wave[i], e2ds[i]) plt.xlabel('Wavelength [nm]') plt.ylabel('Flux e-') plt.title('Extracted spectra') plt.figure() for i in np.arange(nx): plt.plot(wave[i], e2ds[i] / blaze[i]) plt.xlabel('Wavelength [nm]') plt.ylabel('Relative Flux e-') plt.title('Blaze corrected Extracted spectra') # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p, outputs=None) # return a copy of locally defined variables in the memory return dict(locals())
def main(filetype='DARK_DARK'): """ cal_DARK_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_DARK_spirou.py [night_directory] [fitsfilename] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # set up function name main_name = __NAME__ + '.main()' # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # now get custom arguments pos, fmt = [0], [str] names, call = ['filetype'], [filetype] customargs = spirouStartup.GetCustomFromRuntime(p, pos, fmt, names, calls=call, require_night_name=False) p = spirouStartup.LoadArguments(p, None, customargs=customargs, mainfitsdir='tmp', require_night_name=False) # set the DRS type (for file indexing) p['DRS_TYPE'] = 'RAW' p.set_source('DRS_TYPE', __NAME__ + '.main()') # ------------------------------------------------------------------------- # find all "filetype" objects filenames = spirouImage.FindFiles(p, filetype=p['FILETYPE'], allowedtypes=p['ALLOWED_DARK_TYPES']) # convert filenames to a numpy array filenames = np.array(filenames) # ------------------------------------------------------------------------- # julian date to know which file we need to # process together dark_time = np.zeros(len(filenames)) dark_exp, dark_pp_version, dark_wt_temp = [], [], [] basenames, nightnames, dark_cass_temp, dark_humidity = [], [], [], [] # log progress WLOG(p, '', 'Reading all dark file headers') # looping through the file headers for it in range(len(filenames)): # get night name night_name = os.path.dirname(filenames[it]).split(p['TMP_DIR'])[-1] # get header hdr = spirouImage.ReadHeader(p, filepath=filenames[it]) # add MJDATE to dark times dark_time[it] = float(hdr[p['KW_ACQTIME'][0]]) # add other keys (for tabular output) basenames.append(os.path.basename(filenames[it])) nightnames.append(night_name) dark_exp.append(float(hdr[p['KW_EXPTIME'][0]])) dark_pp_version.append(hdr[p['KW_PPVERSION'][0]]) dark_wt_temp.append(float(hdr[p['KW_WEATHER_TOWER_TEMP'][0]])) dark_cass_temp.append(float(hdr[p['KW_CASS_TEMP'][0]])) dark_humidity.append(float(hdr[p['KW_HUMIDITY'][0]])) # ------------------------------------------------------------------------- # match files by date # ------------------------------------------------------------------------- # log progress wmsg = 'Matching dark files by observation time (+/- {0} hrs)' WLOG(p, '', wmsg.format(p['DARK_MASTER_MATCH_TIME'])) # get the time threshold time_thres = p['DARK_MASTER_MATCH_TIME'] # get items grouped by time matched_id = spirouImage.GroupFilesByTime(p, dark_time, time_thres) # ------------------------------------------------------------------------- # get the most recent position lastpos = np.argmax(dark_time) # load up the most recent dark rout = spirouImage.ReadImage(p, filenames[lastpos], log=False) data_ref, hdr_ref, nx, ny = rout # set the night name and update the reduced directory p['ARG_NIGHT_NAME'] = nightnames[lastpos] p.set_source('ARG_NIGHT_NAME', __NAME__ + '.main()') p['REDUCED_DIR'] = spirouConfig.Constants.REDUCED_DIR(p) p.set_source('REDUCED_DIR', __NAME__ + '.main()') # ------------------------------------------------------------------------- # Read individual files and sum groups # ------------------------------------------------------------------------- # log process WLOG(p, '', 'Reading Dark files and combining groups') # Find all unique groups u_groups = np.unique(matched_id) # currently number of bins == number of groups num_bins = len(u_groups) # storage of dark cube dark_cube = np.zeros([num_bins, ny, nx]) bin_cube = np.zeros(num_bins) # loop through groups for g_it, group_num in enumerate(u_groups): # log progress WLOG(p, '', '\tGroup {0} of {1}'.format(g_it + 1, len(u_groups))) # find all files for this group dark_ids = filenames[matched_id == group_num] # load this groups files into a cube cube = [] for filename in dark_ids: # read data data_it, _, _, _ = spirouImage.ReadImage(p, filename, log=False) # add to cube cube.append(data_it) # median dark cube groupdark = np.nanmedian(cube, axis=0) # sum within each bin dark_cube[g_it % num_bins] += groupdark # record the number of cubes that are going into this bin bin_cube[g_it % num_bins] += 1 # need to normalize if we have more than 1 cube per bin for bin_it in range(num_bins): dark_cube[bin_it] /= bin_cube[bin_it] # ------------------------------------------------------------------------- # we perform a median filter over a +/- "med_size" pixel box # ------------------------------------------------------------------------- # log process WLOG(p, '', 'Performing median filter for {0} bins'.format(num_bins)) # get med_size from p med_size = p['DARK_MASTER_MED_SIZE'] # storage of output dark cube dark_cube1 = np.zeros([num_bins, ny, nx]) # loop around the bins for bin_it in range(num_bins): # get the dark for this bin bindark = dark_cube[bin_it] # performing a median filter of the image with [-med_size, med_size] # box in x and 1 pixel wide in y. Skips the pixel considered, # so this is equivalent of a 2*med_size boxcar tmp = [] for jt in range(-med_size, med_size + 1): if jt != 0: tmp.append(np.roll(bindark, [0, jt])) # low frequency image lf_dark = np.nanmedian(tmp, axis=0) # high frequency image dark_cube1[bin_it] = bindark - lf_dark # ------------------------------------------------------------------------- # median the dark cube to create the master dark master_dark = np.nanmedian(dark_cube1, axis=0) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # add to qc header lists qc_values.append('None') qc_names.append('None') qc_logic.append('None') qc_pass.append(1) # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ---------------------------------------------------------------------- # Save master dark to file # ---------------------------------------------------------------------- # set reference filename reffile = filenames[lastpos] # construct folder and filename darkmasterfits, tag = spirouConfig.Constants.DARK_FILE_MASTER(p, reffile) darkmasterfitsname = os.path.basename(darkmasterfits) # log writing of file WLOG(p, '', 'Saving master dark to {0}'.format(darkmasterfitsname)) # construct big dark table colnames = ['FILENAME', 'NIGHT', 'MJDATE', 'EXPTIME', 'WEATHER_TEMP', 'CASS_TEMP', 'RELHUMID', 'PVERSION', 'GROUPID'] values = [basenames, nightnames, dark_time, dark_exp, dark_wt_temp, dark_cass_temp, dark_humidity, dark_pp_version, matched_id] darktable = spirouImage.MakeTable(p, colnames, values) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr_ref) # define new keys to add hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # write master dark + dark table to file p = spirouImage.WriteImageTable(p, darkmasterfits, image=master_dark, table=darktable, hdict=hdict) # ---------------------------------------------------------------------- # Move to calibDB and update calibDB # ---------------------------------------------------------------------- if p['QC']: # set dark master key keydb = 'DARKM' # copy dark fits file to the calibDB folder spirouDB.PutCalibFile(p, darkmasterfits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, darkmasterfitsname, hdr_ref) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, hcfile=None, fpfiles=None): """ cal_SLIT_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_SLIT_spirou.py [night_directory] [files] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) if hcfile is None or fpfiles is None: names, types = ['hcfile', 'fpfiles'], [str, str] customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1], types, names, last_multi=True) else: customargs = dict(hcfile=hcfile, fpfile=fpfiles) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='fpfiles') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, hcfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['HCFILE']) p, fpfitsfiles = spirouStartup.MultiFileSetup(p, files=p['FPFILES']) # set fiber (it doesn't matter with the 2D image but we need this to get # the lamp type for FPFILES and HCFILES, AB == C p['FIBER'] = 'AB' p['FIB_TYP'] = [p['FIBER']] fsource = __NAME__ + '/main()' p.set_sources(['FIBER', 'FIB_TYP'], fsource) # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # add a force plot off p['PLOT_PER_ORDER'] = PLOT_PER_ORDER p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Read FP and HC files # ---------------------------------------------------------------------- # read input fp and hc data rkwargs = dict(filename=fpfitsfiles[0], filenames=fpfitsfiles[1:], framemath='add') p, fpdata, fphdr = spirouImage.ReadImageAndCombine(p, **rkwargs) hcdata, hchdr, _, _ = spirouImage.ReadImage(p, hcfitsfilename) # add data and hdr to loc loc = ParamDict() loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr # set the source sources = ['HCDATA', 'HCHDR'] loc.set_sources(sources, 'spirouImage.ReadImage()') sources = ['FPDATA', 'FPHDR'] loc.set_sources(sources, 'spirouImage.ReadImage()') # --------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- hcdata = spirouImage.FixNonPreProcess(p, hcdata) fpdata = spirouImage.FixNonPreProcess(p, fpdata) # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # add a force plot off p['PLOT_PER_ORDER'] = PLOT_PER_ORDER p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, fphdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, fphdr, name='exptime') # get gain p = spirouImage.GetGain(p, fphdr, name='gain') # get lamp parameters p = spirouTHORCA.GetLampParams(p, hchdr) # get FP_FP DPRTYPE p = spirouImage.ReadParam(p, fphdr, 'KW_DPRTYPE', 'DPRTYPE', dtype=str) # ---------------------------------------------------------------------- # Correction of reference FP # ---------------------------------------------------------------------- # set the number of frames p['NBFRAMES'] = len(fpfitsfiles) p.set_source('NBFRAMES', __NAME__ + '.main()') # Correction of DARK p, fpdatac = spirouImage.CorrectForDark(p, fpdata, fphdr) # Resize hc data # rotate the image and convert from ADU/s to e- fpdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, fpdatac), p=p) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) fpdata1 = spirouImage.ResizeImage(p, fpdata, **bkwargs) # log change in data size WLOG(p, '', ('FPref Image format changed to {0}x{1}').format(*fpdata1.shape)) # Correct for the BADPIX mask (set all bad pixels to zero) bargs = [p, fpdata1, fphdr] p, fpdata1 = spirouImage.CorrectForBadPix(*bargs) p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True) # log progress WLOG(p, '', 'Cleaning FPref hot pixels') # correct hot pixels fpdata1 = spirouEXTOR.CleanHotpix(fpdata1, badpixmask) # add to loc loc['FPDATA1'] = fpdata1 loc.set_source('FPDATA1', __NAME__ + '.main()') # Log the number of dead pixels # get the number of bad pixels with warnings.catch_warnings(record=True) as _: n_bad_pix = np.nansum(fpdata1 <= 0) n_bad_pix_frac = n_bad_pix * 100 / np.product(fpdata1.shape) # Log number wmsg = 'Nb FPref dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ---------------------------------------------------------------------- # Correction of HC # ---------------------------------------------------------------------- # set the number of frames p['NBFRAMES'] = 1 p.set_source('NBFRAMES', __NAME__ + '.main()') # Correction of DARK p, hcdatac = spirouImage.CorrectForDark(p, hcdata, hchdr) # Resize hc data # rotate the image and convert from ADU/s to e- hcdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, hcdatac), p=p) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) hcdata1 = spirouImage.ResizeImage(p, hcdata, **bkwargs) # log change in data size WLOG(p, '', ('HC Image format changed to {0}x{1}').format(*hcdata1.shape)) # Correct for the BADPIX mask (set all bad pixels to zero) bargs = [p, hcdata1, hchdr] p, hcdata1 = spirouImage.CorrectForBadPix(*bargs) p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True) # log progress WLOG(p, '', 'Cleaning HC hot pixels') # correct hot pixels hcdata1 = spirouEXTOR.CleanHotpix(hcdata1, badpixmask) # add to loc loc['HCDATA1'] = hcdata1 loc.set_source('HCDATA1', __NAME__ + '.main()') # Log the number of dead pixels # get the number of bad pixels with warnings.catch_warnings(record=True) as _: n_bad_pix = np.nansum(hcdata1 <= 0) n_bad_pix_frac = n_bad_pix * 100 / np.product(hcdata1.shape) # Log number wmsg = 'Nb HC dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ------------------------------------------------------------------------- # get all FP_FP files # ------------------------------------------------------------------------- fpfilenames = spirouImage.FindFiles(p, filetype=p['DPRTYPE'], allowedtypes=p['ALLOWED_FP_TYPES']) # convert filenames to a numpy array fpfilenames = np.array(fpfilenames) # julian date to know which file we need to # process together fp_time = np.zeros(len(fpfilenames)) basenames, fp_exp, fp_pp_version, nightnames = [], [], [], [] # log progress WLOG(p, '', 'Reading all fp file headers') # looping through the file headers for it in range(len(fpfilenames)): # log progress wmsg = '\tReading file {0} / {1}' WLOG(p, 'info', wmsg.format(it + 1, len(fpfilenames))) # get fp filename fpfilename = fpfilenames[it] # get night name night_name = os.path.dirname(fpfilenames[it]).split(p['TMP_DIR'])[-1] # read data data_it, hdr_it, _, _ = spirouImage.ReadImage(p, fpfilename) # get header hdr = spirouImage.ReadHeader(p, filepath=fpfilenames[it]) # add MJDATE to dark times fp_time[it] = float(hdr[p['KW_ACQTIME'][0]]) # add other keys (for tabular output) basenames.append(os.path.basename(fpfilenames[it])) nightnames.append(night_name) fp_exp.append(float(hdr[p['KW_EXPTIME'][0]])) fp_pp_version.append(hdr[p['KW_PPVERSION'][0]]) # ------------------------------------------------------------------------- # match files by date # ------------------------------------------------------------------------- # log progress wmsg = 'Matching FP files by observation time (+/- {0} hrs)' WLOG(p, '', wmsg.format(p['DARK_MASTER_MATCH_TIME'])) # get the time threshold time_thres = p['FP_MASTER_MATCH_TIME'] # get items grouped by time matched_id = spirouImage.GroupFilesByTime(p, fp_time, time_thres) # ------------------------------------------------------------------------- # construct the master fp file (+ correct for dark/badpix) # ------------------------------------------------------------------------- cargs = [fpdata1, fpfilenames, matched_id] fpcube, transforms = spirouImage.ConstructMasterFP(p, *cargs) # log process wmsg1 = 'Master FP construction complete.' wmsg2 = '\tAdding {0} group images to form FP master image' WLOG(p, 'info', [wmsg1, wmsg2.format(len(fpcube))]) # sum the cube to make fp data masterfp = np.sum(fpcube, axis=0) # add to loc loc['MASTERFP'] = masterfp loc.set_source('MASTERFP', __NAME__ + '.main()') # ------------------------------------------------------------------ # Get localisation coefficients # ------------------------------------------------------------------ # original there is a loop but it is not used --> removed p = spirouImage.FiberParams(p, p['FIBER'], merge=True) # get localisation fit coefficients p, loc = spirouLOCOR.GetCoeffs(p, fphdr, loc) # ------------------------------------------------------------------ # Get master wave solution map # ------------------------------------------------------------------ # get master wave map masterwavefile = spirouDB.GetDatabaseMasterWave(p) # log process wmsg1 = 'Getting master wavelength grid' wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile)) WLOG(p, '', [wmsg1, wmsg2]) # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # read master wave map wout = spirouImage.GetWaveSolution(p, filename=masterwavefile, return_wavemap=True, quiet=True, return_header=True, fiber=wave_fiber) loc['MASTERWAVEP'], loc['MASTERWAVE'] = wout[:2] loc['MASTERWAVEHDR'], loc['WSOURCE'] = wout[2:] # set sources wsource = ['MASTERWAVEP', 'MASTERWAVE', 'MASTERWAVEHDR'] loc.set_sources(wsource, 'spirouImage.GetWaveSolution()') # ---------------------------------------------------------------------- # Read UNe solution # ---------------------------------------------------------------------- wave_u_ne, amp_u_ne = spirouImage.ReadLineList(p) loc['LL_LINE'], loc['AMPL_LINE'] = wave_u_ne, amp_u_ne source = __NAME__ + '.main() + spirouImage.ReadLineList()' loc.set_sources(['LL_LINE', 'AMPL_LINE'], source) # ---------------------------------------------------------------------- # Read cavity length file # ---------------------------------------------------------------------- loc['CAVITY_LEN_COEFFS'] = spirouImage.ReadCavityLength(p) source = __NAME__ + '.main() + spirouImage.ReadCavityLength()' loc.set_source('CAVITY_LEN_COEFFS', source) # ---------------------------------------------------------------------- # Calculate shape map # ---------------------------------------------------------------------- # calculate dx map loc = spirouImage.GetXShapeMap(p, loc) # if dx map is None we shouldn't continue if loc['DXMAP'] is None: fargs = [ loc['MAXDXMAPINFO'][0], loc['MAXDXMAPINFO'][1], loc['MAXDXMAPSTD'], p['SHAPE_QC_DXMAP_STD'] ] fmsg = ('The std of the dxmap for order {0} y-pixel {1} is too large.' ' std = {2} (limit = {3})'.format(*fargs)) wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(fmsg)) WLOG(p, 'warning', 'Cannot continue. Exiting.') # End Message p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals()) # calculate dymap loc = spirouImage.GetYShapeMap(p, loc, fphdr) # ------------------------------------------------------------------ # Need to straighten the dxmap # ------------------------------------------------------------------ # copy it first loc['DXMAP0'] = np.array(loc['DXMAP']) # straighten it loc['DXMAP'] = spirouImage.EATransform(loc['DXMAP'], dymap=loc['DYMAP']) # ------------------------------------------------------------------ # Need to straighten the hc data and fp data for debug # ------------------------------------------------------------------ # log progress WLOG(p, '', 'Shape finding complete. Applying transforms.') # apply very last update of the debananafication tkwargs = dict(dxmap=loc['DXMAP'], dymap=loc['DYMAP']) loc['HCDATA2'] = spirouImage.EATransform(loc['HCDATA1'], **tkwargs) loc['FPDATA2'] = spirouImage.EATransform(loc['FPDATA1'], **tkwargs) loc.set_sources(['HCDATA2', 'FPDATA2'], __NAME__ + '.main()') # ------------------------------------------------------------------ # Plotting # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # plots setup: start interactive plot sPlt.start_interactive_session(p) # plot the shape process for one order sPlt.slit_shape_angle_plot(p, loc) # end interactive section sPlt.end_interactive_session(p) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # TODO: Decide on some quality control criteria? # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # add to qc header lists qc_values.append(loc['MAXDXMAPSTD']) qc_names.append('DXMAP STD') qc_logic.append('DXMAP STD < {0}'.format(p['SHAPE_QC_DXMAP_STD'])) qc_pass.append(1) # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Writing FP big table # ------------------------------------------------------------------ # construct big fp table colnames = [ 'FILENAME', 'NIGHT', 'MJDATE', 'EXPTIME', 'PVERSION', 'GROUPID', 'DXREF', 'DYREF', 'A', 'B', 'C', 'D' ] values = [ basenames, nightnames, fp_time, fp_exp, fp_pp_version, matched_id, transforms[:, 0], transforms[:, 1], transforms[:, 2], transforms[:, 3], transforms[:, 4], transforms[:, 5] ] fptable = spirouImage.MakeTable(p, colnames, values) # ------------------------------------------------------------------ # Writing DXMAP to file # ------------------------------------------------------------------ # get the raw tilt file name raw_shape_file = os.path.basename(p['FITSFILENAME']) # construct file name and path shapexfits, tag = spirouConfig.Constants.SLIT_XSHAPE_FILE(p) shapexfitsname = os.path.basename(shapexfits) # Log that we are saving tilt file wmsg = 'Saving shape x information in file: {0}' WLOG(p, '', wmsg.format(shapexfitsname)) # Copy keys from fits file hdict = spirouImage.CopyOriginalKeys(fphdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='hcfile', values=p['HCFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE2'], dim1name='fpfile', values=p['FPFILES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # write tilt file to file p = spirouImage.WriteImageTable(p, shapexfits, image=loc['DXMAP'], table=fptable, hdict=hdict) # ------------------------------------------------------------------ # Writing DYMAP to file # ------------------------------------------------------------------ # get the raw tilt file name raw_shape_file = os.path.basename(p['FITSFILENAME']) # construct file name and path shapeyfits, tag = spirouConfig.Constants.SLIT_YSHAPE_FILE(p) shapeyfitsname = os.path.basename(shapeyfits) # Log that we are saving tilt file wmsg = 'Saving shape y information in file: {0}' WLOG(p, '', wmsg.format(shapeyfitsname)) # Copy keys from fits file hdict = spirouImage.CopyOriginalKeys(fphdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='hcfile', values=p['HCFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE2'], dim1name='fpfile', values=p['FPFILES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # write tilt file to file p = spirouImage.WriteImageTable(p, shapeyfits, image=loc['DYMAP'], table=fptable, hdict=hdict) # ------------------------------------------------------------------ # Writing Master FP to file # ------------------------------------------------------------------ # get the raw tilt file name raw_shape_file = os.path.basename(p['FITSFILENAME']) # construct file name and path fpmasterfits, tag = spirouConfig.Constants.SLIT_MASTER_FP_FILE(p) fpmasterfitsname = os.path.basename(fpmasterfits) # Log that we are saving tilt file wmsg = 'Saving master FP file: {0}' WLOG(p, '', wmsg.format(fpmasterfitsname)) # Copy keys from fits file hdict = spirouImage.CopyOriginalKeys(fphdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='hcfile', values=p['HCFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE2'], dim1name='fpfile', values=p['FPFILES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # write tilt file to file p = spirouImage.WriteImageTable(p, fpmasterfits, image=masterfp, table=fptable, hdict=hdict) # ------------------------------------------------------------------ # Writing sanity check files # ------------------------------------------------------------------ if p['SHAPE_DEBUG_OUTPUTS']: # log WLOG(p, '', 'Saving debug sanity check files') # construct file names input_fp_file, tag1 = spirouConfig.Constants.SLIT_SHAPE_IN_FP_FILE(p) output_fp_file, tag2 = spirouConfig.Constants.SLIT_SHAPE_OUT_FP_FILE(p) input_hc_file, tag3 = spirouConfig.Constants.SLIT_SHAPE_IN_HC_FILE(p) output_hc_file, tag4 = spirouConfig.Constants.SLIT_SHAPE_OUT_HC_FILE(p) bdxmap_file, tag5 = spirouConfig.Constants.SLIT_SHAPE_BDXMAP_FILE(p) # write input fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) p = spirouImage.WriteImage(p, input_fp_file, loc['FPDATA1'], hdict) # write output fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) p = spirouImage.WriteImage(p, output_fp_file, loc['FPDATA2'], hdict) # write input fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3) p = spirouImage.WriteImage(p, input_hc_file, loc['HCDATA1'], hdict) # write output fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) p = spirouImage.WriteImage(p, output_hc_file, loc['HCDATA2'], hdict) # write overlap file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag5) p = spirouImage.WriteImage(p, bdxmap_file, loc['DXMAP0'], hdict) # ---------------------------------------------------------------------- # Move to calibDB and update calibDB # ---------------------------------------------------------------------- if p['QC']: # add shape x keydb = 'SHAPEX' # copy shape file to the calibDB folder spirouDB.PutCalibFile(p, shapexfits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, shapexfitsname, fphdr) # add shape y keydb = 'SHAPEY' # copy shape file to the calibDB folder spirouDB.PutCalibFile(p, shapeyfits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, shapeyfitsname, fphdr) # add fp master keydb = 'FPMASTER' # copy shape file to the calibDB folder spirouDB.PutCalibFile(p, fpmasterfits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, fpmasterfitsname, fphdr) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, fpfile=None, hcfiles=None): """ cal_WAVE_E2DS.py main function, if night_name and files are None uses arguments from run time i.e.: cal_DARK_spirou.py [night_directory] [fpfile] [hcfiles] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param fpfile: string, or None, the FP file to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :param hcfiles: string, list or None, the list of HC files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # test files TC2 # night_name = 'AT5/AT5-12/2018-05-29_17-41-44/' # fpfile = '2279844a_fp_fp_pp_e2dsff_AB.fits' # hcfiles = ['2279845c_hc_pp_e2dsff_AB.fits'] # test files TC3 # night_name = 'TC3/AT5/AT5-12/2018-07-24_16-17-57/' # fpfile = '2294108a_pp_e2dsff_AB.fits' # hcfiles = ['2294115c_pp_e2dsff_AB.fits'] # night_name = 'TC3/AT5/AT5-12/2018-07-25_16-49-50/' # fpfile = '2294223a_pp_e2dsff_AB.fits' # hcfiles = ['2294230c_pp_e2dsff_AB.fits'] # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) if hcfiles is None or fpfile is None: names, types = ['fpfile', 'hcfiles'], [str, str] customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1], types, names, last_multi=True) else: customargs = dict(hcfiles=hcfiles, fpfile=fpfile) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsdir='reduced', mainfitsfile='hcfiles') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, fpfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['FPFILE']) fiber1 = str(p['FIBER']) p, hcfilenames = spirouStartup.MultiFileSetup(p, files=p['HCFILES']) fiber2 = str(p['FIBER']) # set the hcfilename to the first hcfilenames hcfitsfilename = hcfilenames[0] # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Have to check that the fibers match # ---------------------------------------------------------------------- if fiber1 == fiber2: p['FIBER'] = fiber1 fsource = __NAME__ + '/main() & spirouStartup.GetFiberType()' p.set_source('FIBER', fsource) else: emsg = 'Fiber not matching for {0} and {1}, should be the same' eargs = [hcfitsfilename, fpfitsfilename] WLOG(p, 'error', emsg.format(*eargs)) # set the fiber type p['FIB_TYP'] = [p['FIBER']] p.set_source('FIB_TYP', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Read FP and HC files # ---------------------------------------------------------------------- # read and combine all HC files except the first (fpfitsfilename) rargs = [p, 'add', hcfitsfilename, hcfilenames[1:]] p, hcdata, hchdr = spirouImage.ReadImageAndCombine(*rargs) # read first file (fpfitsfilename) fpdata, fphdr, _, _ = spirouImage.ReadImage(p, fpfitsfilename) # TODO: ------------------------------------------------------------ # TODO remove to test NaNs # TODO: ------------------------------------------------------------ # hcmask = np.isfinite(hcdata) # fpmask = np.isfinite(fpdata) # hcdata[~hcmask] = 0.0 # fpdata[~fpmask] = 0.0 # TODO: ------------------------------------------------------------ # add data and hdr to loc loc = ParamDict() loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr # set the source sources = ['HCDATA', 'HCHDR'] loc.set_sources(sources, 'spirouImage.ReadImageAndCombine()') sources = ['FPDATA', 'FPHDR'] loc.set_sources(sources, 'spirouImage.ReadImage()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hchdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hchdr, name='exptime') # get gain p = spirouImage.GetGain(p, hchdr, name='gain') # get acquisition time p = spirouImage.GetAcqTime(p, hchdr, name='acqtime', kind='julian') bjdref = p['ACQTIME'] # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # get lamp parameters p = spirouTHORCA.GetLampParams(p, hchdr) # get number of orders # we always get fibre A number because AB is doubled in constants file loc['NBO'] = p['QC_LOC_NBO_FPALL']['A'] loc.set_source('NBO', __NAME__ + '.main()') # get number of pixels in x from hcdata size loc['NBPIX'] = loc['HCDATA'].shape[1] loc.set_source('NBPIX', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Read blaze # ---------------------------------------------------------------------- # get tilts p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hchdr) loc.set_source('BLAZE', __NAME__ + '/main() + /spirouImage.ReadBlazeFile') # make copy of blaze (as it's overwritten later) loc['BLAZE2'] = np.copy(loc['BLAZE']) # ---------------------------------------------------------------------- # Read wave solution # ---------------------------------------------------------------------- # wavelength file; we will use the polynomial terms in its header, # NOT the pixel values that would need to be interpolated # set source of wave file wsource = __NAME__ + '/main() + /spirouImage.GetWaveSolution' # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get wave image wout = spirouImage.GetWaveSolution(p, hdr=hchdr, return_wavemap=True, return_filename=True, fiber=wave_fiber) loc['WAVEPARAMS'], loc['WAVE_INIT'], loc['WAVEFILE'], loc['WSOURCE'] = wout loc.set_sources(['WAVE_INIT', 'WAVEFILE', 'WAVEPARAMS', 'WSOURCE'], wsource) poly_wave_sol = loc['WAVEPARAMS'] # ---------------------------------------------------------------------- # Check that wave parameters are consistent with "ic_ll_degr_fit" # ---------------------------------------------------------------------- loc = spirouImage.CheckWaveSolConsistency(p, loc) # ---------------------------------------------------------------------- # Read UNe solution # ---------------------------------------------------------------------- wave_u_ne, amp_u_ne = spirouImage.ReadLineList(p) loc['LL_LINE'], loc['AMPL_LINE'] = wave_u_ne, amp_u_ne source = __NAME__ + '.main() + spirouImage.ReadLineList()' loc.set_sources(['ll_line', 'ampl_line'], source) # ---------------------------------------------------------------------- # Generate wave map from wave solution # ---------------------------------------------------------------------- loc = spirouWAVE.generate_wave_map(p, loc) # ---------------------------------------------------------------------- # Find Gaussian Peaks in HC spectrum # ---------------------------------------------------------------------- loc = spirouWAVE.find_hc_gauss_peaks(p, loc) # ---------------------------------------------------------------------- # Start plotting session # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # start interactive plot sPlt.start_interactive_session(p) # ---------------------------------------------------------------------- # Fit Gaussian peaks (in triplets) to # ---------------------------------------------------------------------- loc = spirouWAVE.fit_gaussian_triplets(p, loc) # ---------------------------------------------------------------------- # Generate Resolution map and line profiles # ---------------------------------------------------------------------- # log progress wmsg = 'Generating resolution map and ' # generate resolution map loc = spirouWAVE.generate_resolution_map(p, loc) # map line profile map if p['DRS_PLOT'] > 0: sPlt.wave_ea_plot_line_profiles(p, loc) # ---------------------------------------------------------------------- # End plotting session # ---------------------------------------------------------------------- # end interactive session if p['DRS_PLOT'] > 0: sPlt.end_interactive_session(p) # ---------------------------------------------------------------------- # Set up all_lines storage # ---------------------------------------------------------------------- # initialise up all_lines storage all_lines_1 = [] # get parameters from p n_ord_start = p['IC_HC_N_ORD_START_2'] n_ord_final = p['IC_HC_N_ORD_FINAL_2'] pixel_shift_inter = p['PIXEL_SHIFT_INTER'] pixel_shift_slope = p['PIXEL_SHIFT_SLOPE'] # get values from loc xgau = np.array(loc['XGAU_T']) dv = np.array(loc['DV_T']) fit_per_order = np.array(loc['POLY_WAVE_SOL']) ew = np.array(loc['EW_T']) peak = np.array(loc['PEAK_T']) amp_catalog = np.array(loc['AMP_CATALOG']) wave_catalog = np.array(loc['WAVE_CATALOG']) ord_t = np.array(loc['ORD_T']) # loop through orders for iord in range(n_ord_start, n_ord_final): # keep relevant lines # -> right order # -> finite dv gg = (ord_t == iord) & (np.isfinite(dv)) nlines = np.nansum(gg) # put lines into ALL_LINES structure # reminder: # gparams[0] = output wavelengths # gparams[1] = output sigma(gauss fit width) # gparams[2] = output amplitude(gauss fit) # gparams[3] = difference in input / output wavelength # gparams[4] = input amplitudes # gparams[5] = output pixel positions # gparams[6] = output pixel sigma width (gauss fit width in pixels) # gparams[7] = output weights for the pixel position chebval = np.polynomial.chebyshev.chebval # dummy array for weights test = np.ones(np.shape(xgau[gg]), 'd') * 1e4 # get the final wavelength value for each peak in order output_wave_1 = np.polyval(fit_per_order[iord][::-1], xgau[gg]) # output_wave_1 = chebval(xgau[gg], fit_per_order[iord]) # convert the pixel equivalent width to wavelength units xgau_ew_ini = xgau[gg] - ew[gg] / 2 xgau_ew_fin = xgau[gg] + ew[gg] / 2 ew_ll_ini = np.polyval(fit_per_order[iord, :], xgau_ew_ini) ew_ll_fin = np.polyval(fit_per_order[iord, :], xgau_ew_fin) # ew_ll_ini = chebval(xgau_ew_ini, fit_per_order[iord]) # ew_ll_fin = chebval(xgau_ew_fin, fit_per_order[iord]) ew_ll = ew_ll_fin - ew_ll_ini # put all lines in the order into array gau_params = np.column_stack( (output_wave_1, ew_ll, peak[gg], wave_catalog[gg] - output_wave_1, amp_catalog[gg], xgau[gg], ew[gg], test)) # append the array for the order into a list all_lines_1.append(gau_params) # save dv in km/s and auxiliary order number # res_1 = np.concatenate((res_1,2.997e5*(input_wave - output_wave_1)/ # output_wave_1)) # ord_save = np.concatenate((ord_save, test*iord)) # add to loc loc['ALL_LINES_1'] = all_lines_1 loc['LL_PARAM_1'] = np.array(fit_per_order) loc['LL_OUT_1'] = np.array(loc['WAVE_MAP2']) loc.set_sources(['ALL_LINES_1', 'LL_PARAM_1'], __NAME__ + '/main()') # For compatibility w/already defined functions, I need to save # here all_lines_2 all_lines_2 = list(all_lines_1) loc['ALL_LINES_2'] = all_lines_2 # loc['LL_PARAM_2'] = np.fliplr(fit_per_order) # loc['LL_OUT_2'] = np.array(loc['WAVE_MAP2']) # loc.set_sources(['ALL_LINES_2', 'LL_PARAM_2'], __NAME__ + '/main()') # ------------------------------------------------------------------ # Littrow test # ------------------------------------------------------------------ start = p['IC_LITTROW_ORDER_INIT_1'] end = p['IC_LITTROW_ORDER_FINAL_1'] # calculate echelle orders o_orders = np.arange(start, end) echelle_order = p['IC_HC_T_ORDER_START'] - o_orders loc['ECHELLE_ORDERS'] = echelle_order loc.set_source('ECHELLE_ORDERS', __NAME__ + '/main()') # reset Littrow fit degree p['IC_LITTROW_FIT_DEG_1'] = 7 # Do Littrow check ckwargs = dict(ll=loc['LL_OUT_1'][start:end, :], iteration=1, log=True) loc = spirouTHORCA.CalcLittrowSolution(p, loc, **ckwargs) # Plot wave solution littrow check if p['DRS_PLOT'] > 0: # plot littrow x pixels against fitted wavelength solution sPlt.wave_littrow_check_plot(p, loc, iteration=1) # ------------------------------------------------------------------ # extrapolate Littrow solution # ------------------------------------------------------------------ ekwargs = dict(ll=loc['LL_OUT_1'], iteration=1) loc = spirouTHORCA.ExtrapolateLittrowSolution(p, loc, **ekwargs) # ------------------------------------------------------------------ # Plot littrow solution # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # plot littrow x pixels against fitted wavelength solution sPlt.wave_littrow_extrap_plot(p, loc, iteration=1) # ------------------------------------------------------------------ # Incorporate FP into solution # ------------------------------------------------------------------ # Copy LL_OUT_1 and LL_PARAM_1 into new constants (for FP integration) loc['LITTROW_EXTRAP_SOL_1'] = np.array(loc['LL_OUT_1']) loc['LITTROW_EXTRAP_PARAM_1'] = np.array(loc['LL_PARAM_1']) # only use FP if switched on in constants file if p['IC_WAVE_USE_FP']: # ------------------------------------------------------------------ # Find FP lines # ------------------------------------------------------------------ # print message to screen wmsg = 'Identification of lines in reference file: {0}' WLOG(p, '', wmsg.format(fpfile)) # ------------------------------------------------------------------ # Get the FP solution # ------------------------------------------------------------------ loc = spirouTHORCA.FPWaveSolutionNew(p, loc) # ------------------------------------------------------------------ # FP solution plots # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # Plot the FP extracted spectrum against wavelength solution sPlt.wave_plot_final_fp_order(p, loc, iteration=1) # Plot the measured FP cavity width offset against line number sPlt.wave_local_width_offset_plot(p, loc) # Plot the FP line wavelength residuals sPlt.wave_fp_wavelength_residuals(p, loc) # ------------------------------------------------------------------ # Create new wavelength solution # ------------------------------------------------------------------ # TODO: Melissa fault - fix later p['IC_HC_N_ORD_START_2'] = min(p['IC_HC_N_ORD_START_2'], p['IC_FP_N_ORD_START']) p['IC_HC_N_ORD_FINAL_2'] = max(p['IC_HC_N_ORD_FINAL_2'], p['IC_FP_N_ORD_FINAL']) start = p['IC_HC_N_ORD_START_2'] end = p['IC_HC_N_ORD_FINAL_2'] # recalculate echelle orders for Fit1DSolution o_orders = np.arange(start, end) echelle_order = p['IC_HC_T_ORDER_START'] - o_orders loc['ECHELLE_ORDERS'] = echelle_order loc.set_source('ECHELLE_ORDERS', __NAME__ + '/main()') # select the orders to fit lls = loc['LITTROW_EXTRAP_SOL_1'][start:end] loc = spirouTHORCA.Fit1DSolution(p, loc, lls, iteration=2) # from here, LL_OUT_2 wil be 0-47 # ------------------------------------------------------------------ # Repeat Littrow test # ------------------------------------------------------------------ start = p['IC_LITTROW_ORDER_INIT_2'] end = p['IC_LITTROW_ORDER_FINAL_2'] # recalculate echelle orders for Littrow check o_orders = np.arange(start, end) echelle_order = p['IC_HC_T_ORDER_START'] - o_orders loc['ECHELLE_ORDERS'] = echelle_order loc.set_source('ECHELLE_ORDERS', __NAME__ + '/main()') # Do Littrow check ckwargs = dict(ll=loc['LL_OUT_2'][start:end, :], iteration=2, log=True) loc = spirouTHORCA.CalcLittrowSolution(p, loc, **ckwargs) # Plot wave solution littrow check if p['DRS_PLOT'] > 0: # plot littrow x pixels against fitted wavelength solution sPlt.wave_littrow_check_plot(p, loc, iteration=2) # ------------------------------------------------------------------ # extrapolate Littrow solution # ------------------------------------------------------------------ ekwargs = dict(ll=loc['LL_OUT_2'], iteration=2) loc = spirouTHORCA.ExtrapolateLittrowSolution(p, loc, **ekwargs) # ------------------------------------------------------------------ # Plot littrow solution # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # plot littrow x pixels against fitted wavelength solution sPlt.wave_littrow_extrap_plot(p, loc, iteration=2) # ------------------------------------------------------------------ # Join 0-47 and 47-49 solutions # ------------------------------------------------------------------ loc = spirouTHORCA.JoinOrders(p, loc) # ------------------------------------------------------------------ # Plot single order, wavelength-calibrated, with found lines # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: sPlt.wave_ea_plot_single_order(p, loc) # ---------------------------------------------------------------------- # Do correlation on FP spectra # ---------------------------------------------------------------------- # ------------------------------------------------------------------ # Compute photon noise uncertainty for FP # ------------------------------------------------------------------ # set up the arguments for DeltaVrms2D dargs = [loc['FPDATA'], loc['LL_FINAL']] dkwargs = dict(sigdet=p['IC_DRIFT_NOISE'], size=p['IC_DRIFT_BOXSIZE'], threshold=p['IC_DRIFT_MAXFLUX']) # run DeltaVrms2D dvrmsref, wmeanref = spirouRV.DeltaVrms2D(*dargs, **dkwargs) # save to loc loc['DVRMSREF'], loc['WMEANREF'] = dvrmsref, wmeanref loc.set_sources(['dvrmsref', 'wmeanref'], __NAME__ + '/main()()') # log the estimated RV uncertainty wmsg = 'On fiber {0} estimated RV uncertainty on spectrum is {1:.3f} m/s' WLOG(p, 'info', wmsg.format(p['FIBER'], wmeanref)) # Use CCF Mask function with drift constants p['CCF_MASK'] = p['DRIFT_CCF_MASK'] p['TARGET_RV'] = p['DRIFT_TARGET_RV'] p['CCF_WIDTH'] = p['DRIFT_CCF_WIDTH'] p['CCF_STEP'] = p['DRIFT_CCF_STEP'] p['RVMIN'] = p['TARGET_RV'] - p['CCF_WIDTH'] p['RVMAX'] = p['TARGET_RV'] + p['CCF_WIDTH'] + p['CCF_STEP'] # get the CCF mask from file (check location of mask) loc = spirouRV.GetCCFMask(p, loc) # TODO Check why Blaze makes bugs in correlbin loc['BLAZE'] = np.ones((loc['NBO'], loc['NBPIX'])) # set sources # loc.set_sources(['flat', 'blaze'], __NAME__ + '/main()') loc.set_source('blaze', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Do correlation on FP # ---------------------------------------------------------------------- # calculate and fit the CCF loc['E2DSFF'] = np.array(loc['FPDATA']) loc.set_source('E2DSFF', __NAME__ + '/main()') p['CCF_FIT_TYPE'] = 1 loc['BERV'] = 0.0 loc['BERV_MAX'] = 0.0 loc['BJD'] = 0.0 # run the RV coravelation function with these parameters loc['WAVE_LL'] = np.array(loc['LL_FINAL']) loc['PARAM_LL'] = np.array(loc['LL_PARAM_FINAL']) loc = spirouRV.Coravelation(p, loc) # ---------------------------------------------------------------------- # Update the Correlation stats with values using fiber C (FP) drift # ---------------------------------------------------------------------- # get the maximum number of orders to use nbmax = p['CCF_NUM_ORDERS_MAX'] # get the average ccf loc['AVERAGE_CCF'] = np.nansum(loc['CCF'][:nbmax], axis=0) # normalize the average ccf normalized_ccf = loc['AVERAGE_CCF'] / np.nanmax(loc['AVERAGE_CCF']) # get the fit for the normalized average ccf ccf_res, ccf_fit = spirouRV.FitCCF(p, loc['RV_CCF'], normalized_ccf, fit_type=1) loc['CCF_RES'] = ccf_res loc['CCF_FIT'] = ccf_fit # get the max cpp loc['MAXCPP'] = np.nansum(loc['CCF_MAX']) / np.nansum( loc['PIX_PASSED_ALL']) # get the RV value from the normalised average ccf fit center location loc['RV'] = float(ccf_res[1]) # get the contrast (ccf fit amplitude) loc['CONTRAST'] = np.abs(100 * ccf_res[0]) # get the FWHM value loc['FWHM'] = ccf_res[2] * spirouCore.spirouMath.fwhm() # set the source keys = [ 'AVERAGE_CCF', 'MAXCPP', 'RV', 'CONTRAST', 'FWHM', 'CCF_RES', 'CCF_FIT' ] loc.set_sources(keys, __NAME__ + '/main()') # ---------------------------------------------------------------------- # log the stats wmsg = ('FP Correlation: C={0:.1f}[%] DRIFT={1:.5f}[km/s] ' 'FWHM={2:.4f}[km/s] maxcpp={3:.1f}') wargs = [loc['CONTRAST'], float(ccf_res[1]), loc['FWHM'], loc['MAXCPP']] WLOG(p, 'info', wmsg.format(*wargs)) # ---------------------------------------------------------------------- # rv ccf plot # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # Plot rv vs ccf (and rv vs ccf_fit) p['OBJNAME'] = 'FP' sPlt.ccf_rv_ccf_plot(p, loc['RV_CCF'], normalized_ccf, ccf_fit) # TODO : Add QC of the FP CCF # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # get parameters ffrom p p['QC_RMS_LITTROW_MAX'] = p['QC_HC_RMS_LITTROW_MAX'] p['QC_DEV_LITTROW_MAX'] = p['QC_HC_DEV_LITTROW_MAX'] # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # quality control on sigma clip (sig1 > qc_hc_wave_sigma_max if loc['SIG1'] > p['QC_HC_WAVE_SIGMA_MAX']: fmsg = 'Sigma too high ({0:.5f} > {1:.5f})' fail_msg.append(fmsg.format(loc['SIG1'], p['QC_HC_WAVE_SIGMA_MAX'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(loc['SIG1']) qc_names.append('SIG1') qc_logic.append('SIG1 > {0:.2f}'.format(p['QC_HC_WAVE_SIGMA_MAX'])) # ---------------------------------------------------------------------- # check the difference between consecutive orders is always positive # get the differences wave_diff = loc['LL_FINAL'][1:] - loc['LL_FINAL'][:-1] if np.min(wave_diff) < 0: fmsg = 'Negative wavelength difference between orders' fail_msg.append(fmsg) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(np.min(wave_diff)) qc_names.append('MIN WAVE DIFF') qc_logic.append('MIN WAVE DIFF < 0') # ---------------------------------------------------------------------- # check for infinites and NaNs in mean residuals from fit if ~np.isfinite(loc['X_MEAN_2']): # add failed message to the fail message list fmsg = 'NaN or Inf in X_MEAN_2' fail_msg.append(fmsg) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(loc['X_MEAN_2']) qc_names.append('X_MEAN_2') qc_logic.append('X_MEAN_2 not finite') # ---------------------------------------------------------------------- # iterate through Littrow test cut values lit_it = 2 # checks every other value # TODO: This QC check (or set of QC checks needs re-writing it is # TODO: nearly impossible to understand for x_it in range(1, len(loc['X_CUT_POINTS_' + str(lit_it)]), 2): # get x cut point x_cut_point = loc['X_CUT_POINTS_' + str(lit_it)][x_it] # get the sigma for this cut point sig_littrow = loc['LITTROW_SIG_' + str(lit_it)][x_it] # get the abs min and max dev littrow values min_littrow = abs(loc['LITTROW_MINDEV_' + str(lit_it)][x_it]) max_littrow = abs(loc['LITTROW_MAXDEV_' + str(lit_it)][x_it]) # get the corresponding order min_littrow_ord = loc['LITTROW_MINDEVORD_' + str(lit_it)][x_it] max_littrow_ord = loc['LITTROW_MAXDEVORD_' + str(lit_it)][x_it] # check if sig littrow is above maximum rms_littrow_max = p['QC_RMS_LITTROW_MAX'] dev_littrow_max = p['QC_DEV_LITTROW_MAX'] if sig_littrow > rms_littrow_max: fmsg = ('Littrow test (x={0}) failed (sig littrow = ' '{1:.2f} > {2:.2f})') fargs = [x_cut_point, sig_littrow, rms_littrow_max] fail_msg.append(fmsg.format(*fargs)) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(sig_littrow) qc_names.append('sig_littrow') qc_logic.append('sig_littrow > {0:.2f}'.format(rms_littrow_max)) # ---------------------------------------------------------------------- # check if min/max littrow is out of bounds if np.max([max_littrow, min_littrow]) > dev_littrow_max: fmsg = ('Littrow test (x={0}) failed (min|max dev = ' '{1:.2f}|{2:.2f} > {3:.2f} for order {4}|{5})') fargs = [ x_cut_point, min_littrow, max_littrow, dev_littrow_max, min_littrow_ord, max_littrow_ord ] fail_msg.append(fmsg.format(*fargs)) passed = False qc_pass.append(0) # TODO: Should this be the QC header values? # TODO: it does not change the outcome of QC (i.e. passed=False) # TODO: So what is the point? # if sig was out of bounds, recalculate if sig_littrow > rms_littrow_max: # conditions check1 = min_littrow > dev_littrow_max check2 = max_littrow > dev_littrow_max # get the residuals respix = loc['LITTROW_YY_' + str(lit_it)][x_it] # check if both are out of bounds if check1 and check2: # remove respective orders worst_order = (min_littrow_ord, max_littrow_ord) respix_2 = np.delete(respix, worst_order) redo_sigma = True # check if min is out of bounds elif check1: # remove respective order worst_order = min_littrow_ord respix_2 = np.delete(respix, worst_order) redo_sigma = True # check if max is out of bounds elif check2: # remove respective order worst_order = max_littrow_ord respix_2 = np.delete(respix, max_littrow_ord) redo_sigma = True # else do not recalculate sigma else: redo_sigma, respix_2, worst_order = False, None, None wmsg = 'No outlying orders, sig littrow not recalculated' fail_msg.append(wmsg.format()) # if outlying order, recalculate stats if redo_sigma: mean = np.nansum(respix_2) / len(respix_2) mean2 = np.nansum(respix_2**2) / len(respix_2) rms = np.sqrt(mean2 - mean**2) if rms > rms_littrow_max: fmsg = ('Littrow test (x={0}) failed (sig littrow = ' '{1:.2f} > {2:.2f} removing order {3})') fargs = [ x_cut_point, rms, rms_littrow_max, worst_order ] fail_msg.append(fmsg.format(*fargs)) else: wargs = [ x_cut_point, rms, rms_littrow_max, worst_order ] wmsg = ('Littrow test (x={0}) passed (sig littrow = ' '{1:.2f} > {2:.2f} removing order {3})') fail_msg.append(wmsg.format(*wargs)) else: qc_pass.append(1) # add to qc header lists qc_values.append(np.max([max_littrow, min_littrow])) qc_names.append('max or min littrow') qc_logic.append('max or min littrow > {0:.2f}' ''.format(dev_littrow_max)) # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # archive result in e2ds spectra # ------------------------------------------------------------------ # get raw input file name(s) raw_infiles1 = [] for hcfile in p['HCFILES']: raw_infiles1.append(os.path.basename(hcfile)) raw_infile2 = os.path.basename(p['FPFILE']) # get wave filename wavefits, tag1 = spirouConfig.Constants.WAVE_FILE_EA_2(p) wavefitsname = os.path.split(wavefits)[-1] # log progress wargs = [p['FIBER'], wavefits] wmsg = 'Write wavelength solution for Fiber {0} in {1}' WLOG(p, '', wmsg.format(*wargs)) # write solution to fitsfilename header # copy original keys hdict = spirouImage.CopyOriginalKeys(loc['HCHDR']) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='fpfile', values=p['FPFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE2'], dim1name='hcfile', values=p['HCFILES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # add wave solution date hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME1'], value=p['MAX_TIME_HUMAN']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME2'], value=p['MAX_TIME_UNIX']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_CODE'], value=__NAME__) # add number of orders hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_ORD_N'], value=loc['LL_PARAM_FINAL'].shape[0]) # add degree of fit hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_LL_DEG'], value=loc['LL_PARAM_FINAL'].shape[1] - 1) # add wave solution hdict = spirouImage.AddKey2DList(p, hdict, p['KW_WAVE_PARAM'], values=loc['LL_PARAM_FINAL']) # add FP CCF drift # target RV and width hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_TARG_RV'], value=p['TARGET_RV']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_WIDTH'], value=p['CCF_WIDTH']) # the rv step # rvstep = np.abs(loc['RV_CCF'][0] - loc['RV_CCF'][1]) # hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CDELT'], value=rvstep) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_STEP'], value=p['CCF_STEP']) # add ccf stats hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_DRIFT'], value=loc['CCF_RES'][1]) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_FWHM'], value=loc['FWHM']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_CONTRAST'], value=loc['CONTRAST']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_MAXCPP'], value=loc['MAXCPP']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_MASK'], value=p['CCF_MASK']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_LINES'], value=np.nansum(loc['TOT_LINE'])) # write the wave "spectrum" hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) p = spirouImage.WriteImage(p, wavefits, loc['LL_FINAL'], hdict) # get filename for E2DS calibDB copy of FITSFILENAME e2dscopy_filename = spirouConfig.Constants.WAVE_E2DS_COPY(p)[0] wargs = [p['FIBER'], os.path.split(e2dscopy_filename)[-1]] wmsg = 'Write reference E2DS spectra for Fiber {0} in {1}' WLOG(p, '', wmsg.format(*wargs)) # make a copy of the E2DS file for the calibBD p = spirouImage.WriteImage(p, e2dscopy_filename, loc['HCDATA'], hdict) # only copy over if QC passed if p['QC']: # loop around hc files and update header with for hcfile in p['HCFILES']: raw_infilepath1 = os.path.join(p['ARG_FILE_DIR'], hcfile) p = spirouImage.UpdateWaveSolution(p, loc, raw_infilepath1) # update fp file raw_infilepath2 = os.path.join(p['ARG_FILE_DIR'], raw_infile2) p = spirouImage.UpdateWaveSolution(p, loc, raw_infilepath2) # ------------------------------------------------------------------ # Save to result table # ------------------------------------------------------------------ # calculate stats for table final_mean = 1000 * loc['X_MEAN_2'] final_var = 1000 * loc['X_VAR_2'] num_lines = int(np.nansum(loc['X_ITER_2'][:, 2])) # loc['X_ITER_2'] err = 1000 * np.sqrt(loc['X_VAR_2'] / num_lines) sig_littrow = 1000 * np.array(loc['LITTROW_SIG_' + str(lit_it)]) # construct filename wavetbl = spirouConfig.Constants.WAVE_TBL_FILE_EA(p) wavetblname = os.path.basename(wavetbl) # construct and write table columnnames = [ 'night_name', 'file_name', 'fiber', 'mean', 'rms', 'N_lines', 'err', 'rms_L500', 'rms_L1000', 'rms_L1500', 'rms_L2000', 'rms_L2500', 'rms_L3000', 'rms_L3500' ] columnformats = [ '{:20s}', '{:30s}', '{:3s}', '{:7.4f}', '{:6.2f}', '{:3d}', '{:6.3f}', '{:6.2f}', '{:6.2f}', '{:6.2f}', '{:6.2f}', '{:6.2f}', '{:6.2f}', '{:6.2f}' ] columnvalues = [[p['ARG_NIGHT_NAME']], [p['ARG_FILE_NAMES'][0]], [p['FIBER']], [final_mean], [final_var], [num_lines], [err], [sig_littrow[0]], [sig_littrow[1]], [sig_littrow[2]], [sig_littrow[3]], [sig_littrow[4]], [sig_littrow[5]], [sig_littrow[6]]] # make table table = spirouImage.MakeTable(p, columns=columnnames, values=columnvalues, formats=columnformats) # merge table wmsg = 'Global result summary saved in {0}' WLOG(p, '', wmsg.format(wavetblname)) spirouImage.MergeTable(p, table, wavetbl, fmt='ascii.rst') # ---------------------------------------------------------------------- # Save resolution and line profiles to file # ---------------------------------------------------------------------- raw_infile = os.path.basename(p['FITSFILENAME']) # get wave filename resfits, tag3 = spirouConfig.Constants.WAVE_RES_FILE_EA(p) resfitsname = os.path.basename(resfits) WLOG(p, '', 'Saving wave resmap to {0}'.format(resfitsname)) # make a copy of the E2DS file for the calibBD # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3) # get res data in correct format resdata, hdicts = spirouTHORCA.GenerateResFiles(p, loc, hdict) # save to file p = spirouImage.WriteImageMulti(p, resfits, resdata, hdicts=hdicts) # ------------------------------------------------------------------ # Save line list table file # ------------------------------------------------------------------ # construct filename # TODO proper column values wavelltbl = spirouConfig.Constants.WAVE_LINE_FILE_EA(p) wavelltblname = os.path.split(wavelltbl)[-1] # construct and write table columnnames = ['order', 'll', 'dv', 'w', 'xi', 'xo', 'dvdx'] columnformats = [ '{:.0f}', '{:12.4f}', '{:13.5f}', '{:12.4f}', '{:12.4f}', '{:12.4f}', '{:8.4f}' ] columnvalues = [] # construct column values (flatten over orders) for it in range(len(loc['X_DETAILS_2'])): for jt in range(len(loc['X_DETAILS_2'][it][0])): row = [ float(it), loc['X_DETAILS_2'][it][0][jt], loc['LL_DETAILS_2'][it][0][jt], loc['X_DETAILS_2'][it][3][jt], loc['X_DETAILS_2'][it][1][jt], loc['X_DETAILS_2'][it][2][jt], loc['SCALE_2'][it][jt] ] columnvalues.append(row) # log saving wmsg = 'List of lines used saved in {0}' WLOG(p, '', wmsg.format(wavelltblname)) # make table columnvalues = np.array(columnvalues).T table = spirouImage.MakeTable(p, columns=columnnames, values=columnvalues, formats=columnformats) # write table spirouImage.WriteTable(p, table, wavelltbl, fmt='ascii.rst') # ------------------------------------------------------------------ # Move to calibDB and update calibDB # ------------------------------------------------------------------ if p['QC']: # set the wave key keydb = 'WAVE_{0}'.format(p['FIBER']) # copy wave file to calibDB folder spirouDB.PutCalibFile(p, wavefits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, wavefitsname, loc['HCHDR']) # set the hcref key keydb = 'HCREF_{0}'.format(p['FIBER']) # copy wave file to calibDB folder spirouDB.PutCalibFile(p, e2dscopy_filename) # update the master calib DB file with new key e2dscopyfits = os.path.split(e2dscopy_filename)[-1] spirouDB.UpdateCalibMaster(p, keydb, e2dscopyfits, loc['HCHDR']) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return p and loc return dict(locals())
def main(directory=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with arguments being None (i.e. get from sys.argv) pos = [0] fmt = [str, str] names = ['directory'] call = [directory] # now get custom arguments customargs = spirouStartup.GetCustomFromRuntime(p, pos, fmt, names, calls=call, require_night_name=False) p = spirouStartup.LoadArguments(p, customargs=customargs, require_night_name=False) # ---------------------------------------------------------------------- # Check if we have an index file # ---------------------------------------------------------------------- # get expected index file name and location index_file = spirouConfig.Constants.INDEX_OUTPUT_FILENAME() path = p['DIRECTORY'] index_path = os.path.join(path, index_file) # get expected columns columns = spirouConfig.Constants.GEN_OUTPUT_COLUMNS(p) # create storage loc = OrderedDict() # if file exists then we have some indexed files if os.path.exists(index_path): rawloc = spirouImage.ReadFitsTable(p, index_path) loc['FILENAME'] = list(rawloc['FILENAME']) loc['LAST_MODIFIED'] = list(rawloc['LAST_MODIFIED']) for col in columns: if col not in rawloc.keys(): WLOG(p, '', '\t- Skipping column {0}'.format(col)) loc[col] = list(np.repeat([''], len(loc['FILENAME']))) else: loc[col] = list(rawloc[col]) # else we have to create this file else: loc['FILENAME'] = [] loc['LAST_MODIFIED'] = [] # loop around columns and add blank list to each for col in columns: loc[col] = [] # ---------------------------------------------------------------------- # Get all files in raw night_name directory # ---------------------------------------------------------------------- # get all files in DRS_DATA_RAW/ARG_NIGHT_NAME files = os.listdir(p['DIRECTORY']) # sort file by name files = np.sort(files) # ---------------------------------------------------------------------- # Loop around all files and extract required header keys # ---------------------------------------------------------------------- # log progress WLOG(p, '', 'Analysing {0} files'.format(len(files))) # loop around files and extract properties for filename in files: # skip any non-fits file files if '.fits' not in filename: continue # skip the index file if filename == os.path.basename(index_file): continue # skip non-preprocessed files (without .fits) if p['PROCESSED_SUFFIX'].split('.fits')[0] not in filename: continue # if already in loc['FILENAME'] then skip if filename in loc['FILENAME']: continue # construct absolute path for file fitsfilename = os.path.join(p['DIRECTORY'], filename) # read file header hdr = spirouImage.ReadHeader(p, filepath=fitsfilename) # add filename loc['FILENAME'].append(filename) loc['LAST_MODIFIED'].append(os.path.getmtime(fitsfilename)) # loop around columns and look for key in header for col in columns: # get value from header if col in hdr: value = str(hdr[col]) else: value = '--' # push into loc loc[col].append(value) # Make sure we have some files if len(loc['FILENAME']) == 0: wmsg = 'No pre-processed (*{0}) files present.' WLOG(p, 'warning', wmsg.format(p['PROCESSED_SUFFIX'])) # ---------------------------------------------------------------------- # archive to table # ---------------------------------------------------------------------- if len(loc['FILENAME']) != 0: # construct table filename outfile = spirouConfig.Constants.OFF_LISTING_RAW_FILE(p) # log progress WLOG(p, '', 'Creating ascii file for listing.') # get column names colnames = ['FILENAME', 'LAST_MODIFIED'] + list(columns) # define the format for each column formats = [None] * len(colnames) # get the values for each column values = [] for col in colnames: values.append(loc[col]) # construct astropy table from column names, values and formats table = spirouImage.MakeTable(p, colnames, values, formats) # log saving of file wmsg = 'Listing of directory on file {0}' WLOG(p, '', wmsg.format(outfile)) # print out to screen WLOG('', '', '') WLOG('', '', 'Listing table:') WLOG('', '', '') spirouImage.PrintTable(table) # ---------------------------------------------------------------------- # Update Index # ---------------------------------------------------------------------- # ask whether to update index question = 'Update/Write index.fits? [Y]es or [N]o' cond = spirouStartup.spirouStartup.spirou_input_yes_no(p, question) # if cond is True can update if cond: # log writing index file wmsg = 'Writing index to file {0}' WLOG(p, '', wmsg.format(index_path)) # update index spirouStartup.SortSaveOutputs(p, loc, index_path) else: WLOG(p, 'warning', 'Skipped writing to index file') # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p, outputs=None) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, reffile=None): """ cal_DRIFT_E2DS_spirou.py main function, if arguments are None uses arguments from run time i.e.: cal_DRIFT_E2DS_spirou.py [night_directory] [reffile] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param reffile: string, the reference file to use :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with reference file being None (i.e. get from sys.argv) if reffile is None: customargs = spirouStartup.GetCustomFromRuntime( p, [0], [str], ['reffile']) else: customargs = dict(reffile=reffile) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='reffile', mainfitsdir='reduced') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, reffilename = spirouStartup.SingleFileSetup(p, filename=p['REFFILE']) p['REFFILENAME'] = reffilename p.set_source('REFFILENAME', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data speref, hdr, nbo, nx = spirouImage.ReadData(p, reffilename) # add to loc loc = ParamDict() loc['SPEREF'] = speref loc['NUMBER_ORDERS'] = nbo loc.set_sources(['speref', 'number_orders'], __NAME__ + '/main()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # get acquisition time p = spirouImage.GetAcqTime(p, hdr, name='acqtime', kind='julian') bjdref = p['ACQTIME'] # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # ---------------------------------------------------------------------- # Read wavelength solution # ---------------------------------------------------------------------- # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get wave image wout = spirouImage.GetWaveSolution(p, hdr=hdr, fiber=wave_fiber, return_wavemap=True, return_filename=True) _, loc['WAVE'], loc['WAVEFILE'], loc['WSOURCE'] = wout source = __NAME__ + '/main() + /spirouImage.GetWaveSolution' loc.set_sources(['WAVE', 'WAVEFILE', 'WSOURCE'], source) # ---------------------------------------------------------------------- # Read Flat file # ---------------------------------------------------------------------- # get flat p, loc['FLAT'] = spirouImage.ReadFlatFile(p, hdr) loc.set_source('FLAT', __NAME__ + '/main() + /spirouImage.ReadFlatFile') # get all values in flat that are zero to 1 loc['FLAT'] = np.where(loc['FLAT'] == 0, 1.0, loc['FLAT']) # ---------------------------------------------------------------------- # Background correction # ---------------------------------------------------------------------- # log that we are performing background correction if p['IC_DRIFT_BACK_CORR']: WLOG(p, '', 'Perform background correction') # get the box size from constants bsize = p['DRIFT_PEAK_MINMAX_BOXSIZE'] # Loop around the orders for order_num in range(loc['NUMBER_ORDERS']): miny, maxy = spirouBACK.MeasureMinMax(loc['SPEREF'][order_num], bsize) loc['SPEREF'][order_num] = loc['SPEREF'][order_num] - miny # ------------------------------------------------------------------ # Compute photon noise uncertainty for reference file # ------------------------------------------------------------------ # set up the arguments for DeltaVrms2D dargs = [loc['SPEREF'], loc['WAVE']] dkwargs = dict(sigdet=p['IC_DRIFT_NOISE'], size=p['IC_DRIFT_BOXSIZE'], threshold=p['IC_DRIFT_MAXFLUX']) # run DeltaVrms2D dvrmsref, wmeanref = spirouRV.DeltaVrms2D(*dargs, **dkwargs) # save to loc loc['DVRMSREF'], loc['WMEANREF'] = dvrmsref, wmeanref loc.set_sources(['dvrmsref', 'wmeanref'], __NAME__ + '/main()()') # log the estimated RV uncertainty wmsg = 'On fiber {0} estimated RV uncertainty on spectrum is {1:.3f} m/s' WLOG(p, 'info', wmsg.format(p['FIBER'], wmeanref)) # ------------------------------------------------------------------ # Reference plots # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot FP spectral order sPlt.drift_plot_selected_wave_ref(p, loc) # plot photon noise uncertainty sPlt.drift_plot_photon_uncertainty(p, loc) # ------------------------------------------------------------------ # Get all other files that match kw_OUTPUT and kw_EXT_TYPE from # ref file # ------------------------------------------------------------------ # get files listfiles, listtypes = spirouImage.GetSimilarDriftFiles(p, hdr) # get the number of files nfiles = len(listfiles) # Log the number of files found wmsgs = [ 'Number of files found on directory = {0}'.format(nfiles), '\tExtensions allowed:' ] for listtype in listtypes: wmsgs.append('\t\t - {0}'.format(listtype)) WLOG(p, 'info', wmsgs) # ------------------------------------------------------------------ # Set up Extract storage for all files # ------------------------------------------------------------------ # decide whether we need to skip (for large number of files) if len(listfiles) >= p['DRIFT_NLARGE']: skip = p['DRIFT_E2DS_FILE_SKIP'] nfiles = int(nfiles / skip) else: skip = 1 # set up storage loc['DRIFT'] = np.zeros((nfiles, loc['NUMBER_ORDERS'])) loc['ERRDRIFT'] = np.zeros((nfiles, loc['NUMBER_ORDERS'])) loc['DELTATIME'] = np.zeros(nfiles) # set loc sources keys = ['drift', 'errdrift', 'deltatime'] loc.set_sources(keys, __NAME__ + '/main()()') # ------------------------------------------------------------------ # Loop around all files: correct for dark, reshape, extract and # calculate dvrms and meanpond # ------------------------------------------------------------------ wref = 1 for i_it in range(nfiles): # get file for this iteration fpfile = listfiles[::skip][i_it] # Log the file we are reading wmsg = 'Reading file {0}' WLOG(p, '', wmsg.format(os.path.split(fpfile)[-1])) # ------------------------------------------------------------------ # read e2ds files and get timestamp # ------------------------------------------------------------------ # read data rout = spirouImage.ReadData(p, filename=fpfile, log=False) loc['SPE'], hdri, nxi, nyi = rout # get acqtime bjdspe = spirouImage.GetAcqTime(p, hdri, name='acqtime', return_value=1, kind='julian') # test whether we want to subtract background if p['IC_DRIFT_BACK_CORR']: # Loop around the orders for order_num in range(loc['NUMBER_ORDERS']): # get the box size from constants bsize = p['DRIFT_PEAK_MINMAX_BOXSIZE'] # Measurethe min and max flux miny, maxy = spirouBACK.MeasureMinMax(loc['SPE'][order_num], bsize) # subtract off the background (miny) loc['SPE'][order_num] = loc['SPE'][order_num] - miny # ------------------------------------------------------------------ # Compute photon noise uncertainty for iteration file # ------------------------------------------------------------------ # set up the arguments for DeltaVrms2D dargs = [loc['SPE'], loc['WAVE']] dkwargs = dict(sigdet=p['IC_DRIFT_NOISE'], size=p['IC_DRIFT_BOXSIZE'], threshold=p['IC_DRIFT_MAXFLUX']) # run DeltaVrms2D dvrmsspe, wmodespe = spirouRV.DeltaVrms2D(*dargs, **dkwargs) # ------------------------------------------------------------------ # Compute the correction of the cosmics and re-normalisation by # comparison with the reference spectrum # ------------------------------------------------------------------ # correction of the cosmics and renomalisation by comparison with # the reference spectrum dargs = [p, loc['SPEREF'], loc['SPE']] dkwargs = dict(threshold=p['IC_DRIFT_MAXFLUX'], size=p['IC_DRIFT_BOXSIZE'], cut=p['IC_DRIFT_CUT_E2DS']) spen, cfluxr, cpt = spirouRV.ReNormCosmic2D(*dargs, **dkwargs) # ------------------------------------------------------------------ # Calculate the RV drift # ------------------------------------------------------------------ dargs = [loc['SPEREF'], spen, loc['WAVE']] dkwargs = dict(sigdet=p['IC_DRIFT_NOISE'], threshold=p['IC_DRIFT_MAXFLUX'], size=p['IC_DRIFT_BOXSIZE']) rv = spirouRV.CalcRVdrift2D(*dargs, **dkwargs) # ------------------------------------------------------------------ # Calculate delta time # ------------------------------------------------------------------ # calculate the time from reference (in hours) deltatime = (bjdspe - bjdref) * 24 # ------------------------------------------------------------------ # Calculate RV properties # ------------------------------------------------------------------ # calculate the mean flux ratio meanfratio = np.nanmean(cfluxr) # calculate the weighted mean radial velocity wref = 1.0 / dvrmsref meanrv = -1.0 * np.nansum(rv * wref) / np.nansum(wref) err_meanrv = np.sqrt(dvrmsref + dvrmsspe) merr = 1. / np.sqrt(np.nansum((1. / err_meanrv)**2)) # Log the RV properties wmsg = ( 'Time from ref={0:.2f} h - Drift mean= {1:.2f} +- {2:.3f} m/s ' '- Flux ratio= {3:.3f} - Nb Comsic= {4}') WLOG(p, '', wmsg.format(deltatime, meanrv, merr, meanfratio, cpt)) # add this iteration to storage loc['DRIFT'][i_it] = -1.0 * rv loc['ERRDRIFT'][i_it] = err_meanrv loc['DELTATIME'][i_it] = deltatime # ------------------------------------------------------------------ # Calculate drift properties # ------------------------------------------------------------------ # get the maximum number of orders to use nomax = nbo # p['IC_DRIFT_N_ORDER_MAX'] # ------------------------------------------------------------------ # if use mean if p['DRIFT_TYPE_E2DS'].upper() == 'WEIGHTED MEAN': # mean radial velocity sumwref = np.nansum(wref[:nomax]) meanrv = np.nansum(loc['DRIFT'][:, :nomax] * wref[:nomax], 1) / sumwref # error in mean radial velocity errdrift2 = loc['ERRDRIFT'][:, :nomax]**2 meanerr = 1.0 / np.sqrt(np.nansum(1.0 / errdrift2, 1)) # add to loc loc['MDRIFT'] = meanrv loc['MERRDRIFT'] = meanerr # else use median else: # median drift loc['MDRIFT'] = np.nanmedian(loc['DRIFT'][:, :nomax], 1) # median err drift loc['MERRDRIFT'] = np.nanmedian(loc['ERRDRIFT'][:, :nomax], 1) # ------------------------------------------------------------------ # set source loc.set_sources(['mdrift', 'merrdrift'], __NAME__ + '/main()()') # ------------------------------------------------------------------ # peak to peak drift driftptp = np.max(loc['MDRIFT']) - np.min(loc['MDRIFT']) driftrms = np.std(loc['MDRIFT']) # log th etotal drift peak-to-peak and rms wmsg = ('Total drift Peak-to-Peak={0:.3f} m/s RMS={1:.3f} m/s in ' '{2:.2f} hour') wargs = [driftptp, driftrms, np.max(loc['DELTATIME'])] WLOG(p, '', wmsg.format(*wargs)) # ------------------------------------------------------------------ # Plot of mean drift # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot delta time against median drift sPlt.drift_plot_dtime_against_mdrift(p, loc, kind='e2ds') # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # TODO: Needs doing # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # add to qc header lists qc_values.append('None') qc_names.append('None') qc_logic.append('None') qc_pass.append(1) # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Save drift values to file # ------------------------------------------------------------------ # get raw input file name raw_infile = os.path.basename(p['REFFILE']) # construct filename driftfits, tag = spirouConfig.Constants.DRIFT_E2DS_FITS_FILE(p) driftfitsname = os.path.split(driftfits)[-1] # log that we are saving drift values wmsg = 'Saving drift values of Fiber {0} in {1}' WLOG(p, '', wmsg.format(p['FIBER'], driftfitsname)) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFLAT'], value=p['FLATFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_REFFILE'], value=raw_infile) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # save drift values p = spirouImage.WriteImage(p, driftfits, loc['DRIFT'], hdict) # ------------------------------------------------------------------ # print .tbl result # ------------------------------------------------------------------ # construct filename drifttbl = spirouConfig.Constants.DRIFT_E2DS_TBL_FILE(p) drifttblname = os.path.split(drifttbl)[-1] # construct and write table columnnames = ['time', 'drift', 'drifterr'] columnformats = ['7.4f', '6.2f', '6.3f'] columnvalues = [loc['DELTATIME'], loc['MDRIFT'], loc['MERRDRIFT']] table = spirouImage.MakeTable(p, columns=columnnames, values=columnvalues, formats=columnformats) # write table wmsg = 'Average Drift saved in {0} Saved ' WLOG(p, '', wmsg.format(drifttblname)) spirouImage.WriteTable(p, table, drifttbl, fmt='ascii.rst') # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, reffile=None): """ cal_DRIFT_E2DS_spirou.py main function, if arguments are None uses arguments from run time i.e.: cal_DRIFT_E2DS_spirou.py [night_directory] [reffile] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param reffile: string, the reference file to use :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with reference file being None (i.e. get from sys.argv) if reffile is None: customargs = spirouStartup.GetCustomFromRuntime( p, [0], [str], ['reffile']) else: customargs = dict(reffile=reffile) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='reffile', mainfitsdir='reduced') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, reffilename = spirouStartup.SingleFileSetup(p, filename=p['REFFILE']) p['REFFILENAME'] = reffilename p.set_source('REFFILENAME', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data speref, hdr, nbo, nx = spirouImage.ReadData(p, reffilename) # add to loc loc = ParamDict() loc['SPEREF'] = speref loc['NUMBER_ORDERS'] = nbo loc.set_sources(['speref', 'number_orders'], __NAME__ + '/main()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # get acquisition time p = spirouImage.GetAcqTime(p, hdr, name='acqtime', kind='julian') bjdref = p['ACQTIME'] # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # manually set OBJNAME to FP p['OBJNAME'] = 'FP' # ---------------------------------------------------------------------- # Earth Velocity calculation # ---------------------------------------------------------------------- if p['IC_IMAGE_TYPE'] == 'H4RG': p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, hdr) # ---------------------------------------------------------------------- # Read wavelength solution # ---------------------------------------------------------------------- # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # # get wave image # wout = spirouImage.GetWaveSolution(p, hdr=hdr, fiber=wave_fiber, # return_wavemap=True) # _, loc['WAVE'] = wout # loc.set_source('WAVE', __NAME__+'/main() + /spirouImage.GetWaveSolution') # get wave image wout = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True, return_filename=True, fiber=wave_fiber) param_ll, wave_ll, wavefile, wsource = wout # save to storage loc['PARAM_LL'], loc['WAVE_LL'], loc['WAVEFILE'], loc['WSOURCE'] = wout source = __NAME__ + '/main() + spirouTHORCA.GetWaveSolution()' loc.set_sources(['WAVE_LL', 'PARAM_LL', 'WAVEFILE', 'WSOURCE'], source) # ---------------------------------------------------------------------- # Read Flat file # ---------------------------------------------------------------------- # get flat p, loc['FLAT'] = spirouImage.ReadFlatFile(p, hdr) loc.set_source('FLAT', __NAME__ + '/main() + /spirouImage.ReadFlatFile') # get all values in flat that are zero to 1 loc['FLAT'] = np.where(loc['FLAT'] == 0, 1.0, loc['FLAT']) # ---------------------------------------------------------------------- # Background correction # ---------------------------------------------------------------------- # log that we are performing background correction if p['IC_DRIFT_BACK_CORR']: WLOG(p, '', 'Perform background correction') # get the box size from constants bsize = p['DRIFT_PEAK_MINMAX_BOXSIZE'] # Loop around the orders for order_num in range(loc['NUMBER_ORDERS']): miny, maxy = spirouBACK.MeasureMinMax(loc['SPEREF'][order_num], bsize) loc['SPEREF'][order_num] = loc['SPEREF'][order_num] - miny # ---------------------------------------------------------------------- # Preliminary set up = no flat, no blaze # ---------------------------------------------------------------------- # reset flat to all ones # loc['FLAT'] = np.ones((nbo, nx)) # set blaze to all ones (if not bug in correlbin !!! # TODO Check why Blaze makes bugs in correlbin loc['BLAZE'] = np.ones((nbo, nx)) # set sources # loc.set_sources(['flat', 'blaze'], __NAME__ + '/main()') loc.set_sources(['blaze'], __NAME__ + '/main()') # ------------------------------------------------------------------ # Compute photon noise uncertainty for reference file # ------------------------------------------------------------------ # set up the arguments for DeltaVrms2D dargs = [loc['SPEREF'], loc['WAVE_LL']] dkwargs = dict(sigdet=p['IC_DRIFT_NOISE'], size=p['IC_DRIFT_BOXSIZE'], threshold=p['IC_DRIFT_MAXFLUX']) # run DeltaVrms2D dvrmsref, wmeanref = spirouRV.DeltaVrms2D(*dargs, **dkwargs) # save to loc loc['DVRMSREF'], loc['WMEANREF'] = dvrmsref, wmeanref loc.set_sources(['dvrmsref', 'wmeanref'], __NAME__ + '/main()()') # log the estimated RV uncertainty wmsg = 'On fiber {0} estimated RV uncertainty on spectrum is {1:.3f} m/s' WLOG(p, 'info', wmsg.format(p['FIBER'], wmeanref)) # ------------------------------------------------------------------ # Reference plots # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot FP spectral order # sPlt.drift_plot_selected_wave_ref(p, loc) # plot photon noise uncertainty sPlt.drift_plot_photon_uncertainty(p, loc) # ---------------------------------------------------------------------- # Get template RV (from ccf_mask) # ---------------------------------------------------------------------- # Use CCF Mask function with drift constants p['CCF_MASK'] = p['DRIFT_CCF_MASK'] p['TARGET_RV'] = p['DRIFT_TARGET_RV'] p['CCF_WIDTH'] = p['DRIFT_CCF_WIDTH'] p['CCF_STEP'] = p['DRIFT_CCF_STEP'] # get the CCF mask from file (check location of mask) loc = spirouRV.GetCCFMask(p, loc) # check and deal with mask in microns (should be in nm) if np.mean(loc['LL_MASK_CTR']) < 2.0: loc['LL_MASK_CTR'] *= 1000.0 loc['LL_MASK_D'] *= 1000.0 # ---------------------------------------------------------------------- # Do correlation # ---------------------------------------------------------------------- # calculate and fit the CCF loc['E2DSFF'] = np.array(loc['SPEREF']) loc.set_source('E2DSFF', __NAME__ + '/main()') p['CCF_FIT_TYPE'] = 1 # run the RV coravelation function with these parameters loc = spirouRV.Coravelation(p, loc) # ---------------------------------------------------------------------- # Correlation stats # ---------------------------------------------------------------------- # get the maximum number of orders to use nbmax = p['CCF_NUM_ORDERS_MAX'] # get the average ccf loc['AVERAGE_CCF'] = np.nansum(loc['CCF'][:nbmax], axis=0) # normalize the average ccf normalized_ccf = loc['AVERAGE_CCF'] / np.nanmax(loc['AVERAGE_CCF']) # get the fit for the normalized average ccf ccf_res, ccf_fit = spirouRV.FitCCF(p, loc['RV_CCF'], normalized_ccf, fit_type=1) loc['CCF_RES'] = ccf_res loc['CCF_FIT'] = ccf_fit # get the max cpp loc['MAXCPP'] = np.nansum(loc['CCF_MAX']) / np.nansum( loc['PIX_PASSED_ALL']) # get the RV value from the normalised average ccf fit center location loc['RV'] = float(ccf_res[1]) # get the contrast (ccf fit amplitude) loc['CONTRAST'] = np.abs(100 * ccf_res[0]) # get the FWHM value loc['FWHM'] = ccf_res[2] * spirouCore.spirouMath.fwhm() # ---------------------------------------------------------------------- # set the source keys = [ 'average_ccf', 'maxcpp', 'rv', 'contrast', 'fwhm', 'ccf_res', 'ccf_fit' ] loc.set_sources(keys, __NAME__ + '/main()') # ---------------------------------------------------------------------- # log the stats wmsg = ('Correlation: C={0:.1f}[%] RV={1:.5f}[km/s] ' 'FWHM={2:.4f}[km/s] maxcpp={3:.1f}') wargs = [loc['CONTRAST'], loc['RV'], loc['FWHM'], loc['MAXCPP']] WLOG(p, 'info', wmsg.format(*wargs)) # get the reference RV in m/s rvref = loc['RV'] * 1000. # ---------------------------------------------------------------------- # rv ccf plot # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # Plot rv vs ccf (and rv vs ccf_fit) sPlt.ccf_rv_ccf_plot(p, loc['RV_CCF'], normalized_ccf, ccf_fit) # ------------------------------------------------------------------ # Get all other files that match kw_OUTPUT and kw_EXT_TYPE from # ref file # ------------------------------------------------------------------ # get files listfiles, listtypes = spirouImage.GetSimilarDriftFiles(p, hdr) # get the number of files nfiles = len(listfiles) # Log the number of files found wmsgs = [ 'Number of files found on directory = {0}'.format(nfiles), '\tExtensions allowed:' ] for listtype in listtypes: wmsgs.append('\t\t - {0}'.format(listtype)) WLOG(p, 'info', wmsgs) # ------------------------------------------------------------------ # Set up Extract storage for all files # ------------------------------------------------------------------ # decide whether we need to skip (for large number of files) if len(listfiles) >= p['DRIFT_NLARGE']: skip = p['DRIFT_E2DS_FILE_SKIP'] nfiles = int(nfiles / skip) else: skip = 1 # set up storage loc['MDRIFT'] = np.zeros(nfiles) loc['MERRDRIFT'] = np.zeros(nfiles) loc['DELTATIME'] = np.zeros(nfiles) loc['FLUXRATIO'] = np.zeros(nfiles) # set loc sources keys = ['mdrift', 'merrdrift', 'deltatime'] loc.set_sources(keys, __NAME__ + '/main()()') # ------------------------------------------------------------------ # Loop around all files: correct for dark, reshape, extract and # calculate dvrms and meanpond # ------------------------------------------------------------------ wref = 1 for i_it in range(nfiles): # get file for this iteration fpfile = listfiles[::skip][i_it] # Log the file we are reading wmsg = 'Reading file {0}' WLOG(p, '', wmsg.format(os.path.split(fpfile)[-1])) # ------------------------------------------------------------------ # read e2ds files and get timestamp # ------------------------------------------------------------------ # read data rout = spirouImage.ReadData(p, filename=fpfile, log=False) loc['SPE'], hdri, nxi, nyi = rout # get acqtime bjdspe = spirouImage.GetAcqTime(p, hdri, name='acqtime', return_value=1, kind='julian') # test whether we want to subtract background if p['IC_DRIFT_BACK_CORR']: # Loop around the orders for order_num in range(loc['NUMBER_ORDERS']): # get the box size from constants bsize = p['DRIFT_PEAK_MINMAX_BOXSIZE'] # Measurethe min and max flux miny, maxy = spirouBACK.MeasureMinMax(loc['SPE'][order_num], bsize) # subtract off the background (miny) loc['SPE'][order_num] = loc['SPE'][order_num] - miny # ------------------------------------------------------------------ # calculate flux ratio # ------------------------------------------------------------------ sorder = p['IC_DRIFT_ORDER_PLOT'] fratio = np.nansum(loc['SPE'][sorder]) / np.nansum( loc['SPEREF'][sorder]) loc['FLUXRATIO'][i_it] = fratio # ------------------------------------------------------------------ # Compute photon noise uncertainty for reference file # ------------------------------------------------------------------ # set up the arguments for DeltaVrms2D dargs = [loc['SPE'], loc['WAVE_LL']] dkwargs = dict(sigdet=p['IC_DRIFT_NOISE'], size=p['IC_DRIFT_BOXSIZE'], threshold=p['IC_DRIFT_MAXFLUX']) # run DeltaVrms2D dvrmsspe, wmeanspe = spirouRV.DeltaVrms2D(*dargs, **dkwargs) # ---------------------------------------------------------------------- # Do correlation # ---------------------------------------------------------------------- # calculate and fit the CCF loc['E2DSFF'] = loc['SPE'] * 1. loc.set_source('E2DSFF', __NAME__ + '/main()') loc = spirouRV.Coravelation(p, loc) # ---------------------------------------------------------------------- # Correlation stats # ---------------------------------------------------------------------- # get the maximum number of orders to use nbmax = p['CCF_NUM_ORDERS_MAX'] # get the average ccf loc['AVERAGE_CCF'] = np.nansum(loc['CCF'][:nbmax], axis=0) # normalize the average ccf normalized_ccf = loc['AVERAGE_CCF'] / np.nanmax(loc['AVERAGE_CCF']) # get the fit for the normalized average ccf ccf_res, ccf_fit = spirouRV.FitCCF(p, loc['RV_CCF'], normalized_ccf, fit_type=1) # calculate the mean RV meanrv = ccf_res[1] * 1000. - rvref # ------------------------------------------------------------------ # Calculate delta time # ------------------------------------------------------------------ # calculate the time from reference (in hours) deltatime = (bjdspe - bjdref) * 24 err_meanrv = np.sqrt(dvrmsref + dvrmsspe) merr = 1. / np.sqrt(np.nansum((1. / err_meanrv)**2)) # Log the RV properties wmsg = ('Time from ref= {0:.2f} h ' '- Flux Ratio= {1:.2f} ' '- Drift mean= {2:.2f} +- ' '{3:.2f} m/s') wargs = [deltatime, loc['FLUXRATIO'][i_it], meanrv, merr] WLOG(p, '', wmsg.format(*wargs)) # add this iteration to storage loc['MDRIFT'][i_it] = meanrv loc['MERRDRIFT'][i_it] = merr loc['DELTATIME'][i_it] = deltatime # ------------------------------------------------------------------ # set source loc.set_sources(['mdrift', 'merrdrift'], __NAME__ + '/main()()') # ------------------------------------------------------------------ # peak to peak drift driftptp = np.max(loc['MDRIFT']) - np.min(loc['MDRIFT']) driftrms = np.std(loc['MDRIFT']) # log th etotal drift peak-to-peak and rms wmsg = ('Total drift Peak-to-Peak={0:.3f} m/s RMS={1:.3f} m/s in ' '{2:.2f} hour') wargs = [driftptp, driftrms, np.max(loc['DELTATIME'])] WLOG(p, '', wmsg.format(*wargs)) # ------------------------------------------------------------------ # Plot of mean drift # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot delta time against median drift sPlt.drift_plot_dtime_against_mdrift(p, loc, kind='e2ds') # ------------------------------------------------------------------ # Save drift values to file # ------------------------------------------------------------------ # # get raw input file name # raw_infile = os.path.basename(p['REFFILE']) # # construct filename # driftfits, tag = spirouConfig.Constants.DRIFTCCF_E2DS_FITS_FILE(p) # driftfitsname = os.path.split(driftfits)[-1] # # log that we are saving drift values # wmsg = 'Saving drift values of Fiber {0} in {1}' # WLOG(p, '', wmsg.format(p['FIBER'], driftfitsname)) # # add keys from original header file # hdict = spirouImage.CopyOriginalKeys(hdr) # # add the reference RV # hdict = spirouImage.AddKey(p, hdict, p['KW_REF_RV'], value=rvref) # # # set the version # hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) # hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # # set the input files # hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFLAT'], value=p['FLATFILE']) # hdict = spirouImage.AddKey(p, hdict, p['KW_REFFILE'], value=raw_infile) # # save drift values # p = spirouImage.WriteImage(p, driftfits, loc['DRIFT'], hdict) # ------------------------------------------------------------------ # print .tbl result # ------------------------------------------------------------------ # construct filename drifttbl = spirouConfig.Constants.DRIFTCCF_E2DS_TBL_FILE(p) drifttblname = os.path.split(drifttbl)[-1] # construct and write table columnnames = ['time', 'drift', 'drifterr'] columnformats = ['7.4f', '6.2f', '6.3f'] columnvalues = [loc['DELTATIME'], loc['MDRIFT'], loc['MERRDRIFT']] table = spirouImage.MakeTable(p, columns=columnnames, values=columnvalues, formats=columnformats) # write table wmsg = 'Average Drift saved in {0} Saved ' WLOG(p, '', wmsg.format(drifttblname)) spirouImage.WriteTable(p, table, drifttbl, fmt='ascii.rst') # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(cores=1, objects=None, filetype='EXT_E2DS_FF_AB'): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # set up function name main_name = __NAME__ + '.main()' # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # now get custom arguments pos, fmt = [0, 1, 2], [int, str, str] names = ['cores', 'objects', 'filetype'] call = [cores, objects, filetype] required = [False, False, False] customargs = spirouStartup.GetCustomFromRuntime(p, pos, fmt, names, calls=call, required=required, require_night_name=False) p = spirouStartup.LoadArguments(p, None, customargs=customargs, mainfitsdir='reduced', require_night_name=False) # keep track of errors errors = [] # ------------------------------------------------------------------------- # find all objects (matching 'objects' if not None) target_list = spirouTelluric.FindObjects(p) # print number found nfound = len(target_list) nstar = 0 for target in list(target_list.keys()): nstar += len(target_list[target]) wmsg = 'Found target "{0}" ({1} observations total)' WLOG(p, 'info', wmsg.format(target, len(target_list[target]))) # list total found wmsg = 'Found {0} observations total for {1} object(s)' WLOG(p, 'info', wmsg.format(nstar, nfound)) # ------------------------------------------------------------------------- # Step 1: Run fit_tellu on all objects # ------------------------------------------------------------------------- # for t_it, target in enumerate(list(target_list.keys())): # # loop around object filenames # for o_it, objfilename in enumerate(target_list[target]): # # Log progress # pargs = [p, 'Fit Tellurics', target, t_it, nfound, # o_it, len(target_list[target])] # spirouTelluric.UpdateProcessDB(*pargs) # # get arguments from filename # args = spirouTelluric.GetDBarguments(p, objfilename) # # run obj_mk_tellu # try: # obj_fit_tellu.main(**args) # except SystemExit as e: # errors.append([pargs[1], objfilename, e]) # # force close all plots # sPlt.closeall() # ------------------------------------------------------------------------- # Step 2: Run mk_obj_template on each science target # ------------------------------------------------------------------------- for t_it, target in enumerate(list(target_list.keys())): # log progress (big) pmsg = 'Make Telluric Template: Processing object = {0} ({1}/{2}' wmsgs = [ '', '=' * 60, '', pmsg.format(target, t_it + 1, nfound), '', '=' * 60, '' ] WLOG(p, 'info', wmsgs, wrap=False) # get last object objfilename = target_list[target][-1] # get arguments from filename args = spirouTelluric.GetDBarguments(p, objfilename) # run obj_mk_obj_template try: obj_mk_obj_template.main(**args) except SystemExit as e: errors.append(['Telluric Template', target, e]) # force close all plots sPlt.closeall() # ------------------------------------------------------------------------- # Step 3: Re-Run fit_tellu on all objects # ------------------------------------------------------------------------- for t_it, target in enumerate(list(target_list.keys())): # loop around object filenames for o_it, objfilename in enumerate(target_list[target]): # Log progress pargs = [ p, 'Fit Tellurics II', target, t_it, nfound, o_it, len(target_list[target]) ] spirouTelluric.UpdateProcessDB(*pargs) # get arguments from filename args = spirouTelluric.GetDBarguments(p, objfilename) # run obj_mk_tellu try: obj_fit_tellu.main(**args) except SystemExit as e: errors.append([pargs[1], objfilename, e]) # force close all plots sPlt.closeall() # ------------------------------------------------------------------------- # Step 4: Re-Run mk_obj_template on each science target # ------------------------------------------------------------------------- for t_it, target in enumerate(list(target_list.keys())): # log progress (big) pmsg = 'Make Telluric Template II: Processing object = {0} ({1}/{2}' wmsgs = [ '', '=' * 60, '', pmsg.format(target, t_it + 1, nfound), '', '=' * 60, '' ] WLOG(p, 'info', wmsgs, wrap=False) # get last object objfilename = target_list[target][-1] # get arguments from filename args = spirouTelluric.GetDBarguments(p, objfilename) # run obj_mk_obj_template try: obj_mk_obj_template.main(**args) except SystemExit as e: errors.append(['Telluric Template', target, e]) # force close all plots sPlt.closeall() # ---------------------------------------------------------------------- # Print all errors # ---------------------------------------------------------------------- if len(errors) > 0: emsgs = ['', '=' * 50, 'Errors were as follows: '] # loop around errors for error in errors: emsgs.append('') emsgs.append('{0}: Object = {1}'.format(error[0], error[1])) emsgs.append('\t{0}'.format(error[2])) emsgs.append('') WLOG(p, 'error', emsgs) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(cores=1, filetype='EXT_E2DS_FF_AB'): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # set up function name main_name = __NAME__ + '.main()' # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # now get custom arguments pos, fmt = [0, 1], [int, str] names, call = ['cores', 'filetype'], [cores, filetype] customargs = spirouStartup.GetCustomFromRuntime(p, pos, fmt, names, calls=call, require_night_name=False) p = spirouStartup.LoadArguments(p, None, customargs=customargs, mainfitsdir='reduced', require_night_name=False) # ------------------------------------------------------------------------- # find all telluric stars telluric_stars = spirouTelluric.FindTelluricStars(p) # print number found nfound = len(telluric_stars) nstar = 0 for tell_star in list(telluric_stars.keys()): nstar += len(telluric_stars[tell_star]) wmsg = 'Found {0} Telluric stars ({1} observations total)' WLOG(p, 'info', wmsg.format(nfound, nstar)) # # ------------------------------------------------------------------------- # # Step 0: Reset telluric database # # ------------------------------------------------------------------------- # #reset = spirouTools.drs_reset.reset_confirmation(p, 'TelluDB') # reset = True # if reset: # spirouTools.drs_reset.reset_telludb(p, False) # keep track of errors errors = [] # ------------------------------------------------------------------------- # Step 1: Run mk_tellu on all telluric stars # ------------------------------------------------------------------------- for t_it, tell_star in enumerate(list(telluric_stars.keys())): # loop around object filenames for o_it, objfilename in enumerate(telluric_stars[tell_star]): # Log progress pargs = [ p, 'Make Tellurics I', tell_star, t_it, nfound, o_it, len(telluric_stars[tell_star]) ] spirouTelluric.UpdateProcessDB(*pargs) # get arguments from filename args = spirouTelluric.GetDBarguments(p, objfilename) # run obj_mk_tellu try: ll = obj_mk_tellu.main(**args) except SystemExit as e: errors.append([pargs[1], objfilename, e]) # force close all plots sPlt.closeall() # ------------------------------------------------------------------------- # Step 2: Run fit tellu on all telluric stars # ------------------------------------------------------------------------- for t_it, tell_star in enumerate(list(telluric_stars.keys())): # loop around object filenames for o_it, objfilename in enumerate(telluric_stars[tell_star]): # Log progress pargs = [ p, 'Fit Tellurics', tell_star, t_it, nfound, o_it, len(telluric_stars[tell_star]) ] spirouTelluric.UpdateProcessDB(*pargs) # get arguments from filename args = spirouTelluric.GetDBarguments(p, objfilename) # run obj_mk_tellu try: obj_fit_tellu.main(**args) except SystemExit as e: errors.append([pargs[1], objfilename, e]) # force close all plots sPlt.closeall() # ------------------------------------------------------------------------- # step 3: Run mk_obj_template on each telluric star obj name # ------------------------------------------------------------------------- for t_it, tell_star in enumerate(list(telluric_stars.keys())): # log progress (big) pmsg = 'Make Telluric Template: Processing object = {0} ({1}/{2}' wmsgs = [ '', '=' * 60, '', pmsg.format(tell_star, t_it + 1, nfound), '', '=' * 60, '' ] WLOG(p, 'info', wmsgs, wrap=False) # get last object objfilename = telluric_stars[tell_star][-1] # get arguments from filename args = spirouTelluric.GetDBarguments(p, objfilename) # run obj_mk_obj_template try: obj_mk_obj_template.main(**args) except SystemExit as e: errors.append(['Telluric Template', tell_star, e]) # force close all plots sPlt.closeall() # ------------------------------------------------------------------------- # step 4: Run mk_tellu on all telluric stars # ------------------------------------------------------------------------- for t_it, tell_star in enumerate(list(telluric_stars.keys())): # loop around object filenames for o_it, objfilename in enumerate(telluric_stars[tell_star]): # Log progress pargs = [ p, 'Make Tellurics II', tell_star, t_it, nfound, o_it, len(telluric_stars[tell_star]) ] spirouTelluric.UpdateProcessDB(*pargs) # get arguments from filename args = spirouTelluric.GetDBarguments(p, objfilename) # run obj_mk_tellu try: obj_mk_tellu.main(**args) except SystemExit as e: errors.append([pargs[1], objfilename, e]) # force close all plots sPlt.closeall() # ---------------------------------------------------------------------- # Print all errors # ---------------------------------------------------------------------- if len(errors) > 0: emsgs = ['', '=' * 50, 'Errors were as follows: '] # loop around errors for error in errors: emsgs.append('') emsgs.append('{0}: Object = {1}'.format(error[0], error[1])) emsgs.append('\t{0}'.format(error[2])) emsgs.append('') WLOG(p, 'error', emsgs) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, fpfile=None, hcfiles=None): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) if hcfiles is None or fpfile is None: names, types = ['fpfile', 'hcfiles'], [str, str] customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1], types, names, last_multi=True) else: customargs = dict(hcfiles=hcfiles, fpfile=fpfile) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsdir='reduced', mainfitsfile='hcfiles') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, fpfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['FPFILE']) fiber1 = str(p['FIBER']) p, hcfilenames = spirouStartup.MultiFileSetup(p, files=p['HCFILES']) fiber2 = str(p['FIBER']) # set the hcfilename to the first hcfilenames hcfitsfilename = hcfilenames[0] # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Have to check that the fibers match # ---------------------------------------------------------------------- if fiber1 == fiber2: p['FIBER'] = fiber1 fsource = __NAME__ + '/main() & spirouStartup.GetFiberType()' p.set_source('FIBER', fsource) else: emsg = 'Fiber not matching for {0} and {1}, should be the same' eargs = [hcfitsfilename, fpfitsfilename] WLOG(p, 'error', emsg.format(*eargs)) # set the fiber type p['FIB_TYP'] = [p['FIBER']] p.set_source('FIB_TYP', __NAME__ + '/main()') # set find line mode find_lines_mode = p['HC_FIND_LINES_MODE'] # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read and combine all HC files except the first (fpfitsfilename) rargs = [p, 'add', hcfitsfilename, hcfilenames[1:]] p, hcdata, hchdr = spirouImage.ReadImageAndCombine(*rargs) # read first file (fpfitsfilename) fpdata, fphdr, _, _ = spirouImage.ReadImage(p, fpfitsfilename) # add data and hdr to loc loc = ParamDict() loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr # set the source sources = ['HCDATA', 'HCHDR'] loc.set_sources(sources, 'spirouImage.ReadImageAndCombine()') sources = ['FPDATA', 'FPHDR'] loc.set_sources(sources, 'spirouImage.ReadImage()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hchdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hchdr, name='exptime') # get gain p = spirouImage.GetGain(p, hchdr, name='gain') # get acquisition time p = spirouImage.GetAcqTime(p, hchdr, name='acqtime', kind='julian') bjdref = p['ACQTIME'] # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # get lamp parameters p = spirouTHORCA.GetLampParams(p, hchdr) # ---------------------------------------------------------------------- # Obtain the flat # ---------------------------------------------------------------------- # get the flat p, loc = spirouFLAT.GetFlat(p, loc, hchdr) # ---------------------------------------------------------------------- # Read blaze # ---------------------------------------------------------------------- # get tilts p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hchdr) loc.set_source('BLAZE', __NAME__ + '/main() + /spirouImage.ReadBlazeFile') # correct the data with the flat # TODO: Should this be used? # log # WLOG(p, '', 'Applying flat correction') # loc['HCDATA'] = loc['HCDATA']/loc['FLAT'] # loc['FPDATA'] = loc['FPDATA']/loc['FLAT'] # ---------------------------------------------------------------------- # Start plotting session # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # start interactive plot sPlt.start_interactive_session(p) # ---------------------------------------------------------------------- # loop around fiber type # ---------------------------------------------------------------------- for fiber in p['FIB_TYP']: # set fiber type for inside loop p['FIBER'] = fiber # ------------------------------------------------------------------ # Instrumental drift computation (if previous solution exists) # ------------------------------------------------------------------ # get key keydb = 'HCREF_{0}'.format(p['FIBER']) # check for key in calibDB if keydb in p['CALIBDB'].keys(): # log process wmsg = ('Doing Instrumental drift computation from previous ' 'calibration') WLOG(p, '', wmsg) # calculate instrument drift loc = spirouTHORCA.CalcInstrumentDrift(p, loc) # ------------------------------------------------------------------ # Wave solution # ------------------------------------------------------------------ # log message for loop wmsg = 'Processing Wavelength Calibration for Fiber {0}' WLOG(p, 'info', wmsg.format(p['FIBER'])) # ------------------------------------------------------------------ # Part 1 of cal_HC # ------------------------------------------------------------------ p, loc = cal_HC_E2DS_spirou.part1(p, loc, mode=find_lines_mode) # ------------------------------------------------------------------ # FP solution # ------------------------------------------------------------------ # log message wmsg = 'Calculating FP wave solution' WLOG(p, '', wmsg) # calculate FP wave solution # spirouTHORCA.FPWaveSolution(p, loc, mode=find_lines_mode) spirouTHORCA.FPWaveSolutionNew(p, loc) # ------------------------------------------------------------------ # FP solution plots # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # Plot the FP extracted spectrum against wavelength solution sPlt.wave_plot_final_fp_order(p, loc, iteration=1) # Plot the measured FP cavity width offset against line number sPlt.wave_local_width_offset_plot(p, loc) # Plot the FP line wavelength residuals sPlt.wave_fp_wavelength_residuals(p, loc) # ------------------------------------------------------------------ # Part 2 of cal_HC # ------------------------------------------------------------------ # set params for part2 p['QC_RMS_LITTROW_MAX'] = p['QC_WAVE_RMS_LITTROW_MAX'] p['QC_DEV_LITTROW_MAX'] = p['QC_WAVE_DEV_LITTROW_MAX'] p['IC_HC_N_ORD_START_2'] = min(p['IC_HC_N_ORD_START_2'], p['IC_FP_N_ORD_START']) p['IC_HC_N_ORD_FINAL_2'] = max(p['IC_HC_N_ORD_FINAL_2'], p['IC_FP_N_ORD_FINAL']) # run part 2 # p, loc = part2test(p, loc) p, loc = cal_HC_E2DS_spirou.part2(p, loc) # ---------------------------------------------------------------------- # End plotting session # ---------------------------------------------------------------------- # end interactive session sPlt.end_interactive_session(p) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, e2dsfiles=None): """ cal_CCF_E2DS_spirou.py main function, if arguments are None uses arguments from run time i.e.: cal_CCF_E2DS_spirou.py [night_directory] [E2DSfilename] [mask] [RV] [width] [step] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param e2dsfiles: list of string, the E2DS files to use :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # need custom args (to accept full path or wild card if e2dsfiles is None: names, types = ['e2dsfiles'], [str] customargs = spirouStartup.GetCustomFromRuntime(p, [0], types, names, last_multi=True) else: customargs = dict(e2dsfiles=e2dsfiles) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsdir='reduced') # ---------------------------------------------------------------------- # Process files (including wildcards) # ---------------------------------------------------------------------- try: e2dsfiles = spirouFile.Paths(p['E2DSFILES'], root=p['ARG_FILE_DIR']).abs_paths except PathException as e: WLOG(p, 'error', e) # loop around files for it, e2dsfile in enumerate(e2dsfiles): # get the base file name e2dsfilename = os.path.basename(e2dsfile) # log the file process wargs = [e2dsfilename, it + 1, len(e2dsfiles)] wmsg = ' * Processing file {0} ({1} of {2})'.format(*wargs) WLOG(p, '', spirouStartup.spirouStartup.HEADER) WLOG(p, '', wmsg) WLOG(p, '', spirouStartup.spirouStartup.HEADER) # ------------------------------------------------------------------ # Check that we can process file # ------------------------------------------------------------------ # check if ufile exists if not os.path.exists(e2dsfile): wmsg = 'File {0} does not exist... skipping' WLOG(p, 'warning', wmsg.format(e2dsfilename)) continue elif ('e2ds' not in e2dsfilename) and ('e2dsff' not in e2dsfilename): wmsg = 'File {0} not a valid E2DS or E2DSFF file' WLOG(p, 'warning', wmsg.format(e2dsfilename)) continue elif '.fits' not in e2dsfilename: wmsg = 'File {0} not a fits file... skipping' WLOG(p, 'warning', wmsg.format(e2dsfilename)) continue # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data e2ds, hdr, nbo, nx = spirouImage.ReadData(p, e2dsfile) # add to loc loc = ParamDict() loc['E2DS'] = e2ds loc['NUMBER_ORDERS'] = nbo loc.set_sources(['E2DS', 'number_orders'], __NAME__ + '/main()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # get acquisition time p = spirouImage.GetAcqTime(p, hdr, name='acqtime', kind='julian') # ---------------------------------------------------------------------- # Read star parameters # ---------------------------------------------------------------------- p = spirouImage.ReadParam(p, hdr, 'KW_OBJRA', dtype=str) p = spirouImage.ReadParam(p, hdr, 'KW_OBJDEC', dtype=str) p = spirouImage.ReadParam(p, hdr, 'KW_OBJEQUIN') p = spirouImage.ReadParam(p, hdr, 'KW_OBJRAPM') p = spirouImage.ReadParam(p, hdr, 'KW_OBJDECPM') p = spirouImage.ReadParam(p, hdr, 'KW_DATE_OBS', dtype=str) p = spirouImage.ReadParam(p, hdr, 'KW_UTC_OBS', dtype=str) # ----------------------------------------------------------------------- # Earth Velocity calculation # ----------------------------------------------------------------------- if p['IC_IMAGE_TYPE'] == 'H4RG': loc = spirouImage.EarthVelocityCorrection(p, loc, method=p['CCF_BERVMODE']) else: loc['BERV'], loc['BJD'] = 0.0, 0.0 loc['BERV_MAX'], loc['BERV_SOURCE'] = 0.0, 'None' loc.set_sources(['BERV', 'BJD', 'BERV_MAX'], __NAME__ + '.main()') # ---------------------------------------------------------------------- # archive ccf to fits file # ---------------------------------------------------------------------- outfilename = str(e2dsfile) # add keys hdict = spirouImage.CopyOriginalKeys(hdr) # add berv values hdict = spirouImage.AddKey(p, hdict, p['KW_BERV'], value=loc['BERV']) hdict = spirouImage.AddKey(p, hdict, p['KW_BJD'], value=loc['BJD']) hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_MAX'], value=loc['BERV_MAX']) hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_SOURCE'], value=loc['BERV_SOURCE']) # write image and add header keys (via hdict) p = spirouImage.WriteImage(p, outfilename, e2ds, hdict) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, reffile=None): """ cal_DRIFTPEAK_E2DS_spirou.py main function, if arguments are None uses arguments from run time i.e.: cal_DRIFTPEAK_E2DS_spirou.py [night_directory] [reffile] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param reffile: string, the reference file to use :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with reference file being None (i.e. get from sys.argv) if reffile is None: customargs = spirouStartup.GetCustomFromRuntime( p, [0], [str], ['reffile']) else: customargs = dict(reffile=reffile) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='reffile', mainfitsdir='reduced') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, reffilename = spirouStartup.SingleFileSetup(p, filename=p['REFFILE']) p['REFFILENAME'] = reffilename p.set_source('REFFILENAME', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data speref, hdr, nbo, nx = spirouImage.ReadData(p, p['REFFILENAME']) # add to loc loc = ParamDict() loc['SPEREF'] = speref loc['NUMBER_ORDERS'] = nbo loc.set_sources(['SPEREF', 'NUMBER_ORDERS'], __NAME__ + '/main()') # ---------------------------------------------------------------------- # Get lamp type # ---------------------------------------------------------------------- # get lamp type if p['KW_EXT_TYPE'][0] in hdr: ext_type = hdr[p['KW_EXT_TYPE'][0]] drift_types = p['DRIFT_PEAK_ALLOWED_TYPES'].keys() found = False for kind in drift_types: if ext_type == kind: loc['LAMP'] = p['DRIFT_PEAK_ALLOWED_TYPES'][kind] found = True if not found: eargs1 = [p['KW_EXT_TYPE'][0], ' or '.join(drift_types)] emsg1 = ( 'Wrong type of image for Drift, header key "{0}" should be' '{1}'.format(*eargs1)) emsg2 = '\tPlease check DRIFT_PEAK_ALLOWED_TYPES' WLOG(p, 'error', [emsg1, emsg2]) else: emsg = 'Header key = "{0}" missing from file {1}' eargs = [p['KW_EXT_TYPE'][0], p['REFFILENAME']] WLOG(p, 'error', emsg.format(*eargs)) loc.set_source('LAMP', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # get acquisition time p = spirouImage.GetAcqTime(p, hdr, name='acqtime', kind='julian') bjdref = p['ACQTIME'] # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # ---------------------------------------------------------------------- # Read wavelength solution # ---------------------------------------------------------------------- # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get wave image source = __NAME__ + '/main() + /spirouImage.GetWaveSolution' wout = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True, return_filename=True, fiber=wave_fiber) _, loc['WAVE'], loc['WAVEFILE'], loc['WSOURCE'] = wout loc.set_sources(['WAVE', 'WAVEFILE', 'WSOURCE'], source) # ---------------------------------------------------------------------- # Read Flat file # ---------------------------------------------------------------------- # get flat p, loc['FLAT'] = spirouImage.ReadFlatFile(p, hdr) loc.set_source('FLAT', __NAME__ + '/main() + /spirouImage.ReadFlatFile') # get all values in flat that are zero to 1 loc['FLAT'] = np.where(loc['FLAT'] == 0, 1.0, loc['FLAT']) # correct for flat file loc['SPEREF'] = loc['SPEREF'] / loc['FLAT'] # ---------------------------------------------------------------------- # Background correction # ---------------------------------------------------------------------- # test whether we want to subtract background if p['IC_DRIFT_BACK_CORR']: # Loop around the orders for order_num in range(loc['NUMBER_ORDERS']): # get the box size from constants bsize = p['DRIFT_PEAK_MINMAX_BOXSIZE'] # Measurethe min and max flux miny, maxy = spirouBACK.MeasureMinMax(loc['SPEREF'][order_num], bsize) # subtract off the background (miny) loc['SPEREF'][order_num] = loc['SPEREF'][order_num] - miny # ---------------------------------------------------------------------- # Identify FP peaks in reference file # ---------------------------------------------------------------------- # log that we are identifying peaks wmsg = 'Identification of lines in reference file: {0}' WLOG(p, '', wmsg.format(p['REFFILE'])) # get the position of FP peaks from reference file loc = spirouRV.CreateDriftFile(p, loc) # ---------------------------------------------------------------------- # Removal of suspiciously wide FP lines # ---------------------------------------------------------------------- loc = spirouRV.RemoveWidePeaks(p, loc) # ---------------------------------------------------------------------- # Get reference drift # ---------------------------------------------------------------------- # are we using gaussfit? gaussfit = p['DRIFT_PEAK_GETDRIFT_GAUSSFIT'] # get drift loc['XREF'] = spirouRV.GetDrift(p, loc['SPEREF'], loc['ORDPEAK'], loc['XPEAK'], gaussfit=gaussfit) loc.set_source('XREF', __NAME__ + '/main()') # remove any drifts that are zero (i.e. peak not measured loc = spirouRV.RemoveZeroPeaks(p, loc) # ------------------------------------------------------------------ # Reference plots # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot FP spectral order sPlt.drift_plot_selected_wave_ref(p, loc) # ------------------------------------------------------------------ # Get all other files that match kw_OUTPUT and kw_EXT_TYPE from # ref file # ------------------------------------------------------------------ # get files listfiles, listtypes = spirouImage.GetSimilarDriftFiles(p, hdr) # get the number of files nfiles = len(listfiles) # Log the number of files found wmsgs = [ 'Number of files found on directory = {0}'.format(nfiles), '\tExtensions allowed:' ] for listtype in listtypes: wmsgs.append('\t\t - {0}'.format(listtype)) WLOG(p, 'info', wmsgs) # ------------------------------------------------------------------ # Set up Extract storage for all files # ------------------------------------------------------------------ # decide whether we need to skip (for large number of files) if len(listfiles) >= p['DRIFT_NLARGE']: skip = p['DRIFT_PEAK_FILE_SKIP'] nfiles = int(nfiles / skip) else: skip = 1 # set up storage loc['DRIFT'] = np.zeros((nfiles, loc['NUMBER_ORDERS'])) loc['DRIFT_LEFT'] = np.zeros((nfiles, loc['NUMBER_ORDERS'])) loc['DRIFT_RIGHT'] = np.zeros((nfiles, loc['NUMBER_ORDERS'])) loc['ERRDRIFT'] = np.zeros((nfiles, loc['NUMBER_ORDERS'])) loc['DELTATIME'] = np.zeros(nfiles) loc['MEANRV'] = np.zeros(nfiles) loc['MEANRV_LEFT'] = np.zeros(nfiles) loc['MEANRV_RIGHT'] = np.zeros(nfiles) loc['MERRDRIFT'] = np.zeros(nfiles) loc['FLUXRATIO'] = np.zeros(nfiles) # add sources source = __NAME__ + '/main()' keys = [ 'drift', 'drift_left', 'drift_right', 'errdrift', 'deltatime', 'meanrv', 'meanrv_left', 'meanrv_right', 'merrdrift', 'fluxratio' ] loc.set_sources(keys, source) # ------------------------------------------------------------------ # Loop around all files: correct for dark, reshape, extract and # calculate dvrms and meanpond # ------------------------------------------------------------------ # get the maximum number of orders to use nomin = p['IC_DRIFT_PEAK_N_ORDER_MIN'] nomax = p['IC_DRIFT_PEAK_N_ORDER_MAX'] # loop around files for i_it in range(nfiles): # get file for this iteration fpfile = listfiles[::skip][i_it] # Log the file we are reading wmsg = 'Reading file {0}' WLOG(p, '', wmsg.format(os.path.split(fpfile)[-1])) # ------------------------------------------------------------------ # read e2ds files and get timestamp # ------------------------------------------------------------------ # read data rout = spirouImage.ReadData(p, filename=fpfile, log=False) loc['SPE'], hdri, nxi, nyi = rout # apply flat loc['SPE'] = loc['SPE'] / loc['FLAT'] # get acqtime bjdspe = spirouImage.GetAcqTime(p, hdri, name='acqtime', return_value=1, kind='julian') # ---------------------------------------------------------------------- # Background correction # ---------------------------------------------------------------------- # test whether we want to subtract background if p['IC_DRIFT_BACK_CORR']: # Loop around the orders for order_num in range(loc['NUMBER_ORDERS']): # get the box size from constants bsize = p['DRIFT_PEAK_MINMAX_BOXSIZE'] # Measurethe min and max flux miny, maxy = spirouBACK.MeasureMinMax(loc['SPE'][order_num], bsize) # subtract off the background (miny) loc['SPE'][order_num] = loc['SPE'][order_num] - miny # ------------------------------------------------------------------ # calculate flux ratio # ------------------------------------------------------------------ sorder = p['IC_DRIFT_ORDER_PLOT'] fratio = np.nansum(loc['SPE'][sorder]) / np.nansum( loc['SPEREF'][sorder]) loc['FLUXRATIO'][i_it] = fratio # ------------------------------------------------------------------ # Calculate delta time # ------------------------------------------------------------------ # calculate the time from reference (in hours) loc['DELTATIME'][i_it] = (bjdspe - bjdref) * 24 # ------------------------------------------------------------------ # Calculate PearsonR coefficient # ------------------------------------------------------------------ pargs = [loc['NUMBER_ORDERS'], loc['SPE'], loc['SPEREF']] correlation_coeffs = spirouRV.PearsonRtest(*pargs) # ---------------------------------------------------------------------- # Get drift with comparison to the reference image # ---------------------------------------------------------------------- # only calculate drift if the correlation between orders and # reference file is above threshold prcut = p['DRIFT_PEAK_PEARSONR_CUT'] if np.min(correlation_coeffs[nomin:nomax]) > prcut: # get drifts for each order dargs = [p, loc['SPE'], loc['ORDPEAK'], loc['XREF']] x = spirouRV.GetDrift(*dargs, gaussfit=gaussfit) # get delta v loc['DV'] = (x - loc['XREF']) * loc['VRPEAK'] # sigma clip loc = spirouRV.SigmaClip(loc, sigma=p['DRIFT_PEAK_SIGMACLIP']) # work out median drifts per order loc = spirouRV.DriftPerOrder(loc, i_it) # work out mean drift across all orders loc = spirouRV.DriftAllOrders(p, loc, i_it, nomin, nomax) # log the mean drift wmsg = ('Time from ref= {0:.2f} h - Flux Ratio= {1:.3f} ' '- Drift mean= {2:.2f} +- {3:.2f} m/s') wargs = [ loc['DELTATIME'][i_it], loc['FLUXRATIO'][i_it], loc['MEANRV'][i_it], loc['MERRDRIFT'][i_it] ] WLOG(p, 'info', wmsg.format(*wargs)) # else we can't use this extract else: if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.plt.ioff() # plot comparison between spe and ref sPlt.drift_plot_correlation_comp(p, loc, correlation_coeffs, i_it) sPlt.plt.show() sPlt.plt.close() # turn interactive plotting back on sPlt.plt.ion() # log that we cannot use this extraction wmsg1 = 'The correlation of some orders compared to the template is' wmsg2 = ' < {0}, something went wrong in the extract.' WLOG(p, 'warning', wmsg1) WLOG(p, 'warning', wmsg2.format(prcut)) # ------------------------------------------------------------------ # peak to peak drift driftptp = np.max(loc['MEANRV']) - np.min(loc['MEANRV']) driftrms = np.std(loc['MEANRV']) # log th etotal drift peak-to-peak and rms wmsg = ('Total drift Peak-to-Peak={0:.3f} m/s RMS={1:.3f} m/s in ' '{2:.2f} hour') wargs = [driftptp, driftrms, np.max(loc['DELTATIME'])] WLOG(p, '', wmsg.format(*wargs)) # ------------------------------------------------------------------ # Plot of mean drift # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot delta time against median drift sPlt.drift_peak_plot_dtime_against_drift(p, loc) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # TODO: Needs doing # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # add to qc header lists qc_values.append('None') qc_names.append('None') qc_logic.append('None') qc_pass.append(1) # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Save drift values to file # ------------------------------------------------------------------ # get raw input file name raw_infile = os.path.basename(p['REFFILE']) # construct filename driftfits, tag = spirouConfig.Constants.DRIFTPEAK_E2DS_FITS_FILE(p) driftfitsname = os.path.split(driftfits)[-1] # log that we are saving drift values wmsg = 'Saving drift values of Fiber {0} in {1}' WLOG(p, '', wmsg.format(p['FIBER'], driftfitsname)) # add keys from original header file hdict = spirouImage.CopyOriginalKeys(hdr) # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFLAT'], value=p['FLATFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_REFFILE'], value=raw_infile) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # save drift values p = spirouImage.WriteImage(p, driftfits, loc['DRIFT'], hdict) # ------------------------------------------------------------------ # print .tbl result # ------------------------------------------------------------------ # construct filename drifttbl = spirouConfig.Constants.DRIFTPEAK_E2DS_TBL_FILE(p) drifttblname = os.path.split(drifttbl)[-1] # construct and write table columnnames = ['time', 'drift', 'drifterr', 'drift_left', 'drift_right'] columnformats = ['7.4f', '6.2f', '6.3f', '6.2f', '6.2f'] columnvalues = [ loc['DELTATIME'], loc['MEANRV'], loc['MERRDRIFT'], loc['MEANRV_LEFT'], loc['MEANRV_RIGHT'] ] table = spirouImage.MakeTable(p, columns=columnnames, values=columnvalues, formats=columnformats) # write table wmsg = 'Average Drift saved in {0} Saved ' WLOG(p, '', wmsg.format(drifttblname)) spirouImage.WriteTable(p, table, drifttbl, fmt='ascii.rst') # ------------------------------------------------------------------ # Plot amp and llpeak # ------------------------------------------------------------------ if p['DRS_PLOT'] and p['DRIFT_PEAK_PLOT_LINE_LOG_AMP']: # start interactive session if needed sPlt.start_interactive_session(p) # plot delta time against median drift sPlt.drift_peak_plot_llpeak_amps(p, loc) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, hcfile=None, fpfiles=None): """ cal_SLIT_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_SLIT_spirou.py [night_directory] [files] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param files: string, list or None, the list of files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) if hcfile is None or fpfiles is None: names, types = ['hcfile', 'fpfiles'], [str, str] customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1], types, names, last_multi=True) else: customargs = dict(hcfile=hcfile, fpfiles=fpfiles) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='fpfiles') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, hcfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['HCFILE']) p, fpfilenames = spirouStartup.MultiFileSetup(p, files=p['FPFILES']) # set fiber (it doesn't matter with the 2D image but we need this to get # the lamp type for FPFILES and HCFILES, AB == C p['FIBER'] = 'AB' p['FIB_TYP'] = [p['FIBER']] fsource = __NAME__ + '/main()' p.set_sources(['FIBER', 'FIB_TYP'], fsource) # set the hcfilename to the first hcfilenames fpfitsfilename = fpfilenames[0] # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # add a force plot off p['PLOT_PER_ORDER'] = PLOT_PER_ORDER p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Read FP and HC files # ---------------------------------------------------------------------- # read and combine all FP files except the first (fpfitsfilename) rargs = [p, 'add', fpfitsfilename, fpfilenames[1:]] p, fpdata, fphdr = spirouImage.ReadImageAndCombine(*rargs) # read first file (hcfitsfilename) hcdata, hchdr, _, _ = spirouImage.ReadImage(p, hcfitsfilename) # add data and hdr to loc loc = ParamDict() loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr # set the source sources = ['HCDATA', 'HCHDR'] loc.set_sources(sources, 'spirouImage.ReadImageAndCombine()') sources = ['FPDATA', 'FPHDR'] loc.set_sources(sources, 'spirouImage.ReadImage()') # --------------------------------------------------------------------- # fix for un-preprocessed files # ---------------------------------------------------------------------- hcdata = spirouImage.FixNonPreProcess(p, hcdata) fpdata = spirouImage.FixNonPreProcess(p, fpdata) # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, fphdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, fphdr, name='exptime') # get gain p = spirouImage.GetGain(p, fphdr, name='gain') # get lamp parameters p = spirouTHORCA.GetLampParams(p, hchdr) # ---------------------------------------------------------------------- # Correction of DARK # ---------------------------------------------------------------------- # p, hcdatac = spirouImage.CorrectForDark(p, hcdata, hchdr) hcdatac = hcdata p['DARKFILE'] = 'None' # p, fpdatac = spirouImage.CorrectForDark(p, fpdata, fphdr) fpdatac = fpdata # ---------------------------------------------------------------------- # Resize hc data # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- hcdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, hcdatac), p=p) # convert NaN to zeros hcdata0 = np.where(~np.isfinite(hcdata), np.zeros_like(hcdata), hcdata) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) hcdata2 = spirouImage.ResizeImage(p, hcdata0, **bkwargs) # log change in data size WLOG(p, '', ('HC Image format changed to ' '{0}x{1}').format(*hcdata2.shape)) # ---------------------------------------------------------------------- # Resize fp data # ---------------------------------------------------------------------- # rotate the image and convert from ADU/s to e- fpdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, fpdatac), p=p) # convert NaN to zeros fpdata0 = np.where(~np.isfinite(fpdata), np.zeros_like(fpdata), fpdata) # resize image bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'], ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'], getshape=False) fpdata2 = spirouImage.ResizeImage(p, fpdata0, **bkwargs) # log change in data size WLOG(p, '', ('FP Image format changed to ' '{0}x{1}').format(*fpdata2.shape)) # ---------------------------------------------------------------------- # Correct for the BADPIX mask (set all bad pixels to zero) # ---------------------------------------------------------------------- # p, hcdata2 = spirouImage.CorrectForBadPix(p, hcdata2, hchdr) # p, fpdata2 = spirouImage.CorrectForBadPix(p, fpdata2, fphdr) p['BADPFILE'] = 'None' # save data to loc loc['HCDATA'] = hcdata2 loc.set_source('HCDATA', __NAME__ + '/main()') # save data to loc loc['FPDATA'] = fpdata2 loc.set_source('FPDATA', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- # get the number of bad pixels n_bad_pix = np.nansum(hcdata2 <= 0) n_bad_pix_frac = n_bad_pix * 100 / np.product(hcdata2.shape) # Log number wmsg = 'Nb HC dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ---------------------------------------------------------------------- # Log the number of dead pixels # ---------------------------------------------------------------------- # get the number of bad pixels n_bad_pix = np.nansum(fpdata2 <= 0) n_bad_pix_frac = n_bad_pix * 100 / np.product(fpdata2.shape) # Log number wmsg = 'Nb FP dead pixels = {0} / {1:.2f} %' WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac)) # ------------------------------------------------------------------ # Get localisation coefficients # ------------------------------------------------------------------ # original there is a loop but it is not used --> removed p = spirouImage.FiberParams(p, p['FIBER'], merge=True) # get localisation fit coefficients p, loc = spirouLOCOR.GetCoeffs(p, fphdr, loc) # ------------------------------------------------------------------ # Get master wave solution map # ------------------------------------------------------------------ # get master wave map masterwavefile = spirouDB.GetDatabaseMasterWave(p) # log process wmsg1 = 'Getting master wavelength grid' wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile)) WLOG(p, '', [wmsg1, wmsg2]) # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # read master wave map wout = spirouImage.GetWaveSolution(p, filename=masterwavefile, return_wavemap=True, quiet=True, return_header=True, fiber=wave_fiber) loc['MASTERWAVEP'], loc['MASTERWAVE'] = wout[:2] loc['MASTERWAVEHDR'], loc['WSOURCE'] = wout[2:] # set sources wsource = ['MASTERWAVEP', 'MASTERWAVE', 'MASTERWAVEHDR'] loc.set_sources(wsource, 'spirouImage.GetWaveSolution()') # ---------------------------------------------------------------------- # Read UNe solution # ---------------------------------------------------------------------- wave_u_ne, amp_u_ne = spirouImage.ReadLineList(p) loc['LL_LINE'], loc['AMPL_LINE'] = wave_u_ne, amp_u_ne source = __NAME__ + '.main() + spirouImage.ReadLineList()' loc.set_sources(['LL_LINE', 'AMPL_LINE'], source) # ---------------------------------------------------------------------- # Read cavity length file # ---------------------------------------------------------------------- loc['CAVITY_LEN_COEFFS'] = spirouImage.ReadCavityLength(p) source = __NAME__ + '.main() + spirouImage.ReadCavityLength()' loc.set_source('CAVITY_LEN_COEFFS', source) # ------------------------------------------------------------------ # Calculate shape map # ------------------------------------------------------------------ loc = spirouImage.GetShapeMap(p, loc) # ------------------------------------------------------------------ # Plotting # ------------------------------------------------------------------ if p['DRS_PLOT'] > 0: # plots setup: start interactive plot sPlt.start_interactive_session(p) # plot the shape process for one order sPlt.slit_shape_angle_plot(p, loc) # end interactive section sPlt.end_interactive_session(p) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # TODO: Decide on some quality control criteria? # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # add to qc header lists qc_values.append('None') qc_names.append('None') qc_logic.append('None') qc_pass.append(1) # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # Writing DXMAP to file # ------------------------------------------------------------------ # get the raw tilt file name raw_shape_file = os.path.basename(p['FITSFILENAME']) # construct file name and path shapefits, tag = spirouConfig.Constants.SLIT_XSHAPE_FILE(p) shapefitsname = os.path.basename(shapefits) # Log that we are saving tilt file wmsg = 'Saving shape information in file: {0}' WLOG(p, '', wmsg.format(shapefitsname)) # Copy keys from fits file hdict = spirouImage.CopyOriginalKeys(fphdr) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='hcfile', values=p['HCFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE2'], dim1name='fpfile', values=p['FPFILES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # write tilt file to file p = spirouImage.WriteImage(p, shapefits, loc['DXMAP'], hdict) # ------------------------------------------------------------------ # Writing sanity check files # ------------------------------------------------------------------ if p['SHAPE_DEBUG_OUTPUTS']: # log WLOG(p, '', 'Saving debug sanity check files') # construct file names input_fp_file, tag1 = spirouConfig.Constants.SLIT_SHAPE_IN_FP_FILE(p) output_fp_file, tag2 = spirouConfig.Constants.SLIT_SHAPE_OUT_FP_FILE(p) input_hc_file, tag3 = spirouConfig.Constants.SLIT_SHAPE_IN_HC_FILE(p) output_hc_file, tag4 = spirouConfig.Constants.SLIT_SHAPE_OUT_HC_FILE(p) overlap_file, tag5 = spirouConfig.Constants.SLIT_SHAPE_OVERLAP_FILE(p) # write input fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) p = spirouImage.WriteImage(p, input_fp_file, loc['FPDATA'], hdict) # write output fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) p = spirouImage.WriteImage(p, output_fp_file, loc['FPDATA2'], hdict) # write input fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3) p = spirouImage.WriteImage(p, input_hc_file, loc['HCDATA'], hdict) # write output fp file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4) p = spirouImage.WriteImage(p, output_hc_file, loc['HCDATA2'], hdict) # write overlap file hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag5) p = spirouImage.WriteImage(p, overlap_file, loc['ORDER_OVERLAP'], hdict) # ---------------------------------------------------------------------- # Move to calibDB and update calibDB # ---------------------------------------------------------------------- if p['QC']: keydb = 'SHAPE' # copy shape file to the calibDB folder spirouDB.PutCalibFile(p, shapefits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, shapefitsname, fphdr) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(runname=None, quiet=False): # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__, quiet=True) # now get custom arguments ckwargs = dict(positions=[0], types=[str], names=['RUNNAME'], calls=[runname], require_night_name=False, required=[False]) customargs = spirouStartup.GetCustomFromRuntime(p, **ckwargs) # add custom args straight to p p = spirouStartup.LoadMinimum(p, customargs=customargs) # ---------------------------------------------------------------------- # Read the run file and extract parameters # ---------------------------------------------------------------------- # construct filename rfile = os.path.join(UNIT_TEST_PATH, p['RUNNAME']) # check if RUNNAME is None if p['RUNNAME'] == 'None': exists = False emsgs = ['No unit test run file defined.'] # check that rfile exists elif not os.path.exists(rfile): emsgs = ['Unit test run file "{0}" does not exist'.format(rfile)] exists = False else: exists = True emsgs = [] # deal with file wrong (or no file defined) --> print valid unit tests if not exists: emsgs.append('') emsgs.append('Available units tests are:') for rfile in os.listdir(UNIT_TEST_PATH): emsgs.append('\t{0}'.format(rfile)) emsgs.append('') emsgs.append('Located at {0}'.format(UNIT_TEST_PATH)) WLOG(p, 'error', emsgs) # get the parameters in the run file rparams = spirouConfig.GetConfigParams(p, filename=rfile) # reset the DRS if not quiet: spirouTools.DRS_Reset(log=False, called=True) # ---------------------------------------------------------------------- # Get runs # ---------------------------------------------------------------------- runs = spirouUnitTests.get_runs(p, rparams, rfile) # ---------------------------------------------------------------------- # group runs (for parallelisation) # ---------------------------------------------------------------------- # get groups that can be run in parallel groups = group_runs(runs) # split groups by max number of processes groups = parallelize(groups, MAX_PROCESSES) # loop around groups for group_name in groups: # process storage pp = [] # loop around sub groups (to be run at the same time) for sub_group in groups[group_name]: # make sub_group a dict sruns = make_subgroup_dict(sub_group, group_name) # do parallel run process = Process(target=unit_wrapper, args=(p, sruns)) process.start() pp.append(process) # do not continue until for process in pp: while process.is_alive(): pass # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- wmsg = 'Recipe {0} has been successfully completed' WLOG(p, 'info', wmsg.format(p['PROGRAM'])) # return a copy of locally defined variables in the memory return dict(locals())
def main(preview=1): # ---------------------------------------------------------------------- # get values from config file p = spirouStartup.Begin(recipe=__NAME__, quiet=True) customargs = spirouStartup.GetCustomFromRuntime(p, [0], [int], ['preview'], calls=[preview], required=[False], require_night_name=False) p = spirouStartup.LoadArguments(p, None, customargs=customargs, require_night_name=False) # ---------------------------------------------------------------------- # if in preview mode tell user if p['PREVIEW']: WLOG(p, 'info', 'Running in preview mode.') # ---------------------------------------------------------------------- # read and ask for new version WLOG(p, '', 'Reading DRS version') # set new version version = ask_for_new_version() # add tag of version if version is not None: # tag head with version git_remove_tag(version) git_tag_head(version) new = True else: version = str(__version__) new = False # ---------------------------------------------------------------------- # update DRS files if not p['PREVIEW']: update_version_file(VERSIONFILE, version) update_py_version(CONSTFILE, version) # ---------------------------------------------------------------------- # create new changelog WLOG(p, '', 'Updating changelog') if not p['PREVIEW']: git_change_log(FILENAME) else: git_change_log('tmp.txt') preview_log('tmp.txt') if new: git_remove_tag(version) # ---------------------------------------------------------------------- # if we are in preview mode should we keep these changes and update version if p['PREVIEW']: uinput = input('Keep changes? [Y]es [N]o:\t') if 'Y' in uinput.upper(): # redo tagging git_remove_tag(version) git_tag_head(version) # update version file and python version file update_version_file(VERSIONFILE, version) update_py_version(CONSTFILE, version) # move the tmp.txt to change log shutil.move('tmp.txt', FILENAME) else: os.remove('tmp.txt') # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p, outputs=None) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, e2dsfile=None, mask=None, rv=None, width=None, step=None): """ cal_CCF_E2DS_spirou.py main function, if arguments are None uses arguments from run time i.e.: cal_CCF_E2DS_spirou.py [night_directory] [E2DSfilename] [mask] [RV] [width] [step] :param night_name: string or None, the folder within data raw directory containing files (also reduced directory) i.e. /data/raw/20170710 would be "20170710" but /data/raw/AT5/20180409 would be "AT5/20180409" :param e2dsfile: string, the E2DS file to use :param mask: string, the mask file to use (i.e. "UrNe.mas") :param rv: float, the target RV to use :param width: float, the CCF width to use :param step: float, the CCF step to use :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) # deal with arguments being None (i.e. get from sys.argv) pos = [0, 1, 2, 3, 4] fmt = [str, str, float, float, float] name = ['e2dsfile', 'ccf_mask', 'target_rv', 'ccf_width', 'ccf_step'] lname = ['input_file', 'CCF_mask', 'RV', 'CCF_width', 'CCF_step'] req = [True, True, True, False, False] call = [e2dsfile, mask, rv, width, step] call_priority = [True, True, True, True, True] # now get custom arguments customargs = spirouStartup.GetCustomFromRuntime(p, pos, fmt, name, req, call, call_priority, lname) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsfile='e2dsfile', mainfitsdir='reduced') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, e2dsfilename = spirouStartup.SingleFileSetup(p, filename=p['E2DSFILE']) # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Deal with optional run time arguments # ---------------------------------------------------------------------- # define default arguments (if ccf_width and ccf_step are not defined # in function call or run time arguments if 'ccf_width' not in p: p['CCF_WIDTH'] = p['IC_CCF_WIDTH'] if 'ccf_step' not in p: p['CCF_STEP'] = p['IC_CCF_STEP'] # ---------------------------------------------------------------------- # Read image file # ---------------------------------------------------------------------- # read the image data e2ds, hdr, nbo, nx = spirouImage.ReadData(p, e2dsfilename) # add to loc loc = ParamDict() loc['E2DS'] = e2ds loc['NUMBER_ORDERS'] = nbo loc.set_sources(['E2DS', 'number_orders'], __NAME__ + '/main()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hdr, name='exptime') # get gain p = spirouImage.GetGain(p, hdr, name='gain') # get acquisition time p = spirouImage.GetAcqTime(p, hdr, name='acqtime', kind='julian') # get obj name p = spirouImage.ReadParam(p, hdr, 'KW_OBJNAME', name='OBJNAME', dtype=str) bjdref = p['ACQTIME'] # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # ---------------------------------------------------------------------- # Earth Velocity calculation # ---------------------------------------------------------------------- if p['IC_IMAGE_TYPE'] == 'H4RG': p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, hdr) # ---------------------------------------------------------------------- # Read wavelength solution # ---------------------------------------------------------------------- # log WLOG(p, '', 'Reading wavelength solution ') # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get wave image wout = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True, return_filename=True, fiber=wave_fiber) param_ll, wave_ll, wavefile, wsource = wout # save to storage loc['PARAM_LL'], loc['WAVE_LL'], loc['WAVEFILE'], loc['WSOURCE'] = wout source = __NAME__ + '/main() + spirouTHORCA.GetWaveSolution()' loc.set_sources(['WAVE_LL', 'PARAM_LL', 'WAVEFILE', 'WSOURCE'], source) # ---------------------------------------------------------------------- # Read Flat file # ---------------------------------------------------------------------- # TODO We do not need to correct FLAT # log # WLOG(p, '', 'Reading Flat-Field ') # get flat # loc['FLAT'] = spirouImage.ReadFlatFile(p, hdr) # loc.set_source('FLAT', __NAME__ + '/main() + /spirouImage.ReadFlatFile') # get all values in flat that are zero to 1 # loc['FLAT'] = np.where(loc['FLAT'] == 0, 1.0, loc['FLAT']) # get blaze # p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hdr) p, blaze0 = spirouImage.ReadBlazeFile(p, hdr) # ---------------------------------------------------------------------- # Preliminary set up = no flat, no blaze # ---------------------------------------------------------------------- # reset flat to all ones # loc['FLAT'] = np.ones((nbo, nx)) # set blaze to all ones (if not bug in correlbin !!! # TODO Check why Blaze makes bugs in correlbin loc['BLAZE'] = np.ones((nbo, nx)) # set sources # loc.set_sources(['flat', 'blaze'], __NAME__ + '/main()') loc.set_sources(['blaze'], __NAME__ + '/main()') # Modification of E2DS array with N.A.N if np.isnan(np.sum(e2ds)): WLOG(p, 'warning', 'NaN values found in e2ds, converting process') # First basic approach Replacing N.A.N by zeros # e2ds[np.isnan(e2ds)] = 0 # Second approach replacing N.A.N by the Adjusted Blaze e2dsb = e2ds / blaze0 for i in np.arange(len(e2ds)): with warnings.catch_warnings(record=True) as _: rap = np.mean(e2dsb[i][np.isfinite(e2dsb[i])]) if np.isnan(rap): rap = 0.0 e2ds[i] = np.where(np.isfinite(e2dsb[i]), e2ds[i], blaze0[i] * rap) # ---------------------------------------------------------------------- # correct extracted image for flat # ---------------------------------------------------------------------- # loc['E2DSFF'] = e2ds/loc['FLAT'] # loc['E2DSFF'] = e2ds*1. loc['E2DSFF'] = e2ds loc.set_source('E2DSFF', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Compute photon noise uncertainty for reference file # ---------------------------------------------------------------------- # set up the arguments for DeltaVrms2D dargs = [loc['E2DS'], loc['WAVE_LL']] dkwargs = dict(sigdet=p['IC_DRIFT_NOISE'], size=p['IC_DRIFT_BOXSIZE'], threshold=p['IC_DRIFT_MAXFLUX']) # run DeltaVrms2D dvrmsref, wmeanref = spirouRV.DeltaVrms2D(*dargs, **dkwargs) # save to loc loc['DVRMSREF'], loc['WMEANREF'] = dvrmsref, wmeanref loc.set_sources(['dvrmsref', 'wmeanref'], __NAME__ + '/main()()') # log the estimated RV uncertainty # wmsg = 'On fiber {0} estimated RV uncertainty on spectrum is {1:.3f} m/s' # WLOG(p, 'info', wmsg.format(p['FIBER'], wmeanref)) wmsg = 'On fiber estimated RV uncertainty on spectrum is {0:.3f} m/s' WLOG(p, 'info', wmsg.format(wmeanref)) # TEST N.A.N # loc['E2DSFF'][20:22,2000:3000]=np.nan # e2ds[20:30,1000:3000]=np.nan # ---------------------------------------------------------------------- # Reference plots # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # start interactive session if needed sPlt.start_interactive_session(p) # plot FP spectral order sPlt.drift_plot_selected_wave_ref(p, loc, x=loc['WAVE_LL'], y=loc['E2DS']) # plot photon noise uncertainty sPlt.drift_plot_photon_uncertainty(p, loc) # ---------------------------------------------------------------------- # Get template RV (from ccf_mask) # ---------------------------------------------------------------------- # get the CCF mask from file (check location of mask) loc = spirouRV.GetCCFMask(p, loc) # check and deal with mask in microns (should be in nm) if np.mean(loc['LL_MASK_CTR']) < 2.0: loc['LL_MASK_CTR'] *= 1000.0 loc['LL_MASK_D'] *= 1000.0 # ---------------------------------------------------------------------- # Do correlation # ---------------------------------------------------------------------- # calculate and fit the CCF loc = spirouRV.Coravelation(p, loc) # ---------------------------------------------------------------------- # Correlation stats # ---------------------------------------------------------------------- # get the maximum number of orders to use nbmax = p['CCF_NUM_ORDERS_MAX'] # get the average ccf loc['AVERAGE_CCF'] = np.nansum(loc['CCF'][:nbmax], axis=0) # normalize the average ccf normalized_ccf = loc['AVERAGE_CCF'] / np.max(loc['AVERAGE_CCF']) # get the fit for the normalized average ccf ccf_res, ccf_fit = spirouRV.FitCCF(p, loc['RV_CCF'], normalized_ccf, fit_type=0) loc['CCF_RES'] = ccf_res loc['CCF_FIT'] = ccf_fit # get the max cpp loc['MAXCPP'] = np.nansum(loc['CCF_MAX']) / np.nansum( loc['PIX_PASSED_ALL']) # get the RV value from the normalised average ccf fit center location loc['RV'] = float(ccf_res[1]) # get the contrast (ccf fit amplitude) loc['CONTRAST'] = np.abs(100 * ccf_res[0]) # get the FWHM value loc['FWHM'] = ccf_res[2] * spirouCore.spirouMath.fwhm() # ---------------------------------------------------------------------- # set the source keys = [ 'average_ccf', 'maxcpp', 'rv', 'contrast', 'fwhm', 'ccf_res', 'ccf_fit' ] loc.set_sources(keys, __NAME__ + '/main()') # ---------------------------------------------------------------------- # log the stats wmsg = ('Correlation: C={0:.1f}[%] RV={1:.5f}[km/s] ' 'FWHM={2:.4f}[km/s] maxcpp={3:.1f}') wargs = [loc['CONTRAST'], loc['RV'], loc['FWHM'], loc['MAXCPP']] WLOG(p, 'info', wmsg.format(*wargs)) # ---------------------------------------------------------------------- # rv ccf plot # ---------------------------------------------------------------------- if p['DRS_PLOT'] > 0: # Plot rv vs ccf (and rv vs ccf_fit) sPlt.ccf_rv_ccf_plot(p, loc['RV_CCF'], normalized_ccf, ccf_fit) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # TODO: Needs doing # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # add to qc header lists qc_values.append('None') qc_names.append('None') qc_logic.append('None') qc_pass.append(1) # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ---------------------------------------------------------------------- # archive ccf to table # ---------------------------------------------------------------------- # construct filename res_table_file = spirouConfig.Constants.CCF_TABLE_FILE(p) # log progress WLOG(p, '', 'Archiving CCF on file {0}'.format(res_table_file)) # define column names columns = ['order', 'maxcpp', 'nlines', 'contrast', 'RV', 'sig'] # define values for each column values = [ loc['ORDERS'], loc['CCF_MAX'] / loc['PIX_PASSED_ALL'], loc['TOT_LINE'], np.abs(100 * loc['CCF_ALL_RESULTS'][:, 0]), loc['CCF_ALL_RESULTS'][:, 1], loc['CCF_ALL_RESULTS'][:, 2] ] # define the format for each column formats = ['2.0f', '5.0f', '4.0f', '4.1f', '9.4f', '7.4f'] # construct astropy table from column names, values and formats table = spirouImage.MakeTable(p, columns, values, formats) # save table to file spirouImage.WriteTable(p, table, res_table_file, fmt='ascii') # ---------------------------------------------------------------------- # archive ccf to fits file # ---------------------------------------------------------------------- raw_infile = os.path.basename(p['E2DSFILE']) # construct folder and filename corfile, tag = spirouConfig.Constants.CCF_FITS_FILE(p) corfilename = os.path.split(corfile)[-1] # log that we are archiving the CCF on file WLOG(p, '', 'Archiving CCF on file {0}'.format(corfilename)) # get constants from p mask = p['CCF_MASK'] # if file exists remove it if os.path.exists(corfile): os.remove(corfile) # add the average ccf to the end of ccf data = np.vstack([loc['CCF'], loc['AVERAGE_CCF']]) # add drs keys hdict = spirouImage.CopyOriginalKeys(hdr) hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE']) hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['E2DSFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_INCCFMASK'], value=p['CCF_MASK']) hdict = spirouImage.AddKey(p, hdict, p['KW_INRV'], value=p['TARGET_RV']) hdict = spirouImage.AddKey(p, hdict, p['KW_INWIDTH'], value=p['CCF_WIDTH']) hdict = spirouImage.AddKey(p, hdict, p['KW_INSTEP'], value=p['CCF_STEP']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # add CCF keys hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CTYPE'], value='km/s') hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CRVAL'], value=loc['RV_CCF'][0]) # the rv step rvstep = np.abs(loc['RV_CCF'][0] - loc['RV_CCF'][1]) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CDELT'], value=rvstep) # add ccf stats hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_RV'], value=loc['CCF_RES'][1]) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_RVC'], value=loc['RV']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_FWHM'], value=loc['FWHM']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_WMREF'], value=loc['WMEANREF']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CONTRAST'], value=loc['CONTRAST']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_MAXCPP'], value=loc['MAXCPP']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_MASK'], value=p['CCF_MASK']) hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_LINES'], value=np.nansum(loc['TOT_LINE'])) # add berv values hdict = spirouImage.AddKey(p, hdict, p['KW_BERV'], value=loc['BERV']) hdict = spirouImage.AddKey(p, hdict, p['KW_BJD'], value=loc['BJD']) hdict = spirouImage.AddKey(p, hdict, p['KW_BERV_MAX'], value=loc['BERV_MAX']) # write image and add header keys (via hdict) p = spirouImage.WriteImage(p, corfile, data, hdict) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return a copy of locally defined variables in the memory return dict(locals())
def main(night_name=None, fpfile=None, hcfiles=None): """ cal_wave_spirou.py main function, if night_name and files are None uses arguments from run time i.e.: cal_wave_spirou.py [night_directory] [fpfile] [hcfiles] :param night_name: string or None, the folder within data reduced directory containing files (also reduced directory) i.e. /data/reduced/20170710 would be "20170710" but /data/reduced/AT5/20180409 is "AT5/20180409" :param fpfile: string, or None, the FP file to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :param hcfiles: string, list or None, the list of HC files to use for arg_file_names and fitsfilename (if None assumes arg_file_names was set from run time) :return ll: dictionary, containing all the local variables defined in main """ # ---------------------------------------------------------------------- # Set up # ---------------------------------------------------------------------- # get parameters from config files/run time args/load paths + calibdb p = spirouStartup.Begin(recipe=__NAME__) if hcfiles is None or fpfile is None: names, types = ['fpfile', 'hcfiles'], [str, str] customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1], types, names, last_multi=True) else: customargs = dict(hcfiles=hcfiles, fpfile=fpfile) # get parameters from configuration files and run time arguments p = spirouStartup.LoadArguments(p, night_name, customargs=customargs, mainfitsdir='reduced', mainfitsfile='hcfiles') # ---------------------------------------------------------------------- # Construct reference filename and get fiber type # ---------------------------------------------------------------------- p, fpfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['FPFILE']) fiber1 = str(p['FIBER']) p, hcfilenames = spirouStartup.MultiFileSetup(p, files=p['HCFILES']) fiber2 = str(p['FIBER']) # set the hcfilename to the first hcfilenames hcfitsfilename = hcfilenames[0] # ---------------------------------------------------------------------- # Once we have checked the e2dsfile we can load calibDB # ---------------------------------------------------------------------- # as we have custom arguments need to load the calibration database p = spirouStartup.LoadCalibDB(p) # ---------------------------------------------------------------------- # Have to check that the fibers match # ---------------------------------------------------------------------- if fiber1 == fiber2: p['FIBER'] = fiber1 fsource = __NAME__ + '/main() & spirouStartup.GetFiberType()' p.set_source('FIBER', fsource) else: emsg = 'Fiber not matching for {0} and {1}, should be the same' eargs = [hcfitsfilename, fpfitsfilename] WLOG(p, 'error', emsg.format(*eargs)) # set the fiber type p['FIB_TYP'] = [p['FIBER']] p.set_source('FIB_TYP', __NAME__ + '/main()') # ---------------------------------------------------------------------- # Read FP and HC files # ---------------------------------------------------------------------- # read and combine all HC files except the first (fpfitsfilename) rargs = [p, 'add', hcfitsfilename, hcfilenames[1:]] p, hcdata, hchdr = spirouImage.ReadImageAndCombine(*rargs) # read first file (fpfitsfilename) fpdata, fphdr, _, _ = spirouImage.ReadImage(p, fpfitsfilename) # add data and hdr to loc loc = ParamDict() loc['HCDATA'], loc['HCHDR'], loc['HCCDR'] = hcdata, hchdr, hchdr.comments loc['FPDATA'], loc['FPHDR'], loc['FPCDR'] = fpdata, fphdr, fphdr.comments # set the source sources = ['HCDATA', 'HCHDR', 'HCCDR'] loc.set_sources(sources, 'spirouImage.ReadImageAndCombine()') sources = ['FPDATA', 'FPHDR', 'FPCDR'] loc.set_sources(sources, 'spirouImage.ReadImage()') # ---------------------------------------------------------------------- # Get basic image properties for reference file # ---------------------------------------------------------------------- # get sig det value p = spirouImage.GetSigdet(p, hchdr, name='sigdet') # get exposure time p = spirouImage.GetExpTime(p, hchdr, name='exptime') # get gain p = spirouImage.GetGain(p, hchdr, name='gain') # get acquisition time p = spirouImage.GetAcqTime(p, hchdr, name='acqtime', kind='julian') bjdref = p['ACQTIME'] # set sigdet and conad keywords (sigdet is changed later) p['KW_CCD_SIGDET'][1] = p['SIGDET'] p['KW_CCD_CONAD'][1] = p['GAIN'] # get lamp parameters p = spirouWAVE2.get_lamp_parameters(p, hchdr) # get number of orders # we always get fibre A number because AB is doubled in constants file loc['NBO'] = p['QC_LOC_NBO_FPALL']['A'] loc.set_source('NBO', __NAME__ + '.main()') # get number of pixels in x from hcdata size loc['NBPIX'] = loc['HCDATA'].shape[1] loc.set_source('NBPIX', __NAME__ + '.main()') # ---------------------------------------------------------------------- # Read blaze # ---------------------------------------------------------------------- # get tilts p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hchdr) loc.set_source('BLAZE', __NAME__ + '/main() + /spirouImage.ReadBlazeFile') # make copy of blaze (as it's overwritten later in CCF part) # TODO is this needed? More sensible to make and set copy in CCF? loc['BLAZE2'] = np.copy(loc['BLAZE']) # ---------------------------------------------------------------------- # Read wave solution # ---------------------------------------------------------------------- # wavelength file; we will use the polynomial terms in its header, # NOT the pixel values that would need to be interpolated # set source of wave file wsource = __NAME__ + '/main() + /spirouImage.GetWaveSolution' # Force A and B to AB solution if p['FIBER'] in ['A', 'B']: wave_fiber = 'AB' else: wave_fiber = p['FIBER'] # get wave image wout = spirouImage.GetWaveSolution(p, hdr=hchdr, return_wavemap=True, return_filename=True, fiber=wave_fiber) loc['WAVEPARAMS'], loc['WAVE_INIT'], loc['WAVEFILE'], loc['WSOURCE'] = wout loc.set_sources(['WAVE_INIT', 'WAVEFILE', 'WAVEPARAMS', 'WSOURCE'], wsource) poly_wave_sol = loc['WAVEPARAMS'] # ---------------------------------------------------------------------- # Check that wave parameters are consistent with "ic_ll_degr_fit" # ---------------------------------------------------------------------- loc = spirouImage.CheckWaveSolConsistency(p, loc) # ---------------------------------------------------------------------- # HC wavelength solution # ---------------------------------------------------------------------- # log that we are running the HC part and the mode wmsg = 'Now running the HC solution, mode = {0}' WLOG(p, 'info', wmsg.format(p['WAVE_MODE_HC'])) # get the solution loc = spirouWAVE2.do_hc_wavesol(p, loc) # ---------------------------------------------------------------------- # Quality control - HC solution # ---------------------------------------------------------------------- # set passed variable and fail message list passed, fail_msg = True, [] qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # quality control on sigma clip (sig1 > qc_hc_wave_sigma_max if loc['SIG1'] > p['QC_HC_WAVE_SIGMA_MAX']: fmsg = 'Sigma too high ({0:.5f} > {1:.5f})' fail_msg.append(fmsg.format(loc['SIG1'], p['QC_HC_WAVE_SIGMA_MAX'])) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(loc['SIG1']) qc_names.append('SIG1 HC') qc_logic.append('SIG1 > {0:.2f}'.format(p['QC_HC_WAVE_SIGMA_MAX'])) # ---------------------------------------------------------------------- # check the difference between consecutive orders is always positive # get the differences wave_diff = loc['WAVE_MAP2'][1:] - loc['WAVE_MAP2'][:-1] if np.min(wave_diff) < 0: fmsg = 'Negative wavelength difference between orders' fail_msg.append(fmsg) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(np.min(wave_diff)) qc_names.append('MIN WAVE DIFF HC') qc_logic.append('MIN WAVE DIFF < 0') # ---------------------------------------------------------------------- # check the difference between consecutive pixels along an order is # always positive # loop through the orders ord_check = np.zeros((loc['NBO']), dtype=bool) for order in range(loc['NBO']): oc = np.all(loc['WAVE_MAP2'][order, 1:] > loc['WAVE_MAP2'][order, :-1]) ord_check[order] = oc # TODO: Melissa Why is this here???? # ord_check[5] = False if np.all(ord_check): qc_pass.append(1) qc_values.append('None') else: fmsg = 'Negative wavelength difference along an order' fail_msg.append(fmsg) passed = False qc_pass.append(0) qc_values.append(np.ndarray.tolist(np.where(~ord_check)[0])) # add to qc header lists # vale: array of orders where it fails qc_names.append('WAVE DIFF ALONG ORDER HC') qc_logic.append('WAVE DIFF ALONG ORDER < 0') # ---------------------------------------------------------------------- # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ---------------------------------------------------------------------- # log the global stats # ---------------------------------------------------------------------- # calculate catalog-fit residuals in km/s res_hc = [] sumres_hc = 0.0 sumres2_hc = 0.0 for order in range(loc['NBO']): # get HC line wavelengths for the order order_mask = loc['ORD_T'] == order hc_x_ord = loc['XGAU_T'][order_mask] hc_ll_ord = np.polyval(loc['POLY_WAVE_SOL'][order][::-1], hc_x_ord) hc_ll_cat = loc['WAVE_CATALOG'][order_mask] hc_ll_diff = hc_ll_ord - hc_ll_cat res_hc.append(hc_ll_diff * speed_of_light / hc_ll_cat) sumres_hc += np.nansum(res_hc[order]) sumres2_hc += np.nansum(res_hc[order]**2) total_lines_hc = len(np.concatenate(res_hc)) final_mean_hc = sumres_hc / total_lines_hc final_var_hc = (sumres2_hc / total_lines_hc) - (final_mean_hc**2) wmsg1 = 'On fiber {0} HC fit line statistic:'.format(p['FIBER']) wargs2 = [ final_mean_hc * 1000.0, np.sqrt(final_var_hc) * 1000.0, total_lines_hc, 1000.0 * np.sqrt(final_var_hc / total_lines_hc) ] wmsg2 = ('\tmean={0:.3f}[m/s] rms={1:.1f} {2} HC lines (error on mean ' 'value:{3:.4f}[m/s])'.format(*wargs2)) WLOG(p, 'info', [wmsg1, wmsg2]) # ---------------------------------------------------------------------- # Save wave map to file # ---------------------------------------------------------------------- # TODO single file-naming function? Ask Neil # get base input filenames bfilenames = [] for raw_file in p['ARG_FILE_NAMES']: bfilenames.append(os.path.basename(raw_file)) # get wave filename wavefits, tag1 = spirouConfig.Constants.WAVE_FILE_EA(p) wavefitsname = os.path.basename(wavefits) # log progress WLOG(p, '', 'Saving wave map to {0}'.format(wavefitsname)) # log progress wargs = [p['FIBER'], wavefitsname] wmsg = 'Write wavelength solution for Fiber {0} in {1}' WLOG(p, '', wmsg.format(*wargs)) # write solution to fitsfilename header # copy original keys hdict = spirouImage.CopyOriginalKeys(loc['HCHDR'], loc['HCCDR']) # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) # TODO add DRS_DATE and DRS_NOW hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BLAZFILE']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # add wave solution date hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME1'], value=p['MAX_TIME_HUMAN']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME2'], value=p['MAX_TIME_UNIX']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_CODE'], value=__NAME__) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file', values=p['ARG_FILE_NAMES']) # add number of orders hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_ORD_N'], value=loc['POLY_WAVE_SOL'].shape[0]) # add degree of fit hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_LL_DEG'], value=loc['POLY_WAVE_SOL'].shape[1] - 1) # add wave solution hdict = spirouImage.AddKey2DList(p, hdict, p['KW_WAVE_PARAM'], values=loc['POLY_WAVE_SOL']) # write the wave "spectrum" p = spirouImage.WriteImage(p, wavefits, loc['WAVE_MAP2'], hdict) # get filename for E2DS calibDB copy of FITSFILENAME e2dscopy_filename, tag2 = spirouConfig.Constants.WAVE_E2DS_COPY(p) wargs = [p['FIBER'], os.path.split(e2dscopy_filename)[-1]] wmsg = 'Write reference E2DS spectra for Fiber {0} in {1}' WLOG(p, '', wmsg.format(*wargs)) # make a copy of the E2DS file for the calibBD hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2) p = spirouImage.WriteImage(p, e2dscopy_filename, loc['HCDATA'], hdict) # ---------------------------------------------------------------------- # Save resolution and line profiles to file # ---------------------------------------------------------------------- raw_infile = os.path.basename(p['FITSFILENAME']) # get wave filename resfits, tag3 = spirouConfig.Constants.WAVE_RES_FILE_EA(p) resfitsname = os.path.basename(resfits) WLOG(p, '', 'Saving wave resmap to {0}'.format(resfitsname)) # make a copy of the E2DS file for the calibBD # set the version hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) # TODO add DRS_DATE and DRS_NOW hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3) # get res data in correct format resdata, hdicts = spirouWAVE2.generate_res_files(p, loc, hdict) # save to file p = spirouImage.WriteImageMulti(p, resfits, resdata, hdicts=hdicts) # ---------------------------------------------------------------------- # Update calibDB # ---------------------------------------------------------------------- if p['QC']: # set the wave key keydb = 'WAVE_{0}'.format(p['FIBER']) # copy wave file to calibDB folder spirouDB.PutCalibFile(p, wavefits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, wavefitsname, loc['HCHDR']) # set the hcref key keydb = 'HCREF_{0}'.format(p['FIBER']) # copy wave file to calibDB folder spirouDB.PutCalibFile(p, e2dscopy_filename) # update the master calib DB file with new key e2dscopyfits = os.path.split(e2dscopy_filename)[-1] spirouDB.UpdateCalibMaster(p, keydb, e2dscopyfits, loc['HCHDR']) # ---------------------------------------------------------------------- # Update header of current files # ---------------------------------------------------------------------- # only copy over if QC passed if p['QC']: rdir = os.path.dirname(wavefits) # loop around hc files and update header with for rawhcfile in p['ARG_FILE_NAMES']: hcfile = os.path.join(rdir, rawhcfile) raw_infilepath1 = os.path.join(p['ARG_FILE_DIR'], hcfile) p = spirouImage.UpdateWaveSolutionHC(p, loc, raw_infilepath1) # ---------------------------------------------------------------------- # HC+FP wavelength solution # ---------------------------------------------------------------------- # check if there's a FP input and if HC solution passed QCs if has_fp and p['QC']: # log that we are doing the FP solution wmsg = 'Now running the combined FP-HC solution, mode = {}' WLOG(p, 'info', wmsg.format(p['WAVE_MODE_FP'])) # do the wavelength solution loc = spirouWAVE2.do_fp_wavesol(p, loc) # ---------------------------------------------------------------------- # Quality control # ---------------------------------------------------------------------- # get parameters ffrom p p['QC_RMS_LITTROW_MAX'] = p['QC_HC_RMS_LITTROW_MAX'] p['QC_DEV_LITTROW_MAX'] = p['QC_HC_DEV_LITTROW_MAX'] # set passed variable and fail message list # passed, fail_msg = True, [] # qc_values, qc_names, qc_logic, qc_pass = [], [], [], [] # ---------------------------------------------------------------------- # check the difference between consecutive orders is always positive # get the differences wave_diff = loc['LL_FINAL'][1:] - loc['LL_FINAL'][:-1] if np.min(wave_diff) < 0: fmsg = 'Negative wavelength difference between orders' fail_msg.append(fmsg) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(np.min(wave_diff)) qc_names.append('MIN WAVE DIFF FP-HC') qc_logic.append('MIN WAVE DIFF < 0') # ---------------------------------------------------------------------- # check for infinites and NaNs in mean residuals from fit if ~np.isfinite(loc['X_MEAN_2']): # add failed message to the fail message list fmsg = 'NaN or Inf in X_MEAN_2' fail_msg.append(fmsg) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(loc['X_MEAN_2']) qc_names.append('X_MEAN_2') qc_logic.append('X_MEAN_2 not finite') # ---------------------------------------------------------------------- # iterate through Littrow test cut values lit_it = 2 # checks every other value # TODO: This QC check (or set of QC checks needs re-writing it is # TODO: nearly impossible to understand for x_it in range(1, len(loc['X_CUT_POINTS_' + str(lit_it)]), 2): # get x cut point x_cut_point = loc['X_CUT_POINTS_' + str(lit_it)][x_it] # get the sigma for this cut point sig_littrow = loc['LITTROW_SIG_' + str(lit_it)][x_it] # get the abs min and max dev littrow values min_littrow = abs(loc['LITTROW_MINDEV_' + str(lit_it)][x_it]) max_littrow = abs(loc['LITTROW_MAXDEV_' + str(lit_it)][x_it]) # get the corresponding order min_littrow_ord = loc['LITTROW_MINDEVORD_' + str(lit_it)][x_it] max_littrow_ord = loc['LITTROW_MAXDEVORD_' + str(lit_it)][x_it] # check if sig littrow is above maximum rms_littrow_max = p['QC_RMS_LITTROW_MAX'] dev_littrow_max = p['QC_DEV_LITTROW_MAX'] if sig_littrow > rms_littrow_max: fmsg = ('Littrow test (x={0}) failed (sig littrow = ' '{1:.2f} > {2:.2f})') fargs = [x_cut_point, sig_littrow, rms_littrow_max] fail_msg.append(fmsg.format(*fargs)) passed = False qc_pass.append(0) else: qc_pass.append(1) # add to qc header lists qc_values.append(sig_littrow) qc_names.append('sig_littrow') qc_logic.append('sig_littrow > {0:.2f}'.format(rms_littrow_max)) # ---------------------------------------------------------------------- # check if min/max littrow is out of bounds if np.max([max_littrow, min_littrow]) > dev_littrow_max: fmsg = ('Littrow test (x={0}) failed (min|max dev = ' '{1:.2f}|{2:.2f} > {3:.2f} for order {4}|{5})') fargs = [ x_cut_point, min_littrow, max_littrow, dev_littrow_max, min_littrow_ord, max_littrow_ord ] fail_msg.append(fmsg.format(*fargs)) passed = False qc_pass.append(0) # TODO: Should this be the QC header values? # TODO: it does not change the outcome of QC (i.e. passed=False) # TODO: So what is the point? # if sig was out of bounds, recalculate if sig_littrow > rms_littrow_max: # conditions check1 = min_littrow > dev_littrow_max check2 = max_littrow > dev_littrow_max # get the residuals respix = loc['LITTROW_YY_' + str(lit_it)][x_it] # check if both are out of bounds if check1 and check2: # remove respective orders worst_order = (min_littrow_ord, max_littrow_ord) respix_2 = np.delete(respix, worst_order) redo_sigma = True # check if min is out of bounds elif check1: # remove respective order worst_order = min_littrow_ord respix_2 = np.delete(respix, worst_order) redo_sigma = True # check if max is out of bounds elif check2: # remove respective order worst_order = max_littrow_ord respix_2 = np.delete(respix, max_littrow_ord) redo_sigma = True # else do not recalculate sigma else: redo_sigma, respix_2, worst_order = False, None, None wmsg = 'No outlying orders, sig littrow not recalculated' fail_msg.append(wmsg.format()) # if outlying order, recalculate stats if redo_sigma: mean = np.nansum(respix_2) / len(respix_2) mean2 = np.nansum(respix_2**2) / len(respix_2) rms = np.sqrt(mean2 - mean**2) if rms > rms_littrow_max: fmsg = ( 'Littrow test (x={0}) failed (sig littrow = ' '{1:.2f} > {2:.2f} removing order {3})') fargs = [ x_cut_point, rms, rms_littrow_max, worst_order ] fail_msg.append(fmsg.format(*fargs)) else: wargs = [ x_cut_point, rms, rms_littrow_max, worst_order ] wmsg = ( 'Littrow test (x={0}) passed (sig littrow = ' '{1:.2f} > {2:.2f} removing order {3})') fail_msg.append(wmsg.format(*wargs)) else: qc_pass.append(1) # add to qc header lists qc_values.append(np.max([max_littrow, min_littrow])) qc_names.append('max or min littrow') qc_logic.append('max or min littrow > {0:.2f}' ''.format(dev_littrow_max)) # finally log the failed messages and set QC = 1 if we pass the # quality control QC = 0 if we fail quality control if passed: WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -') p['QC'] = 1 p.set_source('QC', __NAME__ + '/main()') else: for farg in fail_msg: wmsg = 'QUALITY CONTROL FAILED: {0}' WLOG(p, 'warning', wmsg.format(farg)) p['QC'] = 0 p.set_source('QC', __NAME__ + '/main()') # store in qc_params qc_params = [qc_names, qc_values, qc_logic, qc_pass] # ------------------------------------------------------------------ # archive result in e2ds spectra # ------------------------------------------------------------------ # get raw input file name(s) raw_infiles1 = [] for hcfile in p['HCFILES']: raw_infiles1.append(os.path.basename(hcfile)) raw_infile2 = os.path.basename(p['FPFILE']) # get wave filename wavefits, tag1 = spirouConfig.Constants.WAVE_FILE_EA_2(p) wavefitsname = os.path.split(wavefits)[-1] # log progress wargs = [p['FIBER'], wavefits] wmsg = 'Write wavelength solution for Fiber {0} in {1}' WLOG(p, '', wmsg.format(*wargs)) # write solution to fitsfilename header # copy original keys hdict = spirouImage.CopyOriginalKeys(loc['HCHDR'], loc['HCCDR']) # add version number hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION']) hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID']) # set the input files hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BLAZFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'], value=loc['WAVEFILE']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'], value=loc['WSOURCE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='fpfile', values=p['FPFILE']) hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE2'], dim1name='hcfile', values=p['HCFILES']) # add qc parameters hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC']) hdict = spirouImage.AddQCKeys(p, hdict, qc_params) # add wave solution date hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME1'], value=p['MAX_TIME_HUMAN']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME2'], value=p['MAX_TIME_UNIX']) hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_CODE'], value=__NAME__) # add number of orders hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_ORD_N'], value=loc['LL_PARAM_FINAL'].shape[0]) # add degree of fit hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_LL_DEG'], value=loc['LL_PARAM_FINAL'].shape[1] - 1) # add wave solution hdict = spirouImage.AddKey2DList(p, hdict, p['KW_WAVE_PARAM'], values=loc['LL_PARAM_FINAL']) # add FP CCF drift # target RV and width hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_TARG_RV'], value=p['TARGET_RV']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_WIDTH'], value=p['CCF_WIDTH']) # the rv step # rvstep = np.abs(loc['RV_CCF'][0] - loc['RV_CCF'][1]) # hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CDELT'], value=rvstep) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_STEP'], value=p['CCF_STEP']) # add ccf stats hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_DRIFT'], value=loc['CCF_RES'][1]) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_FWHM'], value=loc['FWHM']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_CONTRAST'], value=loc['CONTRAST']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_MAXCPP'], value=loc['MAXCPP']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_MASK'], value=p['CCF_MASK']) hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_LINES'], value=np.nansum(loc['TOT_LINE'])) # write the wave "spectrum" hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1) p = spirouImage.WriteImage(p, wavefits, loc['LL_FINAL'], hdict) # get filename for E2DS calibDB copy of FITSFILENAME e2dscopy_filename = spirouConfig.Constants.WAVE_E2DS_COPY(p)[0] wargs = [p['FIBER'], os.path.split(e2dscopy_filename)[-1]] wmsg = 'Write reference E2DS spectra for Fiber {0} in {1}' WLOG(p, '', wmsg.format(*wargs)) # make a copy of the E2DS file for the calibBD p = spirouImage.WriteImage(p, e2dscopy_filename, loc['HCDATA'], hdict) # only copy over if QC passed if p['QC']: # loop around hc files and update header with for hcfile in p['HCFILES']: raw_infilepath1 = os.path.join(p['ARG_FILE_DIR'], hcfile) p = spirouImage.UpdateWaveSolution(p, loc, raw_infilepath1) # update fp file raw_infilepath2 = os.path.join(p['ARG_FILE_DIR'], raw_infile2) p = spirouImage.UpdateWaveSolution(p, loc, raw_infilepath2) # ------------------------------------------------------------------ # Save to result table # ------------------------------------------------------------------ # calculate stats for table final_mean = 1000 * loc['X_MEAN_2'] final_var = 1000 * loc['X_VAR_2'] num_lines = loc['TOTAL_LINES_2'] err = 1000 * np.sqrt(loc['X_VAR_2'] / num_lines) sig_littrow = 1000 * np.array(loc['LITTROW_SIG_' + str(lit_it)]) # construct filename wavetbl = spirouConfig.Constants.WAVE_TBL_FILE_EA(p) wavetblname = os.path.basename(wavetbl) # construct and write table columnnames = [ 'night_name', 'file_name', 'fiber', 'mean', 'rms', 'N_lines', 'err', 'rms_L500', 'rms_L1000', 'rms_L1500', 'rms_L2000', 'rms_L2500', 'rms_L3000', 'rms_L3500' ] columnformats = [ '{:20s}', '{:30s}', '{:3s}', '{:7.4f}', '{:6.2f}', '{:3d}', '{:6.3f}', '{:6.2f}', '{:6.2f}', '{:6.2f}', '{:6.2f}', '{:6.2f}', '{:6.2f}', '{:6.2f}' ] columnvalues = [[p['ARG_NIGHT_NAME']], [p['ARG_FILE_NAMES'][0]], [p['FIBER']], [final_mean], [final_var], [num_lines], [err], [sig_littrow[0]], [sig_littrow[1]], [sig_littrow[2]], [sig_littrow[3]], [sig_littrow[4]], [sig_littrow[5]], [sig_littrow[6]]] # make table table = spirouImage.MakeTable(p, columns=columnnames, values=columnvalues, formats=columnformats) # merge table wmsg = 'Global result summary saved in {0}' WLOG(p, '', wmsg.format(wavetblname)) spirouImage.MergeTable(p, table, wavetbl, fmt='ascii.rst') # ------------------------------------------------------------------ # Save line list table file # ------------------------------------------------------------------ # construct filename # TODO proper column values wavelltbl = spirouConfig.Constants.WAVE_LINE_FILE_EA(p) wavelltblname = os.path.split(wavelltbl)[-1] # construct and write table columnnames = ['order', 'll', 'dv', 'w', 'xi', 'xo', 'dvdx'] columnformats = [ '{:.0f}', '{:12.4f}', '{:13.5f}', '{:12.4f}', '{:12.4f}', '{:12.4f}', '{:8.4f}' ] columnvalues = [] # construct column values (flatten over orders) for it in range(len(loc['X_DETAILS_2'])): for jt in range(len(loc['X_DETAILS_2'][it][0])): row = [ float(it), loc['X_DETAILS_2'][it][0][jt], loc['LL_DETAILS_2'][it][0][jt], loc['X_DETAILS_2'][it][3][jt], loc['X_DETAILS_2'][it][1][jt], loc['X_DETAILS_2'][it][2][jt], loc['SCALE_2'][it][jt] ] columnvalues.append(row) # log saving wmsg = 'List of lines used saved in {0}' WLOG(p, '', wmsg.format(wavelltblname)) # make table columnvalues = np.array(columnvalues).T table = spirouImage.MakeTable(p, columns=columnnames, values=columnvalues, formats=columnformats) # write table spirouImage.WriteTable(p, table, wavelltbl, fmt='ascii.rst') # ------------------------------------------------------------------ # Move to calibDB and update calibDB # ------------------------------------------------------------------ if p['QC']: # set the wave key keydb = 'WAVE_{0}'.format(p['FIBER']) # copy wave file to calibDB folder spirouDB.PutCalibFile(p, wavefits) # update the master calib DB file with new key spirouDB.UpdateCalibMaster(p, keydb, wavefitsname, loc['HCHDR']) # set the hcref key keydb = 'HCREF_{0}'.format(p['FIBER']) # copy wave file to calibDB folder spirouDB.PutCalibFile(p, e2dscopy_filename) # update the master calib DB file with new key e2dscopyfits = os.path.split(e2dscopy_filename)[-1] spirouDB.UpdateCalibMaster(p, keydb, e2dscopyfits, loc['HCHDR']) # If the HC solution failed QCs we do not compute FP-HC solution elif has_fp and not p['QC']: wmsg = 'HC solution failed quality controls; FP not processed' WLOG(p, 'warning', wmsg) # If there is no FP file we log that elif not has_fp: wmsg = 'No FP file given; FP-HC combined solution cannot be generated' WLOG(p, 'warning', wmsg) # ---------------------------------------------------------------------- # End Message # ---------------------------------------------------------------------- p = spirouStartup.End(p) # return p and loc return dict(locals())