Esempio n. 1
0
def main(night_name=None, files=None):
    """
    cal_SLIT_spirou.py main function, if night_name and files are None uses
    arguments from run time i.e.:
        cal_SLIT_spirou.py [night_directory] [files]

    :param night_name: string or None, the folder within data raw directory
                                containing files (also reduced directory) i.e.
                                /data/raw/20170710 would be "20170710" but
                                /data/raw/AT5/20180409 would be "AT5/20180409"
    :param files: string, list or None, the list of files to use for
                  arg_file_names and fitsfilename
                  (if None assumes arg_file_names was set from run time)

    :return ll: dictionary, containing all the local variables defined in
                main
    """
    # ----------------------------------------------------------------------
    # Set up
    # ----------------------------------------------------------------------
    # get parameters from config files/run time args/load paths + calibdb
    p = spirouStartup.Begin(recipe=__NAME__)
    p = spirouStartup.LoadArguments(p, night_name, files)
    p = spirouStartup.InitialFileSetup(p, calibdb=True)
    # set the fiber type
    p['FIB_TYP'] = 'AB'
    p.set_source('FIB_TYP', __NAME__ + '/main()')

    # ----------------------------------------------------------------------
    # Read image file
    # ----------------------------------------------------------------------
    # read the image data
    p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add')

    # ----------------------------------------------------------------------
    # fix for un-preprocessed files
    # ----------------------------------------------------------------------
    data = spirouImage.FixNonPreProcess(p, data)

    # ----------------------------------------------------------------------
    # Get basic image properties
    # ----------------------------------------------------------------------
    # get sig det value
    p = spirouImage.GetSigdet(p, hdr, name='sigdet')
    # get exposure time
    p = spirouImage.GetExpTime(p, hdr, name='exptime')
    # get gain
    p = spirouImage.GetGain(p, hdr, name='gain')

    # ----------------------------------------------------------------------
    # Correction of DARK
    # ----------------------------------------------------------------------
    p, datac = spirouImage.CorrectForDark(p, data, hdr)

    # ----------------------------------------------------------------------
    # Resize image
    # ----------------------------------------------------------------------
    # rotate the image and convert from ADU/s to e-
    data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p)
    # convert NaN to zeros
    data0 = np.where(~np.isfinite(data), np.zeros_like(data), data)
    # resize image
    bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'],
                   ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'],
                   getshape=False)
    data2 = spirouImage.ResizeImage(p, data0, **bkwargs)
    # log change in data size
    WLOG(p, '', ('Image format changed to '
                            '{0}x{1}').format(*data2.shape))

    # ----------------------------------------------------------------------
    # Correct for the BADPIX mask (set all bad pixels to zero)
    # ----------------------------------------------------------------------
    p, data2 = spirouImage.CorrectForBadPix(p, data2, hdr)

    # ----------------------------------------------------------------------
    # Background computation
    # ----------------------------------------------------------------------
    if p['IC_DO_BKGR_SUBTRACTION']:
        # log that we are doing background measurement
        WLOG(p, '', 'Doing background measurement on raw frame')
        # get the bkgr measurement
        bargs = [p, data2, hdr]
        # background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs)
        p, background = spirouBACK.MeasureBackgroundMap(*bargs)
    else:
        background = np.zeros_like(data2)
        p['BKGRDFILE'] = 'None'
        p.set_source('BKGRDFILE', __NAME__ + '.main()')

    # correct data2 with background
    data2 = data2 - background

    # ----------------------------------------------------------------------
    # Log the number of dead pixels
    # ----------------------------------------------------------------------
    # get the number of bad pixels
    n_bad_pix = np.nansum(~np.isfinite(data2))
    n_bad_pix_frac = n_bad_pix * 100 / np.product(data2.shape)
    # Log number
    wmsg = 'Nb dead pixels = {0} / {1:.2f} %'
    WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))

    # ----------------------------------------------------------------------
    # Log the number of dead pixels
    # ----------------------------------------------------------------------
    loc = ParamDict()

    # ----------------------------------------------------------------------
    # Loop around fiber types
    # ----------------------------------------------------------------------
    # set fiber
    p['FIBER'] = p['FIB_TYP']
    # ------------------------------------------------------------------
    # Get localisation coefficients
    # ------------------------------------------------------------------
    # original there is a loop but it is not used --> removed
    p = spirouImage.FiberParams(p, p['FIBER'], merge=True)
    # get localisation fit coefficients
    p, loc = spirouLOCOR.GetCoeffs(p, hdr, loc)

    # ------------------------------------------------------------------
    # Calculating the tilt
    # ------------------------------------------------------------------
    # get the tilt by extracting the AB fibers and correlating them
    loc = spirouImage.GetTilt(p, loc, data2)

    # fit the tilt with a polynomial
    loc = spirouImage.FitTilt(p, loc)
    # log the tilt dispersion
    wmsg = 'Tilt dispersion = {0:.3f} deg'
    WLOG(p, 'info', wmsg.format(loc['RMS_TILT']))

    # ------------------------------------------------------------------
    # Plotting
    # ------------------------------------------------------------------
    if p['DRS_PLOT'] > 0:
        # plots setup: start interactive plot
        sPlt.start_interactive_session(p)
        # plot image with selected order shown
        sPlt.slit_sorder_plot(p, loc, data2)
        # plot slit tilt angle and fit
        sPlt.slit_tilt_angle_and_fit_plot(p, loc)
        # end interactive section
        sPlt.end_interactive_session(p)

    # ------------------------------------------------------------------
    # Replace tilt by the global fit
    # ------------------------------------------------------------------
    loc['TILT'] = loc['YFIT_TILT']
    oldsource = loc.get_source('tilt')
    loc.set_source('TILT', oldsource + '+{0}/main()'.format(__NAME__))

    # ----------------------------------------------------------------------
    # Quality control
    # ----------------------------------------------------------------------
    # set passed variable and fail message list
    passed, fail_msg = True, []
    qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
    # check that tilt rms is below required
    if loc['RMS_TILT'] > p['QC_SLIT_RMS']:
        # add failed message to fail message list
        fmsg = 'abnormal RMS of SLIT angle ({0:.2f} > {1:.2f} deg)'
        fail_msg.append(fmsg.format(loc['RMS_TILT'], p['QC_SLIT_RMS']))
        passed = False
        qc_pass.append(0)
    else:
        qc_pass.append(1)
    # add to qc header lists
    qc_values.append(loc['RMS_TILT'])
    qc_names.append('RMS_TILT')
    qc_logic.append('RMS_TILT > {0:.2f}'.format(p['QC_SLIT_RMS']))
    # ----------------------------------------------------------------------
    # check that tilt is less than max tilt required
    max_tilt = np.max(loc['TILT'])
    if max_tilt > p['QC_SLIT_MAX']:
        # add failed message to fail message list
        fmsg = 'abnormal SLIT angle ({0:.2f} > {1:.2f} deg)'
        fail_msg.append(fmsg.format(max_tilt, p['QC_SLIT_MAX']))
        passed = False
        qc_pass.append(0)
    else:
        qc_pass.append(1)
    # add to qc header lists
    qc_values.append(max_tilt)
    qc_names.append('max_tilt')
    qc_logic.append('max_tilt > {0:.2f}'.format(p['QC_SLIT_MAX']))
    # ----------------------------------------------------------------------
    # check that tilt is greater than min tilt required
    min_tilt = np.min(loc['TILT'])
    if min_tilt < p['QC_SLIT_MIN']:
        # add failed message to fail message list
        fmsg = 'abnormal SLIT angle ({0:.2f} < {1:.2f} deg)'
        fail_msg.append(fmsg.format(max_tilt, p['QC_SLIT_MIN']))
        passed = False
        qc_pass.append(0)
    else:
        qc_pass.append(1)
    # add to qc header lists
    qc_values.append(min_tilt)
    qc_names.append('min_tilt')
    qc_logic.append('min_tilt > {0:.2f}'.format(p['QC_SLIT_MIN']))
    # ----------------------------------------------------------------------
    # finally log the failed messages and set QC = 1 if we pass the
    # quality control QC = 0 if we fail quality control
    if passed:
        WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
        p['QC'] = 1
        p.set_source('QC', __NAME__ + '/main()')
    else:
        for farg in fail_msg:
            wmsg = 'QUALITY CONTROL FAILED: {0}'
            WLOG(p, 'warning', wmsg.format(farg))
        p['QC'] = 0
        p.set_source('QC', __NAME__ + '/main()')
    # store in qc_params
    qc_params = [qc_names, qc_values, qc_logic, qc_pass]

    # ----------------------------------------------------------------------
    # Save and record of tilt table
    # ----------------------------------------------------------------------
    # copy the tilt along the orders
    tiltima = np.ones((int(loc['NUMBER_ORDERS']/2), data2.shape[1]))
    tiltima *= loc['TILT'][:, None]
    # get the raw tilt file name
    raw_tilt_file = os.path.basename(p['FITSFILENAME'])
    # construct file name and path
    tiltfits, tag = spirouConfig.Constants.SLIT_TILT_FILE(p)
    tiltfitsname = os.path.basename(tiltfits)
    # Log that we are saving tilt file
    wmsg = 'Saving tilt information in file: {0}'
    WLOG(p, '', wmsg.format(tiltfitsname))
    # Copy keys from fits file
    hdict = spirouImage.CopyOriginalKeys(hdr)
    # add version number
    hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'],
                               value=p['DARKFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'],
                               value=p['BADPFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBACK'],
                               value=p['BKGRDFILE'])
    hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file',
                                     values=p['ARG_FILE_NAMES'])
    # add qc parameters
    # add qc parameters
    hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
    hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
    # add tilt parameters as 1d list
    hdict = spirouImage.AddKey1DList(p, hdict, p['KW_TILT'], values=loc['TILT'])
    # write tilt file to file
    p = spirouImage.WriteImage(p, tiltfits, tiltima, hdict)

    # ----------------------------------------------------------------------
    # Update the calibration data base
    # ----------------------------------------------------------------------
    if p['QC']:
        keydb = 'TILT'
        # copy localisation file to the calibDB folder
        spirouDB.PutCalibFile(p, tiltfits)
        # update the master calib DB file with new key
        spirouDB.UpdateCalibMaster(p, keydb, tiltfitsname, hdr)

    # ----------------------------------------------------------------------
    # End Message
    # ----------------------------------------------------------------------
    p = spirouStartup.End(p)
    # return a copy of locally defined variables in the memory
    return dict(locals())
def main(night_name=None, files=None, fiber_type=None, **kwargs):
    """
    cal_DRIFT_E2DS_spirou.py main function, if night_name and files are
    None uses arguments from run time i.e.:
        cal_DRIFT_E2DS_spirou.py [night_directory] [files]

    :param night_name: string or None, the folder within data raw directory
                                containing files (also reduced directory) i.e.
                                /data/raw/20170710 would be "20170710" but
                                /data/raw/AT5/20180409 would be "AT5/20180409"
    :param files: string, list or None, the list of files to use for
                  arg_file_names and fitsfilename
                  (if None assumes arg_file_names was set from run time)
    :param fiber_type: string, if None does all fiber types (defined in
                       constants_SPIROU FIBER_TYPES (default is AB, A, B, C
                       if defined then only does this fiber type (but must
                       be in FIBER_TYPES)
    :param kwargs: any keyword to overwrite constant in param dict "p"

    :return ll: dictionary, containing all the local variables defined in
                main
    """
    # ----------------------------------------------------------------------
    # Set up
    # ----------------------------------------------------------------------
    # get parameters from config files/run time args/load paths + calibdb
    p = spirouStartup.Begin(recipe=__NAME__)
    p = spirouStartup.LoadArguments(p, night_name, files)
    p = spirouStartup.InitialFileSetup(p, calibdb=True)
    # deal with fiber type
    if fiber_type is None:
        fiber_type = p['FIBER_TYPES']
    if type(fiber_type) == str:
        if fiber_type.upper() == 'ALL':
            fiber_type = p['FIBER_TYPES']
        elif fiber_type in p['FIBER_TYPES']:
            fiber_type = [fiber_type]
        else:
            emsg = 'fiber_type="{0}" not understood'
            WLOG(p, 'error', emsg.format(fiber_type))
    # set fiber type
    p['FIB_TYPE'] = fiber_type
    p.set_source('FIB_TYPE', __NAME__ + '__main__()')
    # Overwrite keys from source
    for kwarg in kwargs:
        p[kwarg] = kwargs[kwarg]

    # ----------------------------------------------------------------------
    # Read image file
    # ----------------------------------------------------------------------
    # read the image data
    p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add')

    # ----------------------------------------------------------------------
    # fix for un-preprocessed files
    # ----------------------------------------------------------------------
    data = spirouImage.FixNonPreProcess(p, data)

    # ----------------------------------------------------------------------
    # Get basic image properties
    # ----------------------------------------------------------------------
    # get sig det value
    p = spirouImage.GetSigdet(p, hdr, name='sigdet')
    # get exposure time
    p = spirouImage.GetExpTime(p, hdr, name='exptime')
    # get gain
    p = spirouImage.GetGain(p, hdr, name='gain')
    # set sigdet and conad keywords (sigdet is changed later)
    p['KW_CCD_SIGDET'][1] = p['SIGDET']
    p['KW_CCD_CONAD'][1] = p['GAIN']
    # now change the value of sigdet if require
    if p['IC_EXT_SIGDET'] > 0:
        p['SIGDET'] = float(p['IC_EXT_SIGDET'])
    # get DPRTYPE from header (Will have it if valid)
    p = spirouImage.ReadParam(p, hdr, 'KW_DPRTYPE', required=False, dtype=str)
    # check the DPRTYPE is not None
    if (p['DPRTYPE'] == 'None') or (['DPRTYPE'] is None):
        emsg1 = 'Error: {0} is not set in header for file {1}'
        eargs = [p['KW_DPRTYPE'][0], p['FITSFILENAME']]
        emsg2 = '\tPlease run pre-processing on file.'
        emsg3 = ('\tIf pre-processing fails or skips file, file is not '
                 'currrently as valid DRS fits file.')
        WLOG(p, 'error', [emsg1.format(*eargs), emsg2, emsg3])
    else:
        p['DPRTYPE'] = p['DPRTYPE'].strip()

    # ----------------------------------------------------------------------
    # Correction of DARK
    # ----------------------------------------------------------------------
    p, datac = spirouImage.CorrectForDark(p, data, hdr)

    # ----------------------------------------------------------------------
    # Resize image
    # ----------------------------------------------------------------------
    # rotate the image and convert from ADU/s to ADU
    data = spirouImage.ConvertToADU(spirouImage.FlipImage(p, datac), p=p)
    # convert NaN to zeros
    data0 = np.where(~np.isfinite(data), np.zeros_like(data), data)
    # resize image
    bkwargs = dict(xlow=p['IC_CCDX_LOW'],
                   xhigh=p['IC_CCDX_HIGH'],
                   ylow=p['IC_CCDY_LOW'],
                   yhigh=p['IC_CCDY_HIGH'],
                   getshape=False)
    data1 = spirouImage.ResizeImage(p, data0, **bkwargs)
    # log change in data size
    wmsg = 'Image format changed to {1}x{0}'
    WLOG(p, '', wmsg.format(*data1.shape))

    # ----------------------------------------------------------------------
    # Correct for the BADPIX mask (set all bad pixels to zero)
    # ----------------------------------------------------------------------
    p, data1 = spirouImage.CorrectForBadPix(p, data1, hdr)

    # ----------------------------------------------------------------------
    # Log the number of dead pixels
    # ----------------------------------------------------------------------
    # get the number of bad pixels
    n_bad_pix = np.sum(~np.isfinite(data1))
    n_bad_pix_frac = n_bad_pix * 100 / np.product(data1.shape)
    # Log number
    wmsg = 'Nb dead pixels = {0} / {1:.4f} %'
    WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))

    # ----------------------------------------------------------------------
    # Get the miny, maxy and max_signal for the central column
    # ----------------------------------------------------------------------
    # get the central column
    y = data1[p['IC_CENT_COL'], :]
    # get the min max and max signal using box smoothed approach
    miny, maxy, max_signal, diff_maxmin = spirouBACK.MeasureMinMaxSignal(p, y)
    # Log max average flux/pixel
    wmsg = 'Maximum average flux/pixel in the spectrum: {0:.1f} [ADU]'
    WLOG(p, 'info', wmsg.format(max_signal / p['NBFRAMES']))

    # ----------------------------------------------------------------------
    # Background computation
    # ----------------------------------------------------------------------
    if p['IC_DO_BKGR_SUBTRACTION']:
        # log that we are doing background measurement
        WLOG(p, '', 'Doing background measurement on raw frame')
        # get the bkgr measurement
        bargs = [p, data1, hdr]
        # background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs)
        p, background = spirouBACK.MeasureBackgroundMap(*bargs)
    else:
        background = np.zeros_like(data1)
        p['BKGRDFILE'] = 'None'
        p.set_source('BKGRDFILE', __NAME__ + '.main()')
    # apply background correction to data (and set to zero where negative)
    data1 = data1 - background

    # ----------------------------------------------------------------------
    # Read tilt slit angle
    # ----------------------------------------------------------------------
    # define loc storage parameter dictionary
    loc = ParamDict()
    # get tilts (if the mode requires it)
    if p['IC_EXTRACT_TYPE'] not in EXTRACT_SHAPE_TYPES:
        p, loc['TILT'] = spirouImage.ReadTiltFile(p, hdr)
        loc.set_source('TILT',
                       __NAME__ + '/main() + /spirouImage.ReadTiltFile')
    else:
        loc['TILT'] = None
        loc.set_source('TILT', __NAME__ + '/main()')

    # ----------------------------------------------------------------------
    #  Earth Velocity calculation
    # ----------------------------------------------------------------------
    if p['IC_IMAGE_TYPE'] == 'H4RG':
        p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, hdr)

    # ----------------------------------------------------------------------
    # Get all fiber data (for all fibers)
    # ----------------------------------------------------------------------
    # TODO: This is temp solution for options 5a and 5b
    loc_fibers = spirouLOCOR.GetFiberData(p, hdr)

    # ------------------------------------------------------------------
    # Deal with debananafication
    # ------------------------------------------------------------------
    # if mode 4a or 4b we need to straighten in x only
    if p['IC_EXTRACT_TYPE'] in ['4a', '4b']:
        # get the shape parameters
        p, shapem_x = spirouImage.GetShapeX(p, hdr)
        p, shape_local = spirouImage.GetShapeLocal(p, hdr)
        # log progress
        WLOG(p, '', 'Debananafying (straightening) image')
        # apply shape transforms
        targs = dict(lin_transform_vect=shape_local, dxmap=shapem_x)
        data2 = spirouImage.EATransform(data1, **targs)

    # if mode 5a or 5b we need to straighten in x and y using the
    #     polynomial fits for location
    elif p['IC_EXTRACT_TYPE'] in ['5a', '5b']:
        # get the shape parameters
        p, shapem_x = spirouImage.GetShapeX(p, hdr)
        p, shapem_y = spirouImage.GetShapeY(p, hdr)
        p, shape_local = spirouImage.GetShapeLocal(p, hdr)
        p, fpmaster = spirouImage.GetFPMaster(p, hdr)
        # get the bad pixel map
        bkwargs = dict(return_map=True, quiet=True)
        p, badpix = spirouImage.CorrectForBadPix(p, data1, hdr, **bkwargs)
        # log progress
        WLOG(p, '', 'Cleaning image')
        # clean the image
        data1 = spirouEXTOR.CleanHotpix(data1, badpix)
        # log progress
        WLOG(p, '', 'Debananafying (straightening) image')
        # apply shape transforms
        targs = dict(lin_transform_vect=shape_local,
                     dxmap=shapem_x,
                     dymap=shapem_y)
        data2 = spirouImage.EATransform(data1, **targs)
    # in any other mode we do not straighten
    else:
        data2 = np.array(data1)

    # ----------------------------------------------------------------------
    # Fiber loop
    # ----------------------------------------------------------------------
    # loop around fiber types
    for fiber in p['FIB_TYPE']:
        # set fiber
        p['FIBER'] = fiber
        p.set_source('FIBER', __NAME__ + '/main()()')

        # ------------------------------------------------------------------
        # Read wavelength solution
        # ------------------------------------------------------------------
        # set source of wave file
        wsource = __NAME__ + '/main() + /spirouImage.GetWaveSolution'
        # Force A and B to AB solution
        if fiber in ['A', 'B']:
            wave_fiber = 'AB'
        else:
            wave_fiber = fiber
        # get wave image
        wkwargs = dict(hdr=hdr,
                       return_wavemap=True,
                       return_filename=True,
                       return_header=True,
                       fiber=wave_fiber)
        wout = spirouImage.GetWaveSolution(p, **wkwargs)
        loc['WAVEPARAMS'], loc['WAVE'], loc['WAVEFILE'] = wout[:3]
        loc['WAVEHDR'], loc['WSOURCE'] = wout[3:]
        source_names = ['WAVE', 'WAVEFILE', 'WAVEPARAMS', 'WAVEHDR']
        loc.set_sources(source_names, wsource)
        # get dates
        loc['WAVE_ACQTIMES'] = spirouDB.GetTimes(p, loc['WAVEHDR'])
        loc.set_source('WAVE_ACQTIMES', __NAME__ + '.main()')
        # get the recipe that produced the wave solution
        if 'WAVECODE' in loc['WAVEHDR']:
            loc['WAVE_CODE'] = loc['WAVEHDR']['WAVECODE']
        else:
            loc['WAVE_CODE'] = 'UNKNOWN'
        loc.set_source('WAVE_CODE', __NAME__ + '.main()')

        # ----------------------------------------------------------------------
        # Get WFP keys
        # ----------------------------------------------------------------------
        # Read the WFP keys - if they don't exist set to None and deal
        #    with later
        p = spirouImage.ReadParam(p,
                                  loc['WAVEHDR'],
                                  'KW_WFP_DRIFT',
                                  name='WFP_DRIFT',
                                  required=False)
        p = spirouImage.ReadParam(p,
                                  loc['WAVEHDR'],
                                  'KW_WFP_FWHM',
                                  name='WFP_FWHM',
                                  required=False)
        p = spirouImage.ReadParam(p,
                                  loc['WAVEHDR'],
                                  'KW_WFP_CONTRAST',
                                  name='WFP_CONTRAST',
                                  required=False)
        p = spirouImage.ReadParam(p,
                                  loc['WAVEHDR'],
                                  'KW_WFP_MAXCPP',
                                  name='WFP_MAXCPP',
                                  required=False)
        p = spirouImage.ReadParam(p,
                                  loc['WAVEHDR'],
                                  'KW_WFP_MASK',
                                  name='WFP_MASK',
                                  required=False)
        p = spirouImage.ReadParam(p,
                                  loc['WAVEHDR'],
                                  'KW_WFP_LINES',
                                  name='WFP_LINES',
                                  required=False)
        p = spirouImage.ReadParam(p,
                                  loc['WAVEHDR'],
                                  'KW_WFP_TARG_RV',
                                  name='WFP_TARG_RV',
                                  required=False)
        p = spirouImage.ReadParam(p,
                                  loc['WAVEHDR'],
                                  'KW_WFP_WIDTH',
                                  name='WFP_WIDTH',
                                  required=False)
        p = spirouImage.ReadParam(p,
                                  loc['WAVEHDR'],
                                  'KW_WFP_STEP',
                                  name='WFP_STEP',
                                  required=False)

        # ----------------------------------------------------------------------
        # Read Flat file
        # ----------------------------------------------------------------------
        fout = spirouImage.ReadFlatFile(p, hdr, return_header=True)
        p, loc['FLAT'], flathdr = fout
        loc.set_source('FLAT',
                       __NAME__ + '/main() + /spirouImage.ReadFlatFile')
        # get flat extraction mode
        if p['KW_E2DS_EXTM'][0] in flathdr:
            flat_ext_mode = flathdr[p['KW_E2DS_EXTM'][0]]
        else:
            flat_ext_mode = None

        # ------------------------------------------------------------------
        # Check extraction method is same as flat extraction method
        # ------------------------------------------------------------------
        # get extraction method and function
        extmethod, extfunc = spirouEXTOR.GetExtMethod(p, p['IC_EXTRACT_TYPE'])
        if not DEBUG:
            # compare flat extraction mode to extraction mode
            spirouEXTOR.CompareExtMethod(p, flat_ext_mode, extmethod, 'FLAT',
                                         'EXTRACTION')
        # ------------------------------------------------------------------
        # Read Blaze file
        # ------------------------------------------------------------------
        p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hdr)
        blazesource = __NAME__ + '/main() + /spirouImage.ReadBlazeFile'
        loc.set_source('BLAZE', blazesource)

        # ------------------------------------------------------------------
        # Get fiber specific parameters from loc_fibers
        # ------------------------------------------------------------------
        # get this fibers parameters
        p = spirouImage.FiberParams(p, p['FIBER'], merge=True)
        # get localisation parameters
        for key in loc_fibers[fiber]:
            loc[key] = loc_fibers[fiber][key]
            loc.set_source(key, loc_fibers[fiber].sources[key])
        # get locofile source
        p['LOCOFILE'] = loc['LOCOFILE']
        p.set_source('LOCOFILE', loc.sources['LOCOFILE'])
        # get the order_profile
        order_profile = loc_fibers[fiber]['ORDER_PROFILE']

        # ------------------------------------------------------------------
        # Set up Extract storage
        # ------------------------------------------------------------------
        # Create array to store extraction (for each order and each pixel
        # along order)
        loc['E2DS'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
        loc['E2DSFF'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
        loc['E2DSLL'] = []
        loc['SPE1'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
        loc['SPE3'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
        loc['SPE4'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
        loc['SPE5'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
        # Create array to store the signal to noise ratios for each order
        loc['SNR'] = np.zeros(loc['NUMBER_ORDERS'])

        # ------------------------------------------------------------------
        # Extract orders
        # ------------------------------------------------------------------
        # source for parameter dictionary
        source = __NAME__ + '/main()'
        # get limits of order extraction
        valid_orders = spirouEXTOR.GetValidOrders(p, loc)
        # loop around each order
        for order_num in valid_orders:
            # -------------------------------------------------------------
            # IC_EXTRACT_TYPE decides the extraction routine
            # -------------------------------------------------------------
            eargs = [p, loc, data2, order_num]
            ekwargs = dict(mode=p['IC_EXTRACT_TYPE'],
                           order_profile=order_profile)
            with warnings.catch_warnings(record=True) as w:
                eout = spirouEXTOR.Extraction(*eargs, **ekwargs)
            # deal with different return
            if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES:
                e2ds, e2dsll, cpt = eout
            else:
                e2ds, cpt = eout
                e2dsll = None
            # -------------------------------------------------------------
            # calculate the noise
            range1, range2 = p['IC_EXT_RANGE1'], p['IC_EXT_RANGE2']
            # set the noise
            noise = p['SIGDET'] * np.sqrt(range1 + range2)
            # get window size
            blaze_win1 = int(data2.shape[0] / 2) - p['IC_EXTFBLAZ']
            blaze_win2 = int(data2.shape[0] / 2) + p['IC_EXTFBLAZ']
            # get average flux per pixel
            flux = np.nansum(
                e2ds[blaze_win1:blaze_win2]) / (2 * p['IC_EXTFBLAZ'])
            # calculate signal to noise ratio = flux/sqrt(flux + noise^2)
            snr = flux / np.sqrt(flux + noise**2)
            # log the SNR RMS
            wmsg = 'On fiber {0} order {1}: S/N= {2:.1f} Nbcosmic= {3}'
            wargs = [p['FIBER'], order_num, snr, cpt]
            WLOG(p, '', wmsg.format(*wargs))
            # add calculations to storage
            loc['E2DS'][order_num] = e2ds
            loc['E2DSFF'][order_num] = e2ds / loc['FLAT'][order_num]
            loc['SNR'][order_num] = snr
            # save the longfile
            if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES:
                loc['E2DSLL'].append(e2dsll)
            # set sources
            loc.set_sources(['e2ds', 'SNR'], source)
            # Log if saturation level reached
            satvalue = (flux / p['GAIN']) / (range1 + range2)
            if satvalue > (p['QC_LOC_FLUMAX'] * p['NBFRAMES']):
                wmsg = 'SATURATION LEVEL REACHED on Fiber {0} order={1}'
                WLOG(p, 'warning', wmsg.format(fiber, order_num))

        # ------------------------------------------------------------------
        # Thermal correction
        # ------------------------------------------------------------------
        # get fiber type
        if fiber in ['AB', 'A', 'B']:
            fibertype = p['DPRTYPE'].split('_')[0]
        else:
            fibertype = p['DPRTYPE'].split('_')[1]

        # apply thermal correction based on fiber type
        if fibertype in p['THERMAL_CORRECTION_TYPE1']:
            # log progress
            wmsg = 'Correcting thermal background for {0}={1} mode={2}'
            wargs = [fiber, fibertype, 1]
            WLOG(p, 'info', wmsg.format(*wargs))
            # correct E2DS
            tkwargs = dict(image=loc['E2DS'], mode=1, fiber=fiber, hdr=hdr)
            p, loc['E2DS'] = spirouBACK.ThermalCorrect(p, **tkwargs)
            # correct E2DSFF
            tkwargs = dict(image=loc['E2DSFF'],
                           mode=1,
                           fiber=fiber,
                           hdr=hdr,
                           flat=loc['FLAT'])
            p, loc['E2DSFF'] = spirouBACK.ThermalCorrect(p, **tkwargs)
        elif fibertype in p['THERMAL_CORRECTION_TYPE2']:
            # log progress
            wmsg = 'Correcting thermal background for {0}={1} mode={2}'
            wargs = [fiber, fibertype, 2]
            WLOG(p, 'info', wmsg.format(*wargs))
            # correct E2DS
            tkwargs = dict(image=loc['E2DS'], mode=2, fiber=fiber, hdr=hdr)
            p, loc['E2DS'] = spirouBACK.ThermalCorrect(p, **tkwargs)
            # correct E2DSFF
            tkwargs = dict(image=loc['E2DSFF'],
                           mode=2,
                           fiber=fiber,
                           hdr=hdr,
                           flat=loc['FLAT'])
            p, loc['E2DSFF'] = spirouBACK.ThermalCorrect(p, **tkwargs)
        else:
            # log progress
            wmsg = 'Not correcting thermal background for {0}={1}'
            wargs = [fiber, fibertype]
            WLOG(p, 'info', wmsg.format(*wargs))
            # set filename for output
            outfile = 'THERMALFILE_{0}'.format(fiber)
            p[outfile] = 'None'
            p.set_source(outfile, __NAME__ + '.main()')

        # ------------------------------------------------------------------
        # Plots
        # ------------------------------------------------------------------
        if p['DRS_PLOT'] > 0:
            # start interactive session if needed
            sPlt.start_interactive_session(p)
            # plot all orders or one order
            if p['IC_FF_PLOT_ALL_ORDERS']:
                # plot image with all order fits (slower)
                sPlt.ext_aorder_fit(p, loc, data1, max_signal / 10.)
            else:
                # plot image with selected order fit and edge fit (faster)
                sPlt.ext_sorder_fit(p, loc, data1, max_signal / 10.)
            # plot e2ds against wavelength
            sPlt.ext_spectral_order_plot(p, loc)

            if p['IC_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES:
                sPlt.ext_debanana_plot(p, loc, data2, max_signal / 10.)

        # ----------------------------------------------------------------------
        # Quality control
        # ----------------------------------------------------------------------
        passed, fail_msg = True, []
        qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
        # ----------------------------------------------------------------------
        # if array is completely NaNs it shouldn't pass
        if np.sum(np.isfinite(loc['E2DS'])) == 0:
            fail_msg.append('E2DS image is all NaNs')
            passed = False
            qc_pass.append(0)
        else:
            qc_pass.append(1)
        # add to qc header lists
        qc_values.append('NaN')
        qc_names.append('image')
        qc_logic.append('image is all NaN')
        # ----------------------------------------------------------------------
        # saturation check: check that the max_signal is lower than
        # qc_max_signal
        max_qcflux = p['QC_MAX_SIGNAL'] * p['NBFRAMES']
        if max_signal > max_qcflux:
            fmsg = 'Too much flux in the image ({0:.2f} > {1:.2f})'
            fail_msg.append(fmsg.format(max_signal, max_qcflux))
            passed = False
            # Question: Why is this test ignored?
            # For some reason this test is ignored in old code
            passed = True
            WLOG(p, 'info', fail_msg[-1])
            qc_pass.append(0)
        else:
            qc_pass.append(1)

        # add to qc header lists
        qc_values.append(max_signal)
        qc_names.append('max_signal')
        qc_logic.append('QC_MAX_SIGNAL > {0:.3f}'.format(max_qcflux))

        # finally log the failed messages and set QC = 1 if we pass the
        # quality control QC = 0 if we fail quality control
        if passed:
            WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
            p['QC'] = 1
            p.set_source('QC', __NAME__ + '/main()')
        else:
            for farg in fail_msg:
                wmsg = 'QUALITY CONTROL FAILED: {0}'
                WLOG(p, 'warning', wmsg.format(farg))
            p['QC'] = 0
            p.set_source('QC', __NAME__ + '/main()')
        # store in qc_params
        qc_params = [qc_names, qc_values, qc_logic, qc_pass]

        # ------------------------------------------------------------------
        # Store extraction in file(s)
        # ------------------------------------------------------------------
        raw_ext_file = os.path.basename(p['FITSFILENAME'])
        # construct filename
        e2dsfits, tag1 = spirouConfig.Constants.EXTRACT_E2DS_FILE(p)
        e2dsfitsname = os.path.split(e2dsfits)[-1]
        e2dsfffits, tag2 = spirouConfig.Constants.EXTRACT_E2DSFF_FILE(p)
        e2dsfffitsname = os.path.split(e2dsfffits)[-1]
        e2dsllfits, tag4 = spirouConfig.Constants.EXTRACT_E2DSLL_FILE(p)
        e2dsfllitsname = os.path.split(e2dsllfits)[-1]
        # log that we are saving E2DS spectrum
        wmsg = 'Saving E2DS spectrum of Fiber {0} in {1}'
        WLOG(p, '', wmsg.format(p['FIBER'], e2dsfitsname))
        wmsg = 'Saving E2DSFF spectrum of Fiber {0} in {1}'
        WLOG(p, '', wmsg.format(p['FIBER'], e2dsfffitsname))
        # add keys from original header file
        hdict = spirouImage.CopyOriginalKeys(hdr)
        # set the version
        hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_DRS_DATE'],
                                   value=p['DRS_DATE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_DATE_NOW'],
                                   value=p['DATE_NOW'])
        hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
        hdict = spirouImage.AddKey(p, hdict, p['KW_FIBER'], value=p['FIBER'])

        # set the input files
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_CDBDARK'],
                                   value=p['DARKFILE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_CDBBAD'],
                                   value=p['BADPFILE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_CDBLOCO'],
                                   value=p['LOCOFILE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_CDBBACK'],
                                   value=p['BKGRDFILE'])
        if p['IC_EXTRACT_TYPE'] not in EXTRACT_SHAPE_TYPES:
            hdict = spirouImage.AddKey(p,
                                       hdict,
                                       p['KW_CDBTILT'],
                                       value=p['TILTFILE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_CDBBLAZE'],
                                   value=p['BLAZFILE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_CDBFLAT'],
                                   value=p['FLATFILE'])
        if p['IC_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES:
            hdict = spirouImage.AddKey(p,
                                       hdict,
                                       p['KW_CDBSHAPEX'],
                                       value=p['SHAPEXFILE'])
            hdict = spirouImage.AddKey(p,
                                       hdict,
                                       p['KW_CDBSHAPEY'],
                                       value=p['SHAPEYFILE'])
            hdict = spirouImage.AddKey(p,
                                       hdict,
                                       p['KW_CDBSHAPE'],
                                       value=p['SHAPEFILE'])
            hdict = spirouImage.AddKey(p,
                                       hdict,
                                       p['KW_CDBFPMASTER'],
                                       value=p['FPMASTERFILE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_CDBTHERMAL'],
                                   value=p['THERMALFILE_{0}'.format(fiber)])

        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_CDBWAVE'],
                                   value=loc['WAVEFILE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_WAVESOURCE'],
                                   value=loc['WSOURCE'])
        hdict = spirouImage.AddKey1DList(p,
                                         hdict,
                                         p['KW_INFILE1'],
                                         dim1name='file',
                                         values=p['ARG_FILE_NAMES'])
        # construct loco filename
        locofile, _ = spirouConfig.Constants.EXTRACT_LOCO_FILE(p)
        locofilename = os.path.basename(locofile)
        # add barycentric keys to header
        hdict = spirouImage.AddKey(p, hdict, p['KW_BERV'], value=loc['BERV'])
        hdict = spirouImage.AddKey(p, hdict, p['KW_BJD'], value=loc['BJD'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_BERV_MAX'],
                                   value=loc['BERV_MAX'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_B_OBS_HOUR'],
                                   value=loc['BERVHOUR'])
        # add barycentric estimate keys to header
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_BERV_EST'],
                                   value=loc['BERV_EST'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_BJD_EST'],
                                   value=loc['BJD_EST'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_BERV_MAX_EST'],
                                   value=loc['BERV_MAX_EST'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_BERV_SOURCE'],
                                   value=loc['BERV_SOURCE'])
        # add qc parameters
        hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
        hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
        # copy extraction method and function to header
        #     (for reproducibility)
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_E2DS_EXTM'],
                                   value=extmethod)
        hdict = spirouImage.AddKey(p, hdict, p['KW_E2DS_FUNC'], value=extfunc)
        # add localization file name to header
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_LOCO_FILE'],
                                   value=locofilename)
        # add wave solution date
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_WAVE_TIME1'],
                                   value=loc['WAVE_ACQTIMES'][0])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_WAVE_TIME2'],
                                   value=loc['WAVE_ACQTIMES'][1])
        # add wave solution number of orders
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_WAVE_ORD_N'],
                                   value=loc['WAVEPARAMS'].shape[0])
        # add wave solution degree of fit
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_WAVE_LL_DEG'],
                                   value=loc['WAVEPARAMS'].shape[1] - 1)
        # -------------------------------------------------------------------------
        # add keys of the wave solution FP CCF
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_WFP_FILE'],
                                   value=loc['WAVEFILE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_WFP_DRIFT'],
                                   value=p['WFP_DRIFT'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_WFP_FWHM'],
                                   value=p['WFP_FWHM'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_WFP_CONTRAST'],
                                   value=p['WFP_CONTRAST'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_WFP_MAXCPP'],
                                   value=p['WFP_MAXCPP'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_WFP_MASK'],
                                   value=p['WFP_MASK'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_WFP_LINES'],
                                   value=p['WFP_LINES'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_WFP_TARG_RV'],
                                   value=p['WFP_TARG_RV'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_WFP_WIDTH'],
                                   value=p['WFP_WIDTH'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_WFP_STEP'],
                                   value=p['WFP_STEP'])

        # write 1D list of the SNR
        hdict = spirouImage.AddKey1DList(p,
                                         hdict,
                                         p['KW_E2DS_SNR'],
                                         values=loc['SNR'])
        # add localization file keys to header
        root = p['KW_ROOT_DRS_LOC'][0]
        hdict = spirouImage.CopyRootKeys(p, hdict, locofile, root=root)
        # add wave solution coefficients
        hdict = spirouImage.AddKey2DList(p,
                                         hdict,
                                         p['KW_WAVE_PARAM'],
                                         values=loc['WAVEPARAMS'])
        # Save E2DS file
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_EXT_TYPE'],
                                   value=p['DPRTYPE'])
        p = spirouImage.WriteImage(p, e2dsfits, loc['E2DS'], hdict)
        # Save E2DSFF file
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_EXT_TYPE'],
                                   value=p['DPRTYPE'])
        p = spirouImage.WriteImage(p, e2dsfffits, loc['E2DSFF'], hdict)
        # Save E2DSLL file
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_EXT_TYPE'],
                                   value=p['DPRTYPE'])
        if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES:
            llstack = np.vstack(loc['E2DSLL'])
            p = spirouImage.WriteImage(p, e2dsllfits, llstack, hdict)

        # ------------------------------------------------------------------
        # 1-dimension spectral S1D (uniform in wavelength)
        # ------------------------------------------------------------------
        # get arguments for E2DS to S1D
        e2dsargs = [loc['WAVE'], loc['E2DSFF'], loc['BLAZE']]
        # get 1D spectrum
        xs1d1, ys1d1 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='wave')
        # Plot the 1D spectrum
        if p['DRS_PLOT'] > 0:
            sPlt.ext_1d_spectrum_plot(p, xs1d1, ys1d1)
        # construct file name
        s1dfile1, tag3 = spirouConfig.Constants.EXTRACT_S1D_FILE1(p)
        s1dfilename1 = os.path.basename(s1dfile1)
        # add header keys
        # set the version
        hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_DRS_DATE'],
                                   value=p['DRS_DATE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_DATE_NOW'],
                                   value=p['DATE_NOW'])
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_EXT_TYPE'],
                                   value=p['DPRTYPE'])
        # log writing to file
        wmsg = 'Saving 1D spectrum (uniform in wavelength) for Fiber {0} in {1}'
        WLOG(p, '', wmsg.format(p['FIBER'], s1dfilename1))
        # Write to file
        columns = ['wavelength', 'flux', 'eflux']
        values = [xs1d1, ys1d1, np.zeros_like(ys1d1)]
        units = ['nm', None, None]
        s1d1 = spirouImage.MakeTable(p, columns, values, units=units)
        spirouImage.WriteTable(p, s1d1, s1dfile1, header=hdict)

        # ------------------------------------------------------------------
        # 1-dimension spectral S1D (uniform in velocity)
        # ------------------------------------------------------------------
        # get arguments for E2DS to S1D
        e2dsargs = [loc['WAVE'], loc['E2DSFF'], loc['BLAZE']]
        # get 1D spectrum
        xs1d2, ys1d2 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='velocity')
        # Plot the 1D spectrum
        if p['DRS_PLOT'] > 0:
            sPlt.ext_1d_spectrum_plot(p, xs1d2, ys1d2)
        # construct file name
        s1dfile2, tag4 = spirouConfig.Constants.EXTRACT_S1D_FILE2(p)
        s1dfilename2 = os.path.basename(s1dfile2)
        # add header keys
        hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_DRS_DATE'],
                                   value=p['DRS_DATE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_DATE_NOW'],
                                   value=p['DATE_NOW'])
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_EXT_TYPE'],
                                   value=p['DPRTYPE'])
        # log writing to file
        wmsg = 'Saving 1D spectrum (uniform in velocity) for Fiber {0} in {1}'
        WLOG(p, '', wmsg.format(p['FIBER'], s1dfilename2))
        # Write to file
        columns = ['wavelength', 'flux', 'eflux']
        values = [xs1d2, ys1d2, np.zeros_like(ys1d2)]
        units = ['nm', None, None]
        s1d2 = spirouImage.MakeTable(p, columns, values, units=units)
        spirouImage.WriteTable(p, s1d2, s1dfile2, header=hdict)

    # ----------------------------------------------------------------------
    # End Message
    # ----------------------------------------------------------------------
    p = spirouStartup.End(p)
    # return a copy of locally defined variables in the memory
    return dict(locals())
def main(night_name=None, files=None):
    """
    cal_loc_RAW_spirou.py main function, if night_name and files are None uses
    arguments from run time i.e.:
        cal_loc_RAW_spirou.py [night_name] [files]

    :param night_name: string or None, the folder within data raw directory
                                containing files (also reduced directory) i.e.
                                /data/raw/20170710 would be "20170710" but
                                /data/raw/AT5/20180409 would be "AT5/20180409"
    :param files: string, list or None, the list of files to use for
                  arg_file_names and fitsfilename
                  (if None assumes arg_file_names was set from run time)

    :return ll: dictionary, containing all the local variables defined in
                main
    """
    # ----------------------------------------------------------------------
    # Set up
    # ----------------------------------------------------------------------
    # get parameters from config files/run time args/load paths + calibdb
    p = spirouStartup.Begin(recipe=__NAME__)
    p = spirouStartup.LoadArguments(p, night_name, files)
    p = spirouStartup.InitialFileSetup(p, calibdb=True)

    # ----------------------------------------------------------------------
    # Read image file
    # ----------------------------------------------------------------------
    # read the image data
    p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add')

    # ----------------------------------------------------------------------
    # fix for un-preprocessed files
    # ----------------------------------------------------------------------
    data = spirouImage.FixNonPreProcess(p, data)

    # ----------------------------------------------------------------------
    # Get basic image properties
    # ----------------------------------------------------------------------
    # get sig det value
    p = spirouImage.GetSigdet(p, hdr, name='sigdet')
    # get exposure time
    p = spirouImage.GetExpTime(p, hdr, name='exptime')
    # get gain
    p = spirouImage.GetGain(p, hdr, name='gain')

    # ----------------------------------------------------------------------
    # Correction of DARK
    # ----------------------------------------------------------------------
    p, datac = spirouImage.CorrectForDark(p, data, hdr)

    # ----------------------------------------------------------------------
    #  Interpolation over bad regions (to fill in the holes)
    # ----------------------------------------------------------------------
    # log process
    # wmsg = 'Interpolating over bad regions'
    # WLOG(p, '', wmsg)
    # run interpolation
    # datac = spirouImage.InterpolateBadRegions(p, datac)

    # ----------------------------------------------------------------------
    # Resize image
    # ----------------------------------------------------------------------
    # rotate the image and convert from ADU/s to e-
    data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p)
    # convert NaN to zeros
    data0 = np.where(~np.isfinite(data), np.zeros_like(data), data)
    # resize image
    bkwargs = dict(xlow=p['IC_CCDX_LOW'],
                   xhigh=p['IC_CCDX_HIGH'],
                   ylow=p['IC_CCDY_LOW'],
                   yhigh=p['IC_CCDY_HIGH'],
                   getshape=False)
    data2 = spirouImage.ResizeImage(p, data0, **bkwargs)
    # log change in data size
    WLOG(p, '', ('Image format changed to ' '{0}x{1}').format(*data2.shape))

    # ----------------------------------------------------------------------
    # Correct for the BADPIX mask (set all bad pixels to zero)
    # ----------------------------------------------------------------------
    p, data2 = spirouImage.CorrectForBadPix(p, data2, hdr)

    # ----------------------------------------------------------------------
    # Background computation
    # ----------------------------------------------------------------------
    if p['IC_DO_BKGR_SUBTRACTION']:
        # log that we are doing background measurement
        WLOG(p, '', 'Doing background measurement on raw frame')
        # get the bkgr measurement
        bargs = [p, data2, hdr]
        # background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs)
        p, background = spirouBACK.MeasureBackgroundMap(*bargs)
    else:
        background = np.zeros_like(data2)
        p['BKGRDFILE'] = 'None'
        p.set_source('BKGRDFILE', __NAME__ + '.main()')
    # apply background correction to data
    data2 = data2 - background

    # ----------------------------------------------------------------------
    # Construct image order_profile
    # ----------------------------------------------------------------------
    # log that we are doing background measurement
    WLOG(p, '', 'Creating Order Profile')
    order_profile = spirouLOCOR.BoxSmoothedImage(data2, p['LOC_BOX_SIZE'])
    # data 2 is now set to the order profile
    data2o = data2.copy()
    data2 = order_profile.copy()

    # ----------------------------------------------------------------------
    # Write image order_profile to file
    # ----------------------------------------------------------------------
    # Construct folder and filename
    rawfits, tag1 = spirouConfig.Constants.LOC_ORDER_PROFILE_FILE(p)
    rawfitsname = os.path.split(rawfits)[-1]
    # log saving order profile
    wmsg = 'Saving processed raw frame in {0}'
    WLOG(p, '', wmsg.format(rawfitsname))
    # add keys from original header file
    hdict = spirouImage.CopyOriginalKeys(hdr)
    hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
    # write to file
    p = spirouImage.WriteImage(p, rawfits, order_profile, hdict)

    # ----------------------------------------------------------------------
    # Move order_profile to calibDB and update calibDB
    # ----------------------------------------------------------------------
    # set key for calibDB
    keydb = 'ORDER_PROFILE_{0}'.format(p['FIBER'])
    # copy dark fits file to the calibDB folder
    spirouDB.PutCalibFile(p, rawfits)
    # update the master calib DB file with new key
    spirouDB.UpdateCalibMaster(p, keydb, rawfitsname, hdr)

    # ######################################################################
    # Localization of orders on central column
    # ######################################################################
    # storage dictionary for localization parameters
    loc = ParamDict()
    # Plots setup: start interactive plot
    if p['DRS_PLOT'] > 0:
        sPlt.start_interactive_session(p)
    # ----------------------------------------------------------------------
    # Measurement and correction of background on the central column
    # ----------------------------------------------------------------------
    loc = spirouBACK.MeasureBkgrdGetCentPixs(p, loc, data2)
    # ----------------------------------------------------------------------
    # Search for order center on the central column - quick estimation
    # ----------------------------------------------------------------------
    # log progress
    WLOG(p, '', 'Searching order center on central column')
    # plot the minimum of ycc and ic_locseuil if in debug and plot mode
    if p['DRS_DEBUG'] > 0 and p['DRS_PLOT']:
        sPlt.debug_locplot_min_ycc_loc_threshold(p, loc['YCC'])
    # find the central positions of the orders in the central
    posc_all = spirouLOCOR.FindPosCentCol(loc['YCC'], p['IC_LOCSEUIL'])
    # depending on the fiber type we may need to skip some pixels and also
    # we need to add back on the ic_offset applied
    start = p['IC_FIRST_ORDER_JUMP']
    posc = posc_all[start:] + p['IC_OFFSET']
    # work out the number of orders to use (minimum of ic_locnbmaxo and number
    #    of orders found in 'LocateCentralOrderPositions')
    number_of_orders = np.min([p['IC_LOCNBMAXO'], len(posc)])
    # log the number of orders than have been detected
    wargs = [p['FIBER'], int(number_of_orders / p['NBFIB']), p['NBFIB']]
    WLOG(p, 'info', ('On fiber {0} {1} orders have been detected '
                     'on {2} fiber(s)').format(*wargs))

    # ----------------------------------------------------------------------
    # Search for order center and profile on specific columns
    # ----------------------------------------------------------------------
    # get fit polynomial orders for position and width
    fitpos, fitwid = p['IC_LOCDFITC'], p['IC_LOCDFITW']
    # Create arrays to store position and width of order for each order
    loc['CTRO'] = np.zeros((number_of_orders, data2.shape[1]), dtype=float)
    loc['SIGO'] = np.zeros((number_of_orders, data2.shape[1]), dtype=float)
    # Create arrays to store coefficients for position and width
    loc['ACC'] = np.zeros((number_of_orders, fitpos + 1))
    loc['ASS'] = np.zeros((number_of_orders, fitpos + 1))
    # Create arrays to store rms values for position and width
    loc['RMS_CENTER'] = np.zeros(number_of_orders)
    loc['RMS_FWHM'] = np.zeros(number_of_orders)
    # Create arrays to store point to point max value for position and width
    loc['MAX_PTP_CENTER'] = np.zeros(number_of_orders)
    loc['MAX_PTP_FRACCENTER'] = np.zeros(number_of_orders)
    loc['MAX_PTP_FWHM'] = np.zeros(number_of_orders)
    loc['MAX_PTP_FRACFWHM'] = np.zeros(number_of_orders)
    # Create arrays to store rejected points
    loc['MAX_RMPTS_POS'] = np.zeros(number_of_orders)
    loc['MAX_RMPTS_WID'] = np.zeros(number_of_orders)
    # set the central col centers in the cpos_orders array
    loc['CTRO'][:, p['IC_CENT_COL']] = posc[0:number_of_orders]
    # set source for all locs
    loc.set_all_sources(__NAME__ + '/main()')
    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # storage for plotting
    loc['XPLOT'], loc['YPLOT'] = [], []
    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    # loop around each order
    rorder_num = 0
    for order_num in range(number_of_orders):
        # find the row centers of the columns
        loc = spirouLOCOR.FindOrderCtrs(p, data2, loc, order_num)
        # only keep the orders with non-zero width
        mask = loc['SIGO'][order_num, :] != 0
        loc['X'] = np.arange(data2.shape[1])[mask]
        loc.set_source('X', __NAME__ + '/main()')
        # check that we have enough data points to fit data
        if len(loc['X']) > (fitpos + 1):
            # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
            # initial fit params
            iofargs = [p, loc, mask, rorder_num]
            # initial fit for center positions for this order
            loc, cf_data = spirouLOCOR.InitialOrderFit(*iofargs, kind='center')
            # initial fit for widths for this order
            loc, wf_data = spirouLOCOR.InitialOrderFit(*iofargs, kind='fwhm')
            # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
            # Log order number and fit at central pixel and width and rms
            wargs = [
                rorder_num, cf_data['cfitval'], wf_data['cfitval'],
                cf_data['rms']
            ]
            WLOG(p, '', ('ORDER: {0} center at pixel {1:.1f} width '
                         '{2:.1f} rms {3:.3f}').format(*wargs))
            # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
            # sigma fit params
            sigfargs = [p, loc, cf_data, mask, order_num, rorder_num]
            # sigma clip fit for center positions for this order
            cf_data = spirouLOCOR.SigClipOrderFit(*sigfargs, kind='center')
            # load results into storage arrags for this order
            loc['ACC'][rorder_num] = cf_data['a']
            loc['RMS_CENTER'][rorder_num] = cf_data['rms']
            loc['MAX_PTP_CENTER'][rorder_num] = cf_data['max_ptp']
            loc['MAX_PTP_FRACCENTER'][rorder_num] = cf_data['max_ptp_frac']
            loc['MAX_RMPTS_POS'][rorder_num] = cf_data['max_rmpts']

            # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
            # sigma fit params
            sigfargs = [p, loc, wf_data, mask, order_num, rorder_num]
            # sigma clip fit for width positions for this order
            wf_data = spirouLOCOR.SigClipOrderFit(*sigfargs, kind='fwhm')
            # load results into storage arrags for this order
            loc['ASS'][rorder_num] = wf_data['a']
            loc['RMS_FWHM'][rorder_num] = wf_data['rms']
            loc['MAX_PTP_FWHM'][rorder_num] = wf_data['max_ptp']
            loc['MAX_PTP_FRACFWHM'][rorder_num] = wf_data['max_ptp_frac']
            loc['MAX_RMPTS_WID'][rorder_num] = wf_data['max_rmpts']
            # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
            # increase the roder_num iterator
            rorder_num += 1
        # else log that the order is unusable
        else:
            WLOG(p, '', 'Order found too much incomplete, discarded')
    # ----------------------------------------------------------------------
    # Plot the image (ready for fit points to be overplotted later)
    if p['DRS_PLOT'] > 0:
        # get saturation threshold
        satseuil = p['IC_SATSEUIL'] * p['GAIN'] * p['NBFRAMES']
        # plot image above saturation threshold
        sPlt.locplot_im_sat_threshold(p, loc, data2, satseuil)
    # ----------------------------------------------------------------------

    # Log that order geometry has been measured
    WLOG(p, 'info', ('On fiber {0} {1} orders geometry have been '
                     'measured').format(p['FIBER'], rorder_num))
    # Get mean rms
    mean_rms_center = np.nansum(
        loc['RMS_CENTER'][:rorder_num]) * 1000 / rorder_num
    mean_rms_fwhm = np.nansum(loc['RMS_FWHM'][:rorder_num]) * 1000 / rorder_num
    # Log mean rms values
    wmsg = 'Average uncertainty on {0}: {1:.2f} [mpix]'
    WLOG(p, 'info', wmsg.format('position', mean_rms_center))
    WLOG(p, 'info', wmsg.format('width', mean_rms_fwhm))

    # ----------------------------------------------------------------------
    # Plot of RMS for positions and widths
    # ----------------------------------------------------------------------
    if p['DRS_PLOT'] > 0:
        sPlt.locplot_order_number_against_rms(p, loc, rorder_num)

    # ----------------------------------------------------------------------
    # Quality control
    # ----------------------------------------------------------------------
    passed, fail_msg = True, []
    qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
    # ----------------------------------------------------------------------
    # check that max number of points rejected in center fit is below threshold
    if np.nansum(loc['MAX_RMPTS_POS']) > p['QC_LOC_MAXLOCFIT_REMOVED_CTR']:
        fmsg = 'abnormal points rejection during ctr fit ({0:.2f} > {1:.2f})'
        fail_msg.append(
            fmsg.format(np.nansum(loc['MAX_RMPTS_POS']),
                        p['QC_LOC_MAXLOCFIT_REMOVED_CTR']))
        passed = False
        qc_pass.append(0)
    else:
        qc_pass.append(1)
    # add to qc header lists
    qc_values.append(np.nansum(loc['MAX_RMPTS_POS']))
    qc_names.append('sum(MAX_RMPTS_POS)')
    qc_logic.append('sum(MAX_RMPTS_POS) > {0:.2f}'
                    ''.format(p['QC_LOC_MAXLOCFIT_REMOVED_CTR']))
    # ----------------------------------------------------------------------
    # check that max number of points rejected in width fit is below threshold
    if np.nansum(loc['MAX_RMPTS_WID']) > p['QC_LOC_MAXLOCFIT_REMOVED_WID']:
        fmsg = 'abnormal points rejection during width fit ({0:.2f} > {1:.2f})'
        fail_msg.append(
            fmsg.format(np.nansum(loc['MAX_RMPTS_WID']),
                        p['QC_LOC_MAXLOCFIT_REMOVED_WID']))
        passed = False
        qc_pass.append(0)
    else:
        qc_pass.append(1)
    # add to qc header lists
    qc_values.append(np.nansum(loc['MAX_RMPTS_WID']))
    qc_names.append('sum(MAX_RMPTS_WID)')
    qc_logic.append('sum(MAX_RMPTS_WID) > {0:.2f}'
                    ''.format(p['QC_LOC_MAXLOCFIT_REMOVED_WID']))
    # ----------------------------------------------------------------------
    # check that the rms in center fit is lower than qc threshold
    if mean_rms_center > p['QC_LOC_RMSMAX_CENTER']:
        fmsg = 'too high rms on center fitting ({0:.2f} > {1:.2f})'
        fail_msg.append(fmsg.format(mean_rms_center,
                                    p['QC_LOC_RMSMAX_CENTER']))
        passed = False
        qc_pass.append(0)
    else:
        qc_pass.append(1)
    # add to qc header lists
    qc_values.append(mean_rms_center)
    qc_names.append('mean_rms_center')
    qc_logic.append('mean_rms_center > {0:.2f}'
                    ''.format(p['QC_LOC_RMSMAX_CENTER']))
    # ----------------------------------------------------------------------
    # check that the rms in center fit is lower than qc threshold
    if mean_rms_fwhm > p['QC_LOC_RMSMAX_FWHM']:
        fmsg = 'too high rms on profile fwhm fitting ({0:.2f} > {1:.2f})'
        fail_msg.append(fmsg.format(mean_rms_fwhm, p['QC_LOC_RMSMAX_CENTER']))
        passed = False
        qc_pass.append(0)
    else:
        qc_pass.append(1)
    # add to qc header lists
    qc_values.append(mean_rms_fwhm)
    qc_names.append('mean_rms_fwhm')
    qc_logic.append('mean_rms_fwhm > {0:.2f}'
                    ''.format(p['QC_LOC_RMSMAX_CENTER']))
    # ----------------------------------------------------------------------
    # check for abnormal number of identified orders
    if rorder_num != p['QC_LOC_NBO']:
        fmsg = ('abnormal number of identified orders (found {0:.2f} '
                'expected {1:.2f})')
        fail_msg.append(fmsg.format(rorder_num, p['QC_LOC_NBO']))
        passed = False
        qc_pass.append(0)
    else:
        qc_pass.append(1)
    # add to qc header lists
    qc_values.append(rorder_num)
    qc_names.append('rorder_num')
    qc_logic.append('rorder_num != {0:.2f}'.format(p['QC_LOC_NBO']))
    # ----------------------------------------------------------------------
    # finally log the failed messages and set QC = 1 if we pass the
    # quality control QC = 0 if we fail quality control
    if passed:
        WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
        p['QC'] = 1
        p.set_source('QC', __NAME__ + '/main()')
    else:
        for farg in fail_msg:
            wmsg = 'QUALITY CONTROL FAILED: {0}'
            WLOG(p, 'warning', wmsg.format(farg))
        p['QC'] = 0
        p.set_source('QC', __NAME__ + '/main()')
    # store in qc_params
    qc_params = [qc_names, qc_values, qc_logic, qc_pass]

    # ----------------------------------------------------------------------
    # Save and record of image of localization with order center and keywords
    # ----------------------------------------------------------------------
    raw_loco_file = os.path.basename(p['FITSFILENAME'])
    # construct filename
    locofits, tag2 = spirouConfig.Constants.LOC_LOCO_FILE(p)
    locofitsname = os.path.split(locofits)[-1]
    # log that we are saving localization file
    WLOG(p, '', ('Saving localization information '
                 'in file: {0}').format(locofitsname))
    # add keys from original header file
    hdict = spirouImage.CopyOriginalKeys(hdr)
    # define new keys to add
    hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=raw_loco_file)
    hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_SIGDET'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_CONAD'])
    hdict = spirouImage.AddKey(p,
                               hdict,
                               p['KW_LOCO_BCKGRD'],
                               value=loc['MEAN_BACKGRD'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_NBO'], value=rorder_num)
    hdict = spirouImage.AddKey(p,
                               hdict,
                               p['KW_LOCO_DEG_C'],
                               value=p['IC_LOCDFITC'])
    hdict = spirouImage.AddKey(p,
                               hdict,
                               p['KW_LOCO_DEG_W'],
                               value=p['IC_LOCDFITW'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_E'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DELTA'])

    hdict = spirouImage.AddKey(p,
                               hdict,
                               p['KW_LOC_MAXFLX'],
                               value=float(loc['MAX_SIGNAL']))
    hdict = spirouImage.AddKey(p,
                               hdict,
                               p['KW_LOC_SMAXPTS_CTR'],
                               value=np.nansum(loc['MAX_RMPTS_POS']))
    hdict = spirouImage.AddKey(p,
                               hdict,
                               p['KW_LOC_SMAXPTS_WID'],
                               value=np.nansum(loc['MAX_RMPTS_WID']))
    hdict = spirouImage.AddKey(p,
                               hdict,
                               p['KW_LOC_RMS_CTR'],
                               value=mean_rms_center)
    hdict = spirouImage.AddKey(p,
                               hdict,
                               p['KW_LOC_RMS_WID'],
                               value=mean_rms_fwhm)
    # write 2D list of position fit coefficients
    hdict = spirouImage.AddKey2DList(p,
                                     hdict,
                                     p['KW_LOCO_CTR_COEFF'],
                                     values=loc['ACC'][0:rorder_num])
    # write 2D list of width fit coefficients
    hdict = spirouImage.AddKey2DList(p,
                                     hdict,
                                     p['KW_LOCO_FWHM_COEFF'],
                                     values=loc['ASS'][0:rorder_num])
    # add qc parameters
    hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
    hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
    # write center fits and add header keys (via hdict)
    center_fits = spirouLOCOR.CalcLocoFits(loc['ACC'], data2.shape[1])
    p = spirouImage.WriteImage(p, locofits, center_fits, hdict)

    # ----------------------------------------------------------------------
    # Save and record of image of sigma
    # ----------------------------------------------------------------------
    # construct filename
    locofits2, tag3 = spirouConfig.Constants.LOC_LOCO_FILE2(p)
    locofits2name = os.path.split(locofits2)[-1]

    # log that we are saving localization file
    wmsg = 'Saving FWHM information in file: {0}'
    WLOG(p, '', wmsg.format(locofits2name))
    # add keys from original header file
    hdict = spirouImage.CopyOriginalKeys(hdr)
    # define new keys to add
    hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBACK'], value=p['BKGRDFILE'])
    hdict = spirouImage.AddKey1DList(p,
                                     hdict,
                                     p['KW_INFILE1'],
                                     dim1name='file',
                                     values=p['ARG_FILE_NAMES'])
    # add outputs
    hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_SIGDET'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_CONAD'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_NBO'], value=rorder_num)
    hdict = spirouImage.AddKey(p,
                               hdict,
                               p['KW_LOCO_DEG_C'],
                               value=p['IC_LOCDFITC'])
    hdict = spirouImage.AddKey(p,
                               hdict,
                               p['KW_LOCO_DEG_W'],
                               value=p['IC_LOCDFITW'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_E'])
    hdict = spirouImage.AddKey(p,
                               hdict,
                               p['KW_LOC_MAXFLX'],
                               value=float(loc['MAX_SIGNAL']))
    hdict = spirouImage.AddKey(p,
                               hdict,
                               p['KW_LOC_SMAXPTS_CTR'],
                               value=np.nansum(loc['MAX_RMPTS_POS']))
    hdict = spirouImage.AddKey(p,
                               hdict,
                               p['KW_LOC_SMAXPTS_WID'],
                               value=np.nansum(loc['MAX_RMPTS_WID']))
    hdict = spirouImage.AddKey(p,
                               hdict,
                               p['KW_LOC_RMS_CTR'],
                               value=mean_rms_center)
    hdict = spirouImage.AddKey(p,
                               hdict,
                               p['KW_LOC_RMS_WID'],
                               value=mean_rms_fwhm)
    # write 2D list of position fit coefficients
    hdict = spirouImage.AddKey2DList(p,
                                     hdict,
                                     p['KW_LOCO_CTR_COEFF'],
                                     values=loc['ACC'][0:rorder_num])
    # write 2D list of width fit coefficients
    hdict = spirouImage.AddKey2DList(p,
                                     hdict,
                                     p['KW_LOCO_FWHM_COEFF'],
                                     values=loc['ASS'][0:rorder_num])
    # add quality control
    hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
    # write image and add header keys (via hdict)
    width_fits = spirouLOCOR.CalcLocoFits(loc['ASS'], data2.shape[1])
    p = spirouImage.WriteImage(p, locofits2, width_fits, hdict)

    # ----------------------------------------------------------------------
    # Save and Record of image of localization
    # ----------------------------------------------------------------------
    if p['IC_LOCOPT1']:
        # construct filename
        locofits3, tag4 = spirouConfig.Constants.LOC_LOCO_FILE3(p)
        locofits3name = os.path.split(locofits3)[-1]
        # log that we are saving localization file
        wmsg1 = 'Saving localization image with superposition of orders in '
        wmsg2 = 'file: {0}'.format(locofits3name)
        WLOG(p, '', [wmsg1, wmsg2])
        # superpose zeros over the fit in the image
        data4 = spirouLOCOR.ImageLocSuperimp(data2o, loc['ACC'][0:rorder_num])
        # save this image to file
        hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_DRS_DATE'],
                                   value=p['DRS_DATE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_DATE_NOW'],
                                   value=p['DATE_NOW'])
        hdict = spirouImage.AddKey(p, hdict, p['KW_FIBER'], value=p['FIBER'])
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_CDBDARK'],
                                   value=p['DARKFILE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_CDBBAD'],
                                   value=p['BADPFILE'])
        p = spirouImage.WriteImage(p, locofits3, data4, hdict)

    # ----------------------------------------------------------------------
    # Update the calibration database
    # ----------------------------------------------------------------------
    if p['QC'] == 1:
        keydb = 'LOC_' + p['FIBER']
        # copy localisation file to the calibDB folder
        spirouDB.PutCalibFile(p, locofits)
        # update the master calib DB file with new key
        spirouDB.UpdateCalibMaster(p, keydb, locofitsname, hdr)

    # ----------------------------------------------------------------------
    # End Message
    # ----------------------------------------------------------------------
    p = spirouStartup.End(p)
    # return a copy of locally defined variables in the memory
    return dict(locals())
Esempio n. 4
0
def main(night_name=None, files=None):
    """
    cal_shape_spirou.py main function, if night_name and files are None uses
    arguments from run time i.e.:
        cal_shape_spirou.py [night_directory] [fitsfilename]

    :param night_name: string or None, the folder within data raw directory
                                containing files (also reduced directory) i.e.
                                /data/raw/20170710 would be "20170710" but
                                /data/raw/AT5/20180409 would be "AT5/20180409"
    :param files: string, list or None, the list of files to use for
                  arg_file_names and fitsfilename
                  (if None assumes arg_file_names was set from run time)

    :return ll: dictionary, containing all the local variables defined in
                main
    """
    # ----------------------------------------------------------------------
    # Set up
    # ----------------------------------------------------------------------
    # get parameters from config files/run time args/load paths + calibdb
    p = spirouStartup.Begin(recipe=__NAME__)
    p = spirouStartup.LoadArguments(p, night_name, files)
    p = spirouStartup.InitialFileSetup(p)

    # ----------------------------------------------------------------------
    # Read image file
    # ----------------------------------------------------------------------
    # read the image data
    p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add')

    # ----------------------------------------------------------------------
    # Once we have checked the e2dsfile we can load calibDB
    # ----------------------------------------------------------------------
    # as we have custom arguments need to load the calibration database
    p = spirouStartup.LoadCalibDB(p)

    # ----------------------------------------------------------------------
    # Get basic image properties for FP file
    # ----------------------------------------------------------------------
    # get sig det value
    p = spirouImage.GetSigdet(p, hdr, name='sigdet')
    # get exposure time
    p = spirouImage.GetExpTime(p, hdr, name='exptime')
    # get gain
    p = spirouImage.GetGain(p, hdr, name='gain')

    # ----------------------------------------------------------------------
    # Correction of reference FP
    # ----------------------------------------------------------------------
    # set the number of frames
    p['NBFRAMES'] = 1
    p.set_source('NBFRAMES', __NAME__ + '.main()')
    # Correction of DARK
    p, datac = spirouImage.CorrectForDark(p, data, hdr)
    # Resize hc data
    # rotate the image and convert from ADU/s to e-
    data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p)
    # resize image
    bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'],
                   ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'],
                   getshape=False)
    data1 = spirouImage.ResizeImage(p, data, **bkwargs)
    # log change in data size
    WLOG(p, '',
         ('FPref Image format changed to {0}x{1}').format(*data1.shape))
    # Correct for the BADPIX mask (set all bad pixels to zero)
    bargs = [p, data1, hdr]
    p, data1 = spirouImage.CorrectForBadPix(*bargs)
    p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True)
    # log progress
    WLOG(p, '', 'Cleaning FPref hot pixels')
    # correct hot pixels
    data1 = spirouEXTOR.CleanHotpix(data1, badpixmask)
    # Log the number of dead pixels
    # get the number of bad pixels
    with warnings.catch_warnings(record=True) as _:
        n_bad_pix = np.nansum(data1 <= 0)
        n_bad_pix_frac = n_bad_pix * 100 / np.product(data1.shape)
    # Log number
    wmsg = 'Nb FPref dead pixels = {0} / {1:.2f} %'
    WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))

    # ----------------------------------------------------------------------
    # Get master FP file
    # ----------------------------------------------------------------------
    # log progress
    WLOG(p, '', 'Getting FP Master from calibDB')
    # get master fp
    p, masterfp = spirouImage.GetFPMaster(p, hdr)

    # ----------------------------------------------------------------------
    # Get transform parameters
    # ----------------------------------------------------------------------
    # log progress
    wargs = [p['ARG_FILE_NAMES'][0], p['FPMASTERFILE']]
    WLOG(p, 'info', 'Calculating transforming for {0} onto {1}'.format(*wargs))

    gout = spirouImage.GetLinearTransformParams(p, masterfp, data1)
    transform, xres, yres = gout

    # ------------------------------------------------------------------
    # Need to straighten the fp data for debug
    # ------------------------------------------------------------------
    p, shapem_x = spirouImage.GetShapeX(p, hdr)
    p, shapem_y = spirouImage.GetShapeY(p, hdr)
    data2 = spirouImage.EATransform(data1, transform, dxmap=shapem_x,
                                    dymap=shapem_y)

    # ----------------------------------------------------------------------
    # Quality control
    # ----------------------------------------------------------------------
    # set passed variable and fail message list
    passed, fail_msg = True, []
    qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
    # ----------------------------------------------------------------------
    # if transform is None means the fp image quality was too poor
    if transform is None:
        fail_msg.append('FP Image quality too poor (sigma clip failed)')
        passed = False
        qc_pass.append(0)
    else:
        qc_pass.append(1)
    qc_values.append('None')
    qc_names.append('Image Quality')
    qc_logic.append('Image too poor')
    # ----------------------------------------------------------------------
    # get residual qc parameter
    qc_res = p['SHAPE_QC_LINEAR_TRANS_RES_THRES']
    # assess quality of x residuals
    if xres > qc_res:
        fail_msg.append('x-resdiuals too high {0} > {1}'.format(xres, qc_res))
        passed = False
        qc_pass.append(0)
    else:
        qc_pass.append(1)
    qc_values.append(xres)
    qc_names.append('XRES')
    qc_logic.append('XRES > {0}'.format(qc_res))
    # assess quality of x residuals
    if yres > qc_res:
        fail_msg.append('y-resdiuals too high {0} > {1}'.format(yres, qc_res))
        passed = False
        qc_pass.append(0)
    else:
        qc_pass.append(1)
    qc_values.append(yres)
    qc_names.append('YRES')
    qc_logic.append('YRES > {0}'.format(qc_res))

    # ----------------------------------------------------------------------
    # finally log the failed messages and set QC = 1 if we pass the
    # quality control QC = 0 if we fail quality control
    if passed:
        WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
        p['QC'] = 1
        p.set_source('QC', __NAME__ + '/main()')
    else:
        for farg in fail_msg:
            wmsg = 'QUALITY CONTROL FAILED: {0}'
            WLOG(p, 'warning', wmsg.format(farg))
        p['QC'] = 0
        p.set_source('QC', __NAME__ + '/main()')
    # store in qc_params
    qc_params = [qc_names, qc_values, qc_logic, qc_pass]

    # ------------------------------------------------------------------
    # Writing shape to file
    # ------------------------------------------------------------------
    # get the raw tilt file name
    raw_shape_file = os.path.basename(p['FITSFILENAME'])
    # construct file name and path
    shapefits, tag = spirouConfig.Constants.SLIT_SHAPE_LOCAL_FILE(p)
    shapefitsname = os.path.basename(shapefits)
    # Log that we are saving tilt file
    wmsg = 'Saving shape information in file: {0}'
    WLOG(p, '', wmsg.format(shapefitsname))
    # Copy keys from fits file
    hdict = spirouImage.CopyOriginalKeys(hdr)
    # add version number
    hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEX'],
                               value=p['SHAPEXFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEY'],
                               value=p['SHAPEYFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFPMASTER'],
                               value=p['FPMASTERFILE'])
    hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file',
                                     values=p['ARG_FILE_NAMES'])
    # add qc parameters
    hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
    hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
    # add the transform parameters
    hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_DX'], value=transform[0])
    hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_DY'], value=transform[1])
    hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_A'], value=transform[2])
    hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_B'], value=transform[3])
    hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_C'], value=transform[4])
    hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_D'], value=transform[5])
    # write tilt file to file
    p = spirouImage.WriteImage(p, shapefits, [transform], hdict)

    # ----------------------------------------------------------------------
    # Move to calibDB and update calibDB
    # ----------------------------------------------------------------------
    if p['QC']:
        # add shape
        keydb = 'SHAPE'
        # copy shape file to the calibDB folder
        spirouDB.PutCalibFile(p, shapefits)
        # update the master calib DB file with new key
        spirouDB.UpdateCalibMaster(p, keydb, shapefitsname, hdr)

    # ------------------------------------------------------------------
    # Writing sanity check files
    # ------------------------------------------------------------------
    if p['SHAPE_DEBUG_OUTPUTS']:
        # log
        WLOG(p, '', 'Saving debug sanity check files')
        # construct file names
        dargs = [p, p['ARG_FILE_NAMES'][0]]
        out1 = spirouConfig.Constants.SLIT_SHAPE_IN_FP_FILE(*dargs)
        input_fp_file, tag1= out1
        out2 = spirouConfig.Constants.SLIT_SHAPE_OUT_FP_FILE(*dargs)
        output_fp_file, tag2 = out2
        # write input fp file
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
        p = spirouImage.WriteImage(p, input_fp_file, data1, hdict)
        # write output fp file
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
        p = spirouImage.WriteImage(p, output_fp_file, data2, hdict)

    # ----------------------------------------------------------------------
    # End Message
    # ----------------------------------------------------------------------
    p = spirouStartup.End(p)
    # return a copy of locally defined variables in the memory
    return dict(locals())
def main(night_name=None, files=None):
    """
    cal_SLIT_spirou.py main function, if night_name and files are None uses
    arguments from run time i.e.:
        cal_SLIT_spirou.py [night_directory] [files]

    :param night_name: string or None, the folder within data raw directory
                                containing files (also reduced directory) i.e.
                                /data/raw/20170710 would be "20170710" but
                                /data/raw/AT5/20180409 would be "AT5/20180409"
    :param files: string, list or None, the list of files to use for
                  arg_file_names and fitsfilename
                  (if None assumes arg_file_names was set from run time)

    :return ll: dictionary, containing all the local variables defined in
                main
    """
    # ----------------------------------------------------------------------
    # Set up
    # ----------------------------------------------------------------------
    # get parameters from config files/run time args/load paths + calibdb
    p = spirouStartup.Begin(recipe=__NAME__)
    p = spirouStartup.LoadArguments(p, night_name, files)
    p = spirouStartup.InitialFileSetup(p, calibdb=True)
    # set the fiber type
    p['FIBER'] = 'AB'
    p.set_source('FIBER', __NAME__ + '/main()')

    # ----------------------------------------------------------------------
    # Read image file
    # ----------------------------------------------------------------------
    # read the image data
    p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add')

    # ----------------------------------------------------------------------
    # fix for un-preprocessed files
    # ----------------------------------------------------------------------
    data = spirouImage.FixNonPreProcess(p, data)

    # ----------------------------------------------------------------------
    # Get basic image properties
    # ----------------------------------------------------------------------
    # get sig det value
    p = spirouImage.GetSigdet(p, hdr, name='sigdet')
    # get exposure time
    p = spirouImage.GetExpTime(p, hdr, name='exptime')
    # get gain
    p = spirouImage.GetGain(p, hdr, name='gain')

    # ----------------------------------------------------------------------
    # Correction of DARK
    # ----------------------------------------------------------------------
    p, datac = spirouImage.CorrectForDark(p, data, hdr)
    datac = data

    # ----------------------------------------------------------------------
    # Resize image
    # ----------------------------------------------------------------------
    # rotate the image and convert from ADU/s to e-
    data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p)
    # convert NaN to zeros
    data0 = np.where(~np.isfinite(data), np.zeros_like(data), data)
    # resize image
    bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'],
                   ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'],
                   getshape=False)
    data2 = spirouImage.ResizeImage(p, data0, **bkwargs)
    # log change in data size
    WLOG(p, '', ('Image format changed to '
                            '{0}x{1}').format(*data2.shape))

    # ----------------------------------------------------------------------
    # Correct for the BADPIX mask (set all bad pixels to zero)
    # ----------------------------------------------------------------------
    p, data2 = spirouImage.CorrectForBadPix(p, data2, hdr)
    p, badpixmap = spirouImage.CorrectForBadPix(p, data2, hdr, return_map=True)

    # ----------------------------------------------------------------------
    # Background computation
    # ----------------------------------------------------------------------
    if p['IC_DO_BKGR_SUBTRACTION']:
        # log that we are doing background measurement
        WLOG(p, '', 'Doing background measurement on raw frame')
        # get the bkgr measurement
        bargs = [p, data2, hdr, badpixmap]
        # background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs)
        p, background = spirouBACK.MeasureBackgroundMap(*bargs)
    else:
        background = np.zeros_like(data2)
        p['BKGRDFILE'] = 'None'
        p.set_source('BKGRDFILE', __NAME__ + '.main()')
    # apply background correction to data
    data2 = data2 - background

    # save data to loc
    loc = ParamDict()
    loc['DATA'] = data2
    loc.set_source('DATA', __NAME__ + '/main()')

    # ----------------------------------------------------------------------
    # Log the number of dead pixels
    # ----------------------------------------------------------------------
    # get the number of bad pixels
    n_bad_pix = np.nansum(data2 <= 0)
    n_bad_pix_frac = n_bad_pix * 100 / np.product(data2.shape)
    # Log number
    wmsg = 'Nb dead pixels = {0} / {1:.2f} %'
    WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))

    # ------------------------------------------------------------------
    # Get localisation coefficients
    # ------------------------------------------------------------------
    # original there is a loop but it is not used --> removed
    p = spirouImage.FiberParams(p, p['FIBER'], merge=True)
    # get localisation fit coefficients
    p, loc = spirouLOCOR.GetCoeffs(p, hdr, loc)

    # ------------------------------------------------------------------
    # Calculate shape map
    # ------------------------------------------------------------------
    loc = spirouImage.GetShapeMap(p, loc)

    # ------------------------------------------------------------------
    # Plotting
    # ------------------------------------------------------------------
    if p['DRS_PLOT'] > 0:
        # plots setup: start interactive plot
        sPlt.start_interactive_session(p)
        # plot the shape process for each order
        sPlt.slit_shape_angle_plot(p, loc)
        # end interactive section
        sPlt.end_interactive_session(p)

    # ------------------------------------------------------------------
    # Writing to file
    # ------------------------------------------------------------------
    # get the raw tilt file name
    raw_shape_file = os.path.basename(p['FITSFILENAME'])
    # construct file name and path
    shapefits, tag = spirouConfig.Constants.SLIT_XSHAPE_FILE(p)
    shapefitsname = os.path.basename(shapefits)
    # Log that we are saving tilt file
    wmsg = 'Saving shape information in file: {0}'
    WLOG(p, '', wmsg.format(shapefitsname))
    # Copy keys from fits file
    # Copy keys from fits file
    hdict = spirouImage.CopyOriginalKeys(hdr)
    # add version number
    hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPE'], value=raw_shape_file)
    # write tilt file to file
    p = spirouImage.WriteImage(p, shapefits, loc['DXMAP'], hdict)

    # ----------------------------------------------------------------------
    # Quality control
    # ----------------------------------------------------------------------
    # TODO: Decide on some quality control criteria?
    # set passed variable and fail message list
    passed, fail_msg = True, []
    # finally log the failed messages and set QC = 1 if we pass the
    # quality control QC = 0 if we fail quality control
    if passed:
        WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
        p['QC'] = 1
        p.set_source('QC', __NAME__ + '/main()')
    else:
        for farg in fail_msg:
            wmsg = 'QUALITY CONTROL FAILED: {0}'
            WLOG(p, 'warning', wmsg.format(farg))
        p['QC'] = 0
        p.set_source('QC', __NAME__ + '/main()')

    # ----------------------------------------------------------------------
    # Move to calibDB and update calibDB
    # ----------------------------------------------------------------------
    if p['QC']:
        keydb = 'SHAPE'
        # copy shape file to the calibDB folder
        spirouDB.PutCalibFile(p, shapefits)
        # update the master calib DB file with new key
        spirouDB.UpdateCalibMaster(p, keydb, shapefitsname, hdr)

    # ----------------------------------------------------------------------
    # End Message
    # ----------------------------------------------------------------------
    p = spirouStartup.End(p)
    # return a copy of locally defined variables in the memory
    return dict(locals())
def main(night_name=None, hcfile=None, fpfiles=None):
    """
    cal_SLIT_spirou.py main function, if night_name and files are None uses
    arguments from run time i.e.:
        cal_SLIT_spirou.py [night_directory] [files]

    :param night_name: string or None, the folder within data raw directory
                                containing files (also reduced directory) i.e.
                                /data/raw/20170710 would be "20170710" but
                                /data/raw/AT5/20180409 would be "AT5/20180409"
    :param files: string, list or None, the list of files to use for
                  arg_file_names and fitsfilename
                  (if None assumes arg_file_names was set from run time)

    :return ll: dictionary, containing all the local variables defined in
                main
    """
    # ----------------------------------------------------------------------
    # Set up
    # ----------------------------------------------------------------------
    # get parameters from config files/run time args/load paths + calibdb
    p = spirouStartup.Begin(recipe=__NAME__)
    if hcfile is None or fpfiles is None:
        names, types = ['hcfile', 'fpfiles'], [str, str]
        customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1],
                                                        types,
                                                        names,
                                                        last_multi=True)
    else:
        customargs = dict(hcfile=hcfile, fpfile=fpfiles)

    # get parameters from configuration files and run time arguments
    p = spirouStartup.LoadArguments(p,
                                    night_name,
                                    customargs=customargs,
                                    mainfitsfile='fpfiles')

    # ----------------------------------------------------------------------
    # Construct reference filename and get fiber type
    # ----------------------------------------------------------------------
    p, hcfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['HCFILE'])
    p, fpfitsfiles = spirouStartup.MultiFileSetup(p, files=p['FPFILES'])
    # set fiber (it doesn't matter with the 2D image but we need this to get
    # the lamp type for FPFILES and HCFILES, AB == C
    p['FIBER'] = 'AB'
    p['FIB_TYP'] = [p['FIBER']]
    fsource = __NAME__ + '/main()'
    p.set_sources(['FIBER', 'FIB_TYP'], fsource)

    # ----------------------------------------------------------------------
    # Once we have checked the e2dsfile we can load calibDB
    # ----------------------------------------------------------------------
    # as we have custom arguments need to load the calibration database
    p = spirouStartup.LoadCalibDB(p)

    # add a force plot off
    p['PLOT_PER_ORDER'] = PLOT_PER_ORDER
    p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()')

    # ----------------------------------------------------------------------
    # Read FP and HC files
    # ----------------------------------------------------------------------
    # read input fp and hc data
    rkwargs = dict(filename=fpfitsfiles[0],
                   filenames=fpfitsfiles[1:],
                   framemath='add')
    p, fpdata, fphdr = spirouImage.ReadImageAndCombine(p, **rkwargs)

    hcdata, hchdr, _, _ = spirouImage.ReadImage(p, hcfitsfilename)

    # add data and hdr to loc
    loc = ParamDict()
    loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr
    loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr
    # set the source
    sources = ['HCDATA', 'HCHDR']
    loc.set_sources(sources, 'spirouImage.ReadImage()')
    sources = ['FPDATA', 'FPHDR']
    loc.set_sources(sources, 'spirouImage.ReadImage()')

    # ---------------------------------------------------------------------
    # fix for un-preprocessed files
    # ----------------------------------------------------------------------
    hcdata = spirouImage.FixNonPreProcess(p, hcdata)
    fpdata = spirouImage.FixNonPreProcess(p, fpdata)

    # ----------------------------------------------------------------------
    # Once we have checked the e2dsfile we can load calibDB
    # ----------------------------------------------------------------------
    # as we have custom arguments need to load the calibration database
    p = spirouStartup.LoadCalibDB(p)

    # add a force plot off
    p['PLOT_PER_ORDER'] = PLOT_PER_ORDER
    p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()')

    # ----------------------------------------------------------------------
    # Get basic image properties for reference file
    # ----------------------------------------------------------------------
    # get sig det value
    p = spirouImage.GetSigdet(p, fphdr, name='sigdet')
    # get exposure time
    p = spirouImage.GetExpTime(p, fphdr, name='exptime')
    # get gain
    p = spirouImage.GetGain(p, fphdr, name='gain')
    # get lamp parameters
    p = spirouTHORCA.GetLampParams(p, hchdr)
    # get FP_FP DPRTYPE
    p = spirouImage.ReadParam(p, fphdr, 'KW_DPRTYPE', 'DPRTYPE', dtype=str)

    # ----------------------------------------------------------------------
    # Correction of reference FP
    # ----------------------------------------------------------------------
    # set the number of frames
    p['NBFRAMES'] = len(fpfitsfiles)
    p.set_source('NBFRAMES', __NAME__ + '.main()')
    # Correction of DARK
    p, fpdatac = spirouImage.CorrectForDark(p, fpdata, fphdr)
    # Resize hc data
    # rotate the image and convert from ADU/s to e-
    fpdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, fpdatac), p=p)
    # resize image
    bkwargs = dict(xlow=p['IC_CCDX_LOW'],
                   xhigh=p['IC_CCDX_HIGH'],
                   ylow=p['IC_CCDY_LOW'],
                   yhigh=p['IC_CCDY_HIGH'],
                   getshape=False)
    fpdata1 = spirouImage.ResizeImage(p, fpdata, **bkwargs)
    # log change in data size
    WLOG(p, '',
         ('FPref Image format changed to {0}x{1}').format(*fpdata1.shape))
    # Correct for the BADPIX mask (set all bad pixels to zero)
    bargs = [p, fpdata1, fphdr]
    p, fpdata1 = spirouImage.CorrectForBadPix(*bargs)
    p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True)
    # log progress
    WLOG(p, '', 'Cleaning FPref hot pixels')
    # correct hot pixels
    fpdata1 = spirouEXTOR.CleanHotpix(fpdata1, badpixmask)
    # add to loc
    loc['FPDATA1'] = fpdata1
    loc.set_source('FPDATA1', __NAME__ + '.main()')
    # Log the number of dead pixels
    # get the number of bad pixels
    with warnings.catch_warnings(record=True) as _:
        n_bad_pix = np.nansum(fpdata1 <= 0)
        n_bad_pix_frac = n_bad_pix * 100 / np.product(fpdata1.shape)
    # Log number
    wmsg = 'Nb FPref dead pixels = {0} / {1:.2f} %'
    WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))

    # ----------------------------------------------------------------------
    # Correction of HC
    # ----------------------------------------------------------------------
    # set the number of frames
    p['NBFRAMES'] = 1
    p.set_source('NBFRAMES', __NAME__ + '.main()')
    # Correction of DARK
    p, hcdatac = spirouImage.CorrectForDark(p, hcdata, hchdr)
    # Resize hc data
    # rotate the image and convert from ADU/s to e-
    hcdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, hcdatac), p=p)
    # resize image
    bkwargs = dict(xlow=p['IC_CCDX_LOW'],
                   xhigh=p['IC_CCDX_HIGH'],
                   ylow=p['IC_CCDY_LOW'],
                   yhigh=p['IC_CCDY_HIGH'],
                   getshape=False)
    hcdata1 = spirouImage.ResizeImage(p, hcdata, **bkwargs)
    # log change in data size
    WLOG(p, '', ('HC Image format changed to {0}x{1}').format(*hcdata1.shape))
    # Correct for the BADPIX mask (set all bad pixels to zero)
    bargs = [p, hcdata1, hchdr]
    p, hcdata1 = spirouImage.CorrectForBadPix(*bargs)
    p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True)
    # log progress
    WLOG(p, '', 'Cleaning HC hot pixels')
    # correct hot pixels
    hcdata1 = spirouEXTOR.CleanHotpix(hcdata1, badpixmask)
    # add to loc
    loc['HCDATA1'] = hcdata1
    loc.set_source('HCDATA1', __NAME__ + '.main()')
    # Log the number of dead pixels
    # get the number of bad pixels
    with warnings.catch_warnings(record=True) as _:
        n_bad_pix = np.nansum(hcdata1 <= 0)
        n_bad_pix_frac = n_bad_pix * 100 / np.product(hcdata1.shape)
    # Log number
    wmsg = 'Nb HC dead pixels = {0} / {1:.2f} %'
    WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))

    # -------------------------------------------------------------------------
    # get all FP_FP files
    # -------------------------------------------------------------------------
    fpfilenames = spirouImage.FindFiles(p,
                                        filetype=p['DPRTYPE'],
                                        allowedtypes=p['ALLOWED_FP_TYPES'])
    # convert filenames to a numpy array
    fpfilenames = np.array(fpfilenames)
    # julian date to know which file we need to
    # process together
    fp_time = np.zeros(len(fpfilenames))
    basenames, fp_exp, fp_pp_version, nightnames = [], [], [], []
    # log progress
    WLOG(p, '', 'Reading all fp file headers')
    # looping through the file headers
    for it in range(len(fpfilenames)):
        # log progress
        wmsg = '\tReading file {0} / {1}'
        WLOG(p, 'info', wmsg.format(it + 1, len(fpfilenames)))
        # get fp filename
        fpfilename = fpfilenames[it]
        # get night name
        night_name = os.path.dirname(fpfilenames[it]).split(p['TMP_DIR'])[-1]
        # read data
        data_it, hdr_it, _, _ = spirouImage.ReadImage(p, fpfilename)
        # get header
        hdr = spirouImage.ReadHeader(p, filepath=fpfilenames[it])
        # add MJDATE to dark times
        fp_time[it] = float(hdr[p['KW_ACQTIME'][0]])
        # add other keys (for tabular output)
        basenames.append(os.path.basename(fpfilenames[it]))
        nightnames.append(night_name)
        fp_exp.append(float(hdr[p['KW_EXPTIME'][0]]))
        fp_pp_version.append(hdr[p['KW_PPVERSION'][0]])

    # -------------------------------------------------------------------------
    # match files by date
    # -------------------------------------------------------------------------
    # log progress
    wmsg = 'Matching FP files by observation time (+/- {0} hrs)'
    WLOG(p, '', wmsg.format(p['DARK_MASTER_MATCH_TIME']))
    # get the time threshold
    time_thres = p['FP_MASTER_MATCH_TIME']
    # get items grouped by time
    matched_id = spirouImage.GroupFilesByTime(p, fp_time, time_thres)

    # -------------------------------------------------------------------------
    # construct the master fp file (+ correct for dark/badpix)
    # -------------------------------------------------------------------------
    cargs = [fpdata1, fpfilenames, matched_id]
    fpcube, transforms = spirouImage.ConstructMasterFP(p, *cargs)
    # log process
    wmsg1 = 'Master FP construction complete.'
    wmsg2 = '\tAdding {0} group images to form FP master image'
    WLOG(p, 'info', [wmsg1, wmsg2.format(len(fpcube))])
    # sum the cube to make fp data
    masterfp = np.sum(fpcube, axis=0)
    # add to loc
    loc['MASTERFP'] = masterfp
    loc.set_source('MASTERFP', __NAME__ + '.main()')

    # ------------------------------------------------------------------
    # Get localisation coefficients
    # ------------------------------------------------------------------
    # original there is a loop but it is not used --> removed
    p = spirouImage.FiberParams(p, p['FIBER'], merge=True)
    # get localisation fit coefficients
    p, loc = spirouLOCOR.GetCoeffs(p, fphdr, loc)

    # ------------------------------------------------------------------
    # Get master wave solution map
    # ------------------------------------------------------------------
    # get master wave map
    masterwavefile = spirouDB.GetDatabaseMasterWave(p)
    # log process
    wmsg1 = 'Getting master wavelength grid'
    wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile))
    WLOG(p, '', [wmsg1, wmsg2])
    # Force A and B to AB solution
    if p['FIBER'] in ['A', 'B']:
        wave_fiber = 'AB'
    else:
        wave_fiber = p['FIBER']
    # read master wave map
    wout = spirouImage.GetWaveSolution(p,
                                       filename=masterwavefile,
                                       return_wavemap=True,
                                       quiet=True,
                                       return_header=True,
                                       fiber=wave_fiber)
    loc['MASTERWAVEP'], loc['MASTERWAVE'] = wout[:2]
    loc['MASTERWAVEHDR'], loc['WSOURCE'] = wout[2:]
    # set sources
    wsource = ['MASTERWAVEP', 'MASTERWAVE', 'MASTERWAVEHDR']
    loc.set_sources(wsource, 'spirouImage.GetWaveSolution()')

    # ----------------------------------------------------------------------
    # Read UNe solution
    # ----------------------------------------------------------------------
    wave_u_ne, amp_u_ne = spirouImage.ReadLineList(p)
    loc['LL_LINE'], loc['AMPL_LINE'] = wave_u_ne, amp_u_ne
    source = __NAME__ + '.main() + spirouImage.ReadLineList()'
    loc.set_sources(['LL_LINE', 'AMPL_LINE'], source)

    # ----------------------------------------------------------------------
    # Read cavity length file
    # ----------------------------------------------------------------------
    loc['CAVITY_LEN_COEFFS'] = spirouImage.ReadCavityLength(p)
    source = __NAME__ + '.main() + spirouImage.ReadCavityLength()'
    loc.set_source('CAVITY_LEN_COEFFS', source)

    # ----------------------------------------------------------------------
    # Calculate shape map
    # ----------------------------------------------------------------------
    # calculate dx map
    loc = spirouImage.GetXShapeMap(p, loc)
    # if dx map is None we shouldn't continue
    if loc['DXMAP'] is None:
        fargs = [
            loc['MAXDXMAPINFO'][0], loc['MAXDXMAPINFO'][1], loc['MAXDXMAPSTD'],
            p['SHAPE_QC_DXMAP_STD']
        ]
        fmsg = ('The std of the dxmap for order {0} y-pixel {1} is too large.'
                ' std = {2} (limit = {3})'.format(*fargs))
        wmsg = 'QUALITY CONTROL FAILED: {0}'
        WLOG(p, 'warning', wmsg.format(fmsg))
        WLOG(p, 'warning', 'Cannot continue. Exiting.')
        # End Message
        p = spirouStartup.End(p)
        # return a copy of locally defined variables in the memory
        return dict(locals())

    # calculate dymap
    loc = spirouImage.GetYShapeMap(p, loc, fphdr)

    # ------------------------------------------------------------------
    # Need to straighten the dxmap
    # ------------------------------------------------------------------
    # copy it first
    loc['DXMAP0'] = np.array(loc['DXMAP'])
    # straighten it
    loc['DXMAP'] = spirouImage.EATransform(loc['DXMAP'], dymap=loc['DYMAP'])

    # ------------------------------------------------------------------
    # Need to straighten the hc data and fp data for debug
    # ------------------------------------------------------------------
    # log progress
    WLOG(p, '', 'Shape finding complete. Applying transforms.')
    # apply very last update of the debananafication
    tkwargs = dict(dxmap=loc['DXMAP'], dymap=loc['DYMAP'])
    loc['HCDATA2'] = spirouImage.EATransform(loc['HCDATA1'], **tkwargs)
    loc['FPDATA2'] = spirouImage.EATransform(loc['FPDATA1'], **tkwargs)
    loc.set_sources(['HCDATA2', 'FPDATA2'], __NAME__ + '.main()')

    # ------------------------------------------------------------------
    # Plotting
    # ------------------------------------------------------------------
    if p['DRS_PLOT'] > 0:
        # plots setup: start interactive plot
        sPlt.start_interactive_session(p)
        # plot the shape process for one order
        sPlt.slit_shape_angle_plot(p, loc)
        # end interactive section
        sPlt.end_interactive_session(p)

    # ----------------------------------------------------------------------
    # Quality control
    # ----------------------------------------------------------------------
    # TODO: Decide on some quality control criteria?
    # set passed variable and fail message list
    passed, fail_msg = True, []
    qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
    # finally log the failed messages and set QC = 1 if we pass the
    # quality control QC = 0 if we fail quality control
    if passed:
        WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
        p['QC'] = 1
        p.set_source('QC', __NAME__ + '/main()')
    else:
        for farg in fail_msg:
            wmsg = 'QUALITY CONTROL FAILED: {0}'
            WLOG(p, 'warning', wmsg.format(farg))
        p['QC'] = 0
        p.set_source('QC', __NAME__ + '/main()')
    # add to qc header lists
    qc_values.append(loc['MAXDXMAPSTD'])
    qc_names.append('DXMAP STD')
    qc_logic.append('DXMAP STD < {0}'.format(p['SHAPE_QC_DXMAP_STD']))
    qc_pass.append(1)
    # store in qc_params
    qc_params = [qc_names, qc_values, qc_logic, qc_pass]

    # ------------------------------------------------------------------
    # Writing FP big table
    # ------------------------------------------------------------------
    # construct big fp table
    colnames = [
        'FILENAME', 'NIGHT', 'MJDATE', 'EXPTIME', 'PVERSION', 'GROUPID',
        'DXREF', 'DYREF', 'A', 'B', 'C', 'D'
    ]
    values = [
        basenames, nightnames, fp_time, fp_exp, fp_pp_version, matched_id,
        transforms[:, 0], transforms[:, 1], transforms[:, 2], transforms[:, 3],
        transforms[:, 4], transforms[:, 5]
    ]
    fptable = spirouImage.MakeTable(p, colnames, values)

    # ------------------------------------------------------------------
    # Writing DXMAP to file
    # ------------------------------------------------------------------
    # get the raw tilt file name
    raw_shape_file = os.path.basename(p['FITSFILENAME'])
    # construct file name and path
    shapexfits, tag = spirouConfig.Constants.SLIT_XSHAPE_FILE(p)
    shapexfitsname = os.path.basename(shapexfits)
    # Log that we are saving tilt file
    wmsg = 'Saving shape x information in file: {0}'
    WLOG(p, '', wmsg.format(shapexfitsname))
    # Copy keys from fits file
    hdict = spirouImage.CopyOriginalKeys(fphdr)
    # add version number
    hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
    hdict = spirouImage.AddKey1DList(p,
                                     hdict,
                                     p['KW_INFILE1'],
                                     dim1name='hcfile',
                                     values=p['HCFILE'])
    hdict = spirouImage.AddKey1DList(p,
                                     hdict,
                                     p['KW_INFILE2'],
                                     dim1name='fpfile',
                                     values=p['FPFILES'])
    # add qc parameters
    hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
    hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
    # write tilt file to file
    p = spirouImage.WriteImageTable(p,
                                    shapexfits,
                                    image=loc['DXMAP'],
                                    table=fptable,
                                    hdict=hdict)

    # ------------------------------------------------------------------
    # Writing DYMAP to file
    # ------------------------------------------------------------------
    # get the raw tilt file name
    raw_shape_file = os.path.basename(p['FITSFILENAME'])
    # construct file name and path
    shapeyfits, tag = spirouConfig.Constants.SLIT_YSHAPE_FILE(p)
    shapeyfitsname = os.path.basename(shapeyfits)
    # Log that we are saving tilt file
    wmsg = 'Saving shape y information in file: {0}'
    WLOG(p, '', wmsg.format(shapeyfitsname))
    # Copy keys from fits file
    hdict = spirouImage.CopyOriginalKeys(fphdr)
    # add version number
    hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
    hdict = spirouImage.AddKey1DList(p,
                                     hdict,
                                     p['KW_INFILE1'],
                                     dim1name='hcfile',
                                     values=p['HCFILE'])
    hdict = spirouImage.AddKey1DList(p,
                                     hdict,
                                     p['KW_INFILE2'],
                                     dim1name='fpfile',
                                     values=p['FPFILES'])
    # add qc parameters
    hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
    hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
    # write tilt file to file
    p = spirouImage.WriteImageTable(p,
                                    shapeyfits,
                                    image=loc['DYMAP'],
                                    table=fptable,
                                    hdict=hdict)

    # ------------------------------------------------------------------
    # Writing Master FP to file
    # ------------------------------------------------------------------
    # get the raw tilt file name
    raw_shape_file = os.path.basename(p['FITSFILENAME'])
    # construct file name and path
    fpmasterfits, tag = spirouConfig.Constants.SLIT_MASTER_FP_FILE(p)
    fpmasterfitsname = os.path.basename(fpmasterfits)
    # Log that we are saving tilt file
    wmsg = 'Saving master FP file: {0}'
    WLOG(p, '', wmsg.format(fpmasterfitsname))
    # Copy keys from fits file
    hdict = spirouImage.CopyOriginalKeys(fphdr)
    # add version number
    hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
    hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
    hdict = spirouImage.AddKey1DList(p,
                                     hdict,
                                     p['KW_INFILE1'],
                                     dim1name='hcfile',
                                     values=p['HCFILE'])
    hdict = spirouImage.AddKey1DList(p,
                                     hdict,
                                     p['KW_INFILE2'],
                                     dim1name='fpfile',
                                     values=p['FPFILES'])
    # add qc parameters
    hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
    hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
    # write tilt file to file
    p = spirouImage.WriteImageTable(p,
                                    fpmasterfits,
                                    image=masterfp,
                                    table=fptable,
                                    hdict=hdict)

    # ------------------------------------------------------------------
    # Writing sanity check files
    # ------------------------------------------------------------------
    if p['SHAPE_DEBUG_OUTPUTS']:
        # log
        WLOG(p, '', 'Saving debug sanity check files')
        # construct file names
        input_fp_file, tag1 = spirouConfig.Constants.SLIT_SHAPE_IN_FP_FILE(p)
        output_fp_file, tag2 = spirouConfig.Constants.SLIT_SHAPE_OUT_FP_FILE(p)
        input_hc_file, tag3 = spirouConfig.Constants.SLIT_SHAPE_IN_HC_FILE(p)
        output_hc_file, tag4 = spirouConfig.Constants.SLIT_SHAPE_OUT_HC_FILE(p)
        bdxmap_file, tag5 = spirouConfig.Constants.SLIT_SHAPE_BDXMAP_FILE(p)
        # write input fp file
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
        p = spirouImage.WriteImage(p, input_fp_file, loc['FPDATA1'], hdict)
        # write output fp file
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
        p = spirouImage.WriteImage(p, output_fp_file, loc['FPDATA2'], hdict)
        # write input fp file
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
        p = spirouImage.WriteImage(p, input_hc_file, loc['HCDATA1'], hdict)
        # write output fp file
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
        p = spirouImage.WriteImage(p, output_hc_file, loc['HCDATA2'], hdict)
        # write overlap file
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag5)
        p = spirouImage.WriteImage(p, bdxmap_file, loc['DXMAP0'], hdict)

    # ----------------------------------------------------------------------
    # Move to calibDB and update calibDB
    # ----------------------------------------------------------------------
    if p['QC']:
        # add shape x
        keydb = 'SHAPEX'
        # copy shape file to the calibDB folder
        spirouDB.PutCalibFile(p, shapexfits)
        # update the master calib DB file with new key
        spirouDB.UpdateCalibMaster(p, keydb, shapexfitsname, fphdr)
        # add shape y
        keydb = 'SHAPEY'
        # copy shape file to the calibDB folder
        spirouDB.PutCalibFile(p, shapeyfits)
        # update the master calib DB file with new key
        spirouDB.UpdateCalibMaster(p, keydb, shapeyfitsname, fphdr)
        # add fp master
        keydb = 'FPMASTER'
        # copy shape file to the calibDB folder
        spirouDB.PutCalibFile(p, fpmasterfits)
        # update the master calib DB file with new key
        spirouDB.UpdateCalibMaster(p, keydb, fpmasterfitsname, fphdr)
    # ----------------------------------------------------------------------
    # End Message
    # ----------------------------------------------------------------------
    p = spirouStartup.End(p)
    # return a copy of locally defined variables in the memory
    return dict(locals())
Esempio n. 7
0
def main(night_name=None, files=None):
    """
    cal_FF_RAW_spirou.py main function, if night_name and files are None uses
    arguments from run time i.e.:
        cal_FF_RAW_spirou.py [night_directory] [files]

    :param night_name: string or None, the folder within data raw directory
                                containing files (also reduced directory) i.e.
                                /data/raw/20170710 would be "20170710" but
                                /data/raw/AT5/20180409 would be "AT5/20180409"
    :param files: string, list or None, the list of files to use for
                  arg_file_names and fitsfilename
                  (if None assumes arg_file_names was set from run time)

    :return ll: dictionary, containing all the local variables defined in
                main
    """
    # ----------------------------------------------------------------------
    # Set up
    # ----------------------------------------------------------------------
    # get parameters from config files/run time args/load paths + calibdb
    p = spirouStartup.Begin(recipe=__NAME__)
    p = spirouStartup.LoadArguments(p, night_name, files)
    p = spirouStartup.InitialFileSetup(p, calibdb=True)

    # run specific start up
    p['FIB_TYPE'] = p['FIBER_TYPES']
    p.set_source('FIB_TYPE', __NAME__ + '__main__()')

    # ----------------------------------------------------------------------
    # Read image file
    # ----------------------------------------------------------------------
    # read the image data
    p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add')

    # ----------------------------------------------------------------------
    # fix for un-preprocessed files
    # ----------------------------------------------------------------------
    data = spirouImage.FixNonPreProcess(p, data)

    # ----------------------------------------------------------------------
    # Get basic image properties
    # ----------------------------------------------------------------------
    # get sig det value
    p = spirouImage.GetSigdet(p, hdr, name='sigdet')
    # get exposure time
    p = spirouImage.GetExpTime(p, hdr, name='exptime')
    # get gain
    p = spirouImage.GetGain(p, hdr, name='gain')
    # set sigdet and conad keywords (sigdet is changed later)
    p['KW_CCD_SIGDET'][1] = p['SIGDET']
    p['KW_CCD_CONAD'][1] = p['GAIN']

    # ----------------------------------------------------------------------
    # Correction of DARK
    # ----------------------------------------------------------------------
    p, datac = spirouImage.CorrectForDark(p, data, hdr)

    # ----------------------------------------------------------------------
    # Resize image
    # ----------------------------------------------------------------------
    # rotate the image and convert from ADU/s to e-
    data = spirouImage.ConvertToADU(spirouImage.FlipImage(p, datac), p=p)
    # convert NaN to zeros
    data0 = np.where(~np.isfinite(data), np.zeros_like(data), data)
    # resize image
    bkwargs = dict(xlow=p['IC_CCDX_LOW'],
                   xhigh=p['IC_CCDX_HIGH'],
                   ylow=p['IC_CCDY_LOW'],
                   yhigh=p['IC_CCDY_HIGH'],
                   getshape=False)
    data1 = spirouImage.ResizeImage(p, data0, **bkwargs)
    # log change in data size
    WLOG(p, '', ('Image format changed to '
                 '{0}x{1}').format(*data1.shape[::-1]))
    # ----------------------------------------------------------------------
    # Correct for the BADPIX mask (set all bad pixels to zero)
    # ----------------------------------------------------------------------
    p, data1 = spirouImage.CorrectForBadPix(p, data1, hdr)

    # ----------------------------------------------------------------------
    # Log the number of dead pixels
    # ----------------------------------------------------------------------
    # get the number of bad pixels
    n_bad_pix = np.sum(~np.isfinite(data1))
    n_bad_pix_frac = n_bad_pix * 100 / np.product(data1.shape)
    # Log number
    wmsg = 'Nb dead pixels = {0} / {1:.4f} %'
    WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))

    # ----------------------------------------------------------------------
    # Get the miny, maxy and max_signal for the central column
    # ----------------------------------------------------------------------
    # get the central column
    y = data1[p['IC_CENT_COL'], :]
    # get the min max and max signal using box smoothed approach
    miny, maxy, max_signal, diff_maxmin = spirouBACK.MeasureMinMaxSignal(p, y)
    # Log max average flux/pixel
    wmsg = ('Maximum average flux (95th percentile) /pixel in the spectrum: '
            '{0:.1f} [ADU]')
    WLOG(p, 'info', wmsg.format(max_signal / p['NBFRAMES']))

    # ----------------------------------------------------------------------
    # Background computation
    # ----------------------------------------------------------------------
    # p['ic_bkgr_percent'] = 3.0
    if p['IC_DO_BKGR_SUBTRACTION']:
        # log that we are doing background measurement
        WLOG(p, '', 'Doing background measurement on raw frame')
        # get the bkgr measurement
        bargs = [p, data1, hdr]
        # background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs)
        p, background = spirouBACK.MeasureBackgroundMap(*bargs)
    else:
        background = np.zeros_like(data1)
        p['BKGRDFILE'] = 'None'
        p.set_source('BKGRDFILE', __NAME__ + '.main()')
    # apply background correction to data
    data1 = data1 - background

    # ----------------------------------------------------------------------
    # Read tilt slit angle
    # ----------------------------------------------------------------------
    # define loc storage parameter dictionary
    loc = ParamDict()
    # get tilts
    if p['IC_FF_EXTRACT_TYPE'] not in EXTRACT_SHAPE_TYPES:
        p, loc['TILT'] = spirouImage.ReadTiltFile(p, hdr)
    else:
        loc['TILT'] = None
    loc.set_source('TILT', __NAME__ + '/main()')

    # ----------------------------------------------------------------------
    # Get all fiber data (for all fibers)
    # ----------------------------------------------------------------------
    # TODO: This is temp solution for options 5a and 5b
    loc_fibers = spirouLOCOR.GetFiberData(p, hdr)

    # ------------------------------------------------------------------
    # Deal with debananafication
    # ------------------------------------------------------------------
    # if mode 4a or 4b we need to straighten in x only
    if p['IC_FF_EXTRACT_TYPE'] in ['4a', '4b']:
        # get the shape parameters
        p, shapem_x = spirouImage.GetShapeX(p, hdr)
        p, shape_local = spirouImage.GetShapeLocal(p, hdr)
        # log progress
        WLOG(p, '', 'Debananafying (straightening) image')
        # apply shape transforms
        targs = dict(lin_transform_vect=shape_local, dxmap=shapem_x)
        data2 = spirouImage.EATransform(data1, **targs)
    # if mode 5a or 5b we need to straighten in x and y using the
    #     polynomial fits for location
    elif p['IC_FF_EXTRACT_TYPE'] in ['5a', '5b']:
        # get the shape parameters
        p, shapem_x = spirouImage.GetShapeX(p, hdr)
        p, shapem_y = spirouImage.GetShapeY(p, hdr)
        p, shape_local = spirouImage.GetShapeLocal(p, hdr)
        p, fpmaster = spirouImage.GetFPMaster(p, hdr)
        # get the bad pixel map
        bkwargs = dict(return_map=True, quiet=True)
        p, badpix = spirouImage.CorrectForBadPix(p, data1, hdr, **bkwargs)
        # log progress
        WLOG(p, '', 'Cleaning image')
        # clean the image
        data1 = spirouEXTOR.CleanHotpix(data1, badpix)
        # log progress
        WLOG(p, '', 'Debananafying (straightening) image')
        # apply shape transforms
        targs = dict(lin_transform_vect=shape_local,
                     dxmap=shapem_x,
                     dymap=shapem_y)
        data2 = spirouImage.EATransform(data1, **targs)
    # in any other mode we do not straighten
    else:
        data2 = np.array(data1)

    # ----------------------------------------------------------------------
    # Fiber loop
    # ----------------------------------------------------------------------
    # loop around fiber types
    for fiber in p['FIB_TYPE']:
        # set fiber in p
        p['FIBER'] = fiber
        p.set_source('FIBER', __NAME__ + '/main()')

        # get fiber parameters
        params2add = spirouImage.FiberParams(p, p['FIBER'])
        for param in params2add:
            p[param] = params2add[param]
            p.set_source(param, __NAME__ + '.main()')

        # ------------------------------------------------------------------
        # Get fiber specific parameters from loc_fibers
        # ------------------------------------------------------------------
        # get this fibers parameters
        p = spirouImage.FiberParams(p, p['FIBER'], merge=True)
        # get localisation parameters
        for key in loc_fibers[fiber]:
            loc[key] = loc_fibers[fiber][key]
            loc.set_source(key, loc_fibers[fiber].sources[key])
        # get locofile source
        p['LOCOFILE'] = loc['LOCOFILE']
        p.set_source('LOCOFILE', loc.sources['LOCOFILE'])
        # get the order_profile
        order_profile = loc_fibers[fiber]['ORDER_PROFILE']

        # ------------------------------------------------------------------
        # Set up Extract storage
        # ------------------------------------------------------------------
        # Create array to store extraction (for each order and each pixel
        # along order)
        loc['E2DS'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
        loc['E2DSLL'] = []
        # Create array to store the blaze (for each order and at each pixel
        # along order)
        loc['BLAZE'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
        # Create array to store the flat (for each order and at each pixel
        # along order)
        loc['FLAT'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
        # Create array to store the signal to noise ratios for each order
        loc['SNR'] = np.zeros(loc['NUMBER_ORDERS'])
        # Create array to store the rms for each order
        loc['RMS'] = np.zeros(loc['NUMBER_ORDERS'])

        # Manually set the sigdet to be used in extraction weighting
        if p['IC_FF_SIGDET'] > 0:
            p['SIGDET'] = float(p['IC_FF_SIGDET'])
        # ------------------------------------------------------------------
        # Extract orders
        # old code time: 1 loop, best of 3: 22.3 s per loop
        # new code time: 3.16 s ± 237 ms per loop
        # ------------------------------------------------------------------
        # get limits of order extraction
        valid_orders = spirouFLAT.GetValidOrders(p, loc)
        # loop around each order
        for order_num in valid_orders:
            # extract this order
            eargs = [p, loc, data2, order_num]
            ekwargs = dict(mode=p['IC_FF_EXTRACT_TYPE'],
                           order_profile=order_profile)
            with warnings.catch_warnings(record=True) as w:
                eout = spirouEXTOR.Extraction(*eargs, **ekwargs)
            # deal with different return
            if p['IC_FF_EXTRACT_TYPE'] in EXTRACT_LL_TYPES:
                e2ds, e2dsll, cpt = eout
            else:
                e2ds, cpt = eout
                e2dsll = None
            # calculate the noise
            range1, range2 = p['IC_EXT_RANGE1'], p['IC_EXT_RANGE2']
            noise = p['SIGDET'] * np.sqrt(range1 + range2)
            # get window size
            blaze_win1 = int(data2.shape[1] / 2) - p['IC_EXTFBLAZ']
            blaze_win2 = int(data2.shape[1] / 2) + p['IC_EXTFBLAZ']
            # get average flux per pixel
            flux = np.nansum(
                e2ds[blaze_win1:blaze_win2]) / (2 * p['IC_EXTFBLAZ'])
            # calculate signal to noise ratio = flux/sqrt(flux + noise^2)
            snr = flux / np.sqrt(flux + noise**2)
            # remove edge of orders at low S/N
            with warnings.catch_warnings(record=True) as _:
                blazemask = e2ds < (flux / p['IC_FRACMINBLAZE'])
                e2ds = np.where(blazemask, np.nan, e2ds)
            #            e2ds = np.where(e2ds < p['IC_MINBLAZE'], 0., e2ds)
            # calcualte the blaze function
            blaze = spirouFLAT.MeasureBlazeForOrder(p, e2ds)
            # calculate the flat
            flat = e2ds / blaze
            # calculate the rms
            rms = np.nanstd(flat)
            # log the SNR RMS
            wmsg = 'On fiber {0} order {1}: S/N= {2:.1f}  - FF rms={3:.2f} %'
            wargs = [fiber, order_num, snr, rms * 100.0]
            WLOG(p, '', wmsg.format(*wargs))
            # add calculations to storage
            loc['E2DS'][order_num] = e2ds
            loc['SNR'][order_num] = snr
            loc['RMS'][order_num] = rms
            loc['BLAZE'][order_num] = blaze
            loc['FLAT'][order_num] = flat
            # save the longfile
            if p['IC_FF_EXTRACT_TYPE'] in EXTRACT_LL_TYPES:
                loc['E2DSLL'].append(e2dsll)
            # set sources
            source = __NAME__ + '/main()()'
            loc.set_sources(['e2ds', 'SNR', 'RMS', 'blaze', 'flat'], source)
            # Log if saturation level reached
            satvalue = (flux / p['GAIN']) / (range1 + range2)
            if satvalue > (p['QC_LOC_FLUMAX'] * p['NBFRAMES']):
                wmsg = 'SATURATION LEVEL REACHED on Fiber {0} order={1}'
                WLOG(p, 'warning', wmsg.format(fiber, order_num))

        # ----------------------------------------------------------------------
        # Plots
        # ----------------------------------------------------------------------
        if p['DRS_PLOT'] > 0:
            # start interactive session if needed
            sPlt.start_interactive_session(p)
            # plot all orders or one order
            if p['IC_FF_PLOT_ALL_ORDERS']:
                # plot image with all order fits (slower)
                sPlt.ff_aorder_fit_edges(p, loc, data1)
            else:
                # plot image with selected order fit and edge fit (faster)
                sPlt.ff_sorder_fit_edges(p, loc, data1)
            # plot tilt adjusted e2ds and blaze for selected order
            sPlt.ff_sorder_tiltadj_e2ds_blaze(p, loc)
            # plot flat for selected order
            sPlt.ff_sorder_flat(p, loc)
            # plot the RMS for all but skipped orders
            # sPlt.ff_rms_plot(p, loc)

            if p['IC_FF_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES:
                sPlt.ff_debanana_plot(p, loc, data2)
        # ------------------------------------------------------------------
        # Quality control
        # ------------------------------------------------------------------
        passed, fail_msg = True, []
        qc_values, qc_names, qc_logic, qc_pass = [], [], [], []

        # saturation check: check that the max_signal is lower than
        # qc_max_signal
        # if max_signal > (p['QC_MAX_SIGNAL'] * p['nbframes']):
        #     fmsg = 'Too much flux in the image (max authorized={0})'
        #     fail_msg.append(fmsg.format(p['QC_MAX_SIGNAL'] * p['nbframes']))
        #     passed = False
        #     # For some reason this test is ignored in old code
        #     passed = True
        #     WLOG(p, 'info', fail_msg[-1])

        # get mask for removing certain orders in the RMS calculation
        remove_orders = np.array(p['FF_RMS_PLOT_SKIP_ORDERS'])
        mask = np.in1d(np.arange(len(loc['RMS'])), remove_orders)
        # apply mask and calculate the maximum RMS
        max_rms = np.nanmax(loc['RMS'][~mask])
        # apply the quality control based on the new RMS
        if max_rms > p['QC_FF_RMS']:
            fmsg = 'abnormal RMS of FF ({0:.3f} > {1:.3f})'
            fail_msg.append(fmsg.format(max_rms, p['QC_FF_RMS']))
            passed = False
            qc_pass.append(0)
        else:
            qc_pass.append(1)
        # add to qc header lists
        qc_values.append(max_rms)
        qc_names.append('max_rms')
        qc_logic.append('max_rms > {0:.3f}'.format(p['QC_FF_RMS']))

        # finally log the failed messages and set QC = 1 if we pass the
        # quality control QC = 0 if we fail quality control
        if passed:
            wmsg = 'QUALITY CONTROL SUCCESSFUL - Well Done -'
            WLOG(p, 'info', wmsg)
            p['QC'] = 1
            p.set_source('QC', __NAME__ + '/main()')
        else:
            for farg in fail_msg:
                wmsg = 'QUALITY CONTROL FAILED: {0}'
                WLOG(p, 'warning', wmsg.format(farg))
            p['QC'] = 0
            p.set_source('QC', __NAME__ + '/main()')
        # store in qc_params
        qc_params = [qc_names, qc_values, qc_logic, qc_pass]

        # ----------------------------------------------------------------------
        # Store Blaze in file
        # ----------------------------------------------------------------------
        # get raw flat filename
        raw_flat_file = os.path.basename(p['FITSFILENAME'])
        e2dsllfits, tag4 = spirouConfig.Constants.EXTRACT_E2DSLL_FILE(p)
        # get extraction method and function
        efout = spirouEXTOR.GetExtMethod(p, p['IC_FF_EXTRACT_TYPE'])
        extmethod, extfunc = efout
        # construct filename
        blazefits, tag1 = spirouConfig.Constants.FF_BLAZE_FILE(p)
        blazefitsname = os.path.split(blazefits)[-1]
        # log that we are saving blaze file
        wmsg = 'Saving blaze spectrum for fiber: {0} in {1}'
        WLOG(p, '', wmsg.format(fiber, blazefitsname))
        # add keys from original header file
        hdict = spirouImage.CopyOriginalKeys(hdr)
        # define new keys to add
        hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_DRS_DATE'],
                                   value=p['DRS_DATE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_DATE_NOW'],
                                   value=p['DATE_NOW'])
        hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
        hdict = spirouImage.AddKey(p, hdict, p['KW_FIBER'], value=p['FIBER'])
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_CDBDARK'],
                                   value=p['DARKFILE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_CDBBAD'],
                                   value=p['BADPFILE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_CDBLOCO'],
                                   value=p['LOCOFILE'])
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_CDBBACK'],
                                   value=p['BKGRDFILE'])
        if p['IC_FF_EXTRACT_TYPE'] not in EXTRACT_SHAPE_TYPES:
            hdict = spirouImage.AddKey(p,
                                       hdict,
                                       p['KW_CDBTILT'],
                                       value=p['TILTFILE'])
        if p['IC_FF_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES:
            hdict = spirouImage.AddKey(p,
                                       hdict,
                                       p['KW_CDBSHAPEX'],
                                       value=p['SHAPEXFILE'])
            hdict = spirouImage.AddKey(p,
                                       hdict,
                                       p['KW_CDBSHAPEY'],
                                       value=p['SHAPEYFILE'])
            hdict = spirouImage.AddKey(p,
                                       hdict,
                                       p['KW_CDBSHAPE'],
                                       value=p['SHAPEFILE'])
            hdict = spirouImage.AddKey(p,
                                       hdict,
                                       p['KW_CDBFPMASTER'],
                                       value=p['FPMASTERFILE'])
        hdict = spirouImage.AddKey1DList(p,
                                         hdict,
                                         p['KW_INFILE1'],
                                         dim1name='file',
                                         values=p['ARG_FILE_NAMES'])
        # add some properties back
        hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_SIGDET'])
        hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_CONAD'])
        # add qc parameters
        hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
        hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
        # copy extraction method and function to header
        #     (for reproducibility)
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_E2DS_EXTM'],
                                   value=extmethod)
        hdict = spirouImage.AddKey(p, hdict, p['KW_E2DS_FUNC'], value=extfunc)
        # output keys
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_EXT_TYPE'],
                                   value=p['DPRTYPE'])
        # write 1D list of the SNR
        hdict = spirouImage.AddKey1DList(p,
                                         hdict,
                                         p['KW_EXTRA_SN'],
                                         values=loc['SNR'])
        # write center fits and add header keys (via hdict)
        p = spirouImage.WriteImage(p, blazefits, loc['BLAZE'], hdict)

        # ----------------------------------------------------------------------
        # Store Flat-field in file
        # ----------------------------------------------------------------------
        # construct filename
        flatfits, tag2 = spirouConfig.Constants.FF_FLAT_FILE(p)
        flatfitsname = os.path.split(flatfits)[-1]
        # log that we are saving blaze file
        wmsg = 'Saving FF spectrum for fiber: {0} in {1}'
        WLOG(p, '', wmsg.format(fiber, flatfitsname))
        # write 1D list of the RMS (add to hdict from blaze)
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
        hdict = spirouImage.AddKey1DList(p,
                                         hdict,
                                         p['KW_FLAT_RMS'],
                                         values=loc['RMS'])
        # write center fits and add header keys (via same hdict as blaze)
        p = spirouImage.WriteImage(p, flatfits, loc['FLAT'], hdict)

        # Save E2DSLL file
        hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
        hdict = spirouImage.AddKey(p,
                                   hdict,
                                   p['KW_EXT_TYPE'],
                                   value=p['DPRTYPE'])
        if p['IC_FF_EXTRACT_TYPE'] in EXTRACT_LL_TYPES:
            llstack = np.vstack(loc['E2DSLL'])
            p = spirouImage.WriteImage(p, e2dsllfits, llstack, hdict)

        # ------------------------------------------------------------------
        # Update the calibration database
        # ------------------------------------------------------------------
        if p['QC'] == 1:
            # copy flatfits to calibdb
            keydb = 'FLAT_' + p['FIBER']
            # copy localisation file to the calibDB folder
            spirouDB.PutCalibFile(p, flatfits)
            # update the master calib DB file with new key
            spirouDB.UpdateCalibMaster(p, keydb, flatfitsname, hdr)
            # copy blazefits to calibdb
            keydb = 'BLAZE_' + p['FIBER']
            # copy localisation file to the calibDB folder
            spirouDB.PutCalibFile(p, blazefits)
            # update the master calib DB file with new key
            spirouDB.UpdateCalibMaster(p, keydb, blazefitsname, hdr)

    # ----------------------------------------------------------------------
    # End Message
    # ----------------------------------------------------------------------
    p = spirouStartup.End(p)
    # return a copy of locally defined variables in the memory
    return dict(locals())