for ord in range(0, n_ord * 2, 2):
for coeff in range(ordfit):
i = (ord * n_coeff) + coeff
poly_c[i % n_coeff, i // n_coeff] = hdr_wave["LOCTR" + str(i)]
poly_c = poly_c[:, poly_c[0, :] != 0]
datac = (pyfits.getdata(slope_file))
hdr = pyfits.getheader(slope_file)
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p)
# convert NaN to zeros
data0 = np.where(~np.isfinite(data), np.zeros_like(data), data)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'],
xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'],
yhigh=p['IC_CCDY_HIGH'],
getshape=False)
data1 = spirouImage.ResizeImage(p, data0, **bkwargs)
# dxmap that will contain the dx per pixel
# if the file does not exist, we fill the map
# with zeros
if glob.glob(outname) != []:
print(outname + ' exists... we use its values as a starting point')
def main(night_name=None, files=None):
"""
cal_loc_RAW_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_loc_RAW_spirou.py [night_name] [files]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files)
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add')
# ----------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
data = spirouImage.FixNonPreProcess(p, data)
# ----------------------------------------------------------------------
# Get basic image properties
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# ----------------------------------------------------------------------
# Correction of DARK
# ----------------------------------------------------------------------
p, datac = spirouImage.CorrectForDark(p, data, hdr)
# ----------------------------------------------------------------------
# Interpolation over bad regions (to fill in the holes)
# ----------------------------------------------------------------------
# log process
# wmsg = 'Interpolating over bad regions'
# WLOG(p, '', wmsg)
# run interpolation
# datac = spirouImage.InterpolateBadRegions(p, datac)
# ----------------------------------------------------------------------
# Resize image
# ----------------------------------------------------------------------
# rotate the image and convert from ADU/s to e-
data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p)
# convert NaN to zeros
data0 = np.where(~np.isfinite(data), np.zeros_like(data), data)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'],
xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'],
yhigh=p['IC_CCDY_HIGH'],
getshape=False)
data2 = spirouImage.ResizeImage(p, data0, **bkwargs)
# log change in data size
WLOG(p, '', ('Image format changed to ' '{0}x{1}').format(*data2.shape))
# ----------------------------------------------------------------------
# Correct for the BADPIX mask (set all bad pixels to zero)
# ----------------------------------------------------------------------
p, data2 = spirouImage.CorrectForBadPix(p, data2, hdr)
# ----------------------------------------------------------------------
# Background computation
# ----------------------------------------------------------------------
if p['IC_DO_BKGR_SUBTRACTION']:
# log that we are doing background measurement
WLOG(p, '', 'Doing background measurement on raw frame')
# get the bkgr measurement
bargs = [p, data2, hdr]
# background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs)
p, background = spirouBACK.MeasureBackgroundMap(*bargs)
else:
background = np.zeros_like(data2)
p['BKGRDFILE'] = 'None'
p.set_source('BKGRDFILE', __NAME__ + '.main()')
# apply background correction to data
data2 = data2 - background
# ----------------------------------------------------------------------
# Construct image order_profile
# ----------------------------------------------------------------------
# log that we are doing background measurement
WLOG(p, '', 'Creating Order Profile')
order_profile = spirouLOCOR.BoxSmoothedImage(data2, p['LOC_BOX_SIZE'])
# data 2 is now set to the order profile
data2o = data2.copy()
data2 = order_profile.copy()
# ----------------------------------------------------------------------
# Write image order_profile to file
# ----------------------------------------------------------------------
# Construct folder and filename
rawfits, tag1 = spirouConfig.Constants.LOC_ORDER_PROFILE_FILE(p)
rawfitsname = os.path.split(rawfits)[-1]
# log saving order profile
wmsg = 'Saving processed raw frame in {0}'
WLOG(p, '', wmsg.format(rawfitsname))
# add keys from original header file
hdict = spirouImage.CopyOriginalKeys(hdr)
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
# write to file
p = spirouImage.WriteImage(p, rawfits, order_profile, hdict)
# ----------------------------------------------------------------------
# Move order_profile to calibDB and update calibDB
# ----------------------------------------------------------------------
# set key for calibDB
keydb = 'ORDER_PROFILE_{0}'.format(p['FIBER'])
# copy dark fits file to the calibDB folder
spirouDB.PutCalibFile(p, rawfits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, rawfitsname, hdr)
# ######################################################################
# Localization of orders on central column
# ######################################################################
# storage dictionary for localization parameters
loc = ParamDict()
# Plots setup: start interactive plot
if p['DRS_PLOT'] > 0:
sPlt.start_interactive_session(p)
# ----------------------------------------------------------------------
# Measurement and correction of background on the central column
# ----------------------------------------------------------------------
loc = spirouBACK.MeasureBkgrdGetCentPixs(p, loc, data2)
# ----------------------------------------------------------------------
# Search for order center on the central column - quick estimation
# ----------------------------------------------------------------------
# log progress
WLOG(p, '', 'Searching order center on central column')
# plot the minimum of ycc and ic_locseuil if in debug and plot mode
if p['DRS_DEBUG'] > 0 and p['DRS_PLOT']:
sPlt.debug_locplot_min_ycc_loc_threshold(p, loc['YCC'])
# find the central positions of the orders in the central
posc_all = spirouLOCOR.FindPosCentCol(loc['YCC'], p['IC_LOCSEUIL'])
# depending on the fiber type we may need to skip some pixels and also
# we need to add back on the ic_offset applied
start = p['IC_FIRST_ORDER_JUMP']
posc = posc_all[start:] + p['IC_OFFSET']
# work out the number of orders to use (minimum of ic_locnbmaxo and number
# of orders found in 'LocateCentralOrderPositions')
number_of_orders = np.min([p['IC_LOCNBMAXO'], len(posc)])
# log the number of orders than have been detected
wargs = [p['FIBER'], int(number_of_orders / p['NBFIB']), p['NBFIB']]
WLOG(p, 'info', ('On fiber {0} {1} orders have been detected '
'on {2} fiber(s)').format(*wargs))
# ----------------------------------------------------------------------
# Search for order center and profile on specific columns
# ----------------------------------------------------------------------
# get fit polynomial orders for position and width
fitpos, fitwid = p['IC_LOCDFITC'], p['IC_LOCDFITW']
# Create arrays to store position and width of order for each order
loc['CTRO'] = np.zeros((number_of_orders, data2.shape[1]), dtype=float)
loc['SIGO'] = np.zeros((number_of_orders, data2.shape[1]), dtype=float)
# Create arrays to store coefficients for position and width
loc['ACC'] = np.zeros((number_of_orders, fitpos + 1))
loc['ASS'] = np.zeros((number_of_orders, fitpos + 1))
# Create arrays to store rms values for position and width
loc['RMS_CENTER'] = np.zeros(number_of_orders)
loc['RMS_FWHM'] = np.zeros(number_of_orders)
# Create arrays to store point to point max value for position and width
loc['MAX_PTP_CENTER'] = np.zeros(number_of_orders)
loc['MAX_PTP_FRACCENTER'] = np.zeros(number_of_orders)
loc['MAX_PTP_FWHM'] = np.zeros(number_of_orders)
loc['MAX_PTP_FRACFWHM'] = np.zeros(number_of_orders)
# Create arrays to store rejected points
loc['MAX_RMPTS_POS'] = np.zeros(number_of_orders)
loc['MAX_RMPTS_WID'] = np.zeros(number_of_orders)
# set the central col centers in the cpos_orders array
loc['CTRO'][:, p['IC_CENT_COL']] = posc[0:number_of_orders]
# set source for all locs
loc.set_all_sources(__NAME__ + '/main()')
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# storage for plotting
loc['XPLOT'], loc['YPLOT'] = [], []
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# loop around each order
rorder_num = 0
for order_num in range(number_of_orders):
# find the row centers of the columns
loc = spirouLOCOR.FindOrderCtrs(p, data2, loc, order_num)
# only keep the orders with non-zero width
mask = loc['SIGO'][order_num, :] != 0
loc['X'] = np.arange(data2.shape[1])[mask]
loc.set_source('X', __NAME__ + '/main()')
# check that we have enough data points to fit data
if len(loc['X']) > (fitpos + 1):
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# initial fit params
iofargs = [p, loc, mask, rorder_num]
# initial fit for center positions for this order
loc, cf_data = spirouLOCOR.InitialOrderFit(*iofargs, kind='center')
# initial fit for widths for this order
loc, wf_data = spirouLOCOR.InitialOrderFit(*iofargs, kind='fwhm')
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Log order number and fit at central pixel and width and rms
wargs = [
rorder_num, cf_data['cfitval'], wf_data['cfitval'],
cf_data['rms']
]
WLOG(p, '', ('ORDER: {0} center at pixel {1:.1f} width '
'{2:.1f} rms {3:.3f}').format(*wargs))
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# sigma fit params
sigfargs = [p, loc, cf_data, mask, order_num, rorder_num]
# sigma clip fit for center positions for this order
cf_data = spirouLOCOR.SigClipOrderFit(*sigfargs, kind='center')
# load results into storage arrags for this order
loc['ACC'][rorder_num] = cf_data['a']
loc['RMS_CENTER'][rorder_num] = cf_data['rms']
loc['MAX_PTP_CENTER'][rorder_num] = cf_data['max_ptp']
loc['MAX_PTP_FRACCENTER'][rorder_num] = cf_data['max_ptp_frac']
loc['MAX_RMPTS_POS'][rorder_num] = cf_data['max_rmpts']
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# sigma fit params
sigfargs = [p, loc, wf_data, mask, order_num, rorder_num]
# sigma clip fit for width positions for this order
wf_data = spirouLOCOR.SigClipOrderFit(*sigfargs, kind='fwhm')
# load results into storage arrags for this order
loc['ASS'][rorder_num] = wf_data['a']
loc['RMS_FWHM'][rorder_num] = wf_data['rms']
loc['MAX_PTP_FWHM'][rorder_num] = wf_data['max_ptp']
loc['MAX_PTP_FRACFWHM'][rorder_num] = wf_data['max_ptp_frac']
loc['MAX_RMPTS_WID'][rorder_num] = wf_data['max_rmpts']
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# increase the roder_num iterator
rorder_num += 1
# else log that the order is unusable
else:
WLOG(p, '', 'Order found too much incomplete, discarded')
# ----------------------------------------------------------------------
# Plot the image (ready for fit points to be overplotted later)
if p['DRS_PLOT'] > 0:
# get saturation threshold
satseuil = p['IC_SATSEUIL'] * p['GAIN'] * p['NBFRAMES']
# plot image above saturation threshold
sPlt.locplot_im_sat_threshold(p, loc, data2, satseuil)
# ----------------------------------------------------------------------
# Log that order geometry has been measured
WLOG(p, 'info', ('On fiber {0} {1} orders geometry have been '
'measured').format(p['FIBER'], rorder_num))
# Get mean rms
mean_rms_center = np.nansum(
loc['RMS_CENTER'][:rorder_num]) * 1000 / rorder_num
mean_rms_fwhm = np.nansum(loc['RMS_FWHM'][:rorder_num]) * 1000 / rorder_num
# Log mean rms values
wmsg = 'Average uncertainty on {0}: {1:.2f} [mpix]'
WLOG(p, 'info', wmsg.format('position', mean_rms_center))
WLOG(p, 'info', wmsg.format('width', mean_rms_fwhm))
# ----------------------------------------------------------------------
# Plot of RMS for positions and widths
# ----------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
sPlt.locplot_order_number_against_rms(p, loc, rorder_num)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# check that max number of points rejected in center fit is below threshold
if np.nansum(loc['MAX_RMPTS_POS']) > p['QC_LOC_MAXLOCFIT_REMOVED_CTR']:
fmsg = 'abnormal points rejection during ctr fit ({0:.2f} > {1:.2f})'
fail_msg.append(
fmsg.format(np.nansum(loc['MAX_RMPTS_POS']),
p['QC_LOC_MAXLOCFIT_REMOVED_CTR']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(np.nansum(loc['MAX_RMPTS_POS']))
qc_names.append('sum(MAX_RMPTS_POS)')
qc_logic.append('sum(MAX_RMPTS_POS) > {0:.2f}'
''.format(p['QC_LOC_MAXLOCFIT_REMOVED_CTR']))
# ----------------------------------------------------------------------
# check that max number of points rejected in width fit is below threshold
if np.nansum(loc['MAX_RMPTS_WID']) > p['QC_LOC_MAXLOCFIT_REMOVED_WID']:
fmsg = 'abnormal points rejection during width fit ({0:.2f} > {1:.2f})'
fail_msg.append(
fmsg.format(np.nansum(loc['MAX_RMPTS_WID']),
p['QC_LOC_MAXLOCFIT_REMOVED_WID']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(np.nansum(loc['MAX_RMPTS_WID']))
qc_names.append('sum(MAX_RMPTS_WID)')
qc_logic.append('sum(MAX_RMPTS_WID) > {0:.2f}'
''.format(p['QC_LOC_MAXLOCFIT_REMOVED_WID']))
# ----------------------------------------------------------------------
# check that the rms in center fit is lower than qc threshold
if mean_rms_center > p['QC_LOC_RMSMAX_CENTER']:
fmsg = 'too high rms on center fitting ({0:.2f} > {1:.2f})'
fail_msg.append(fmsg.format(mean_rms_center,
p['QC_LOC_RMSMAX_CENTER']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(mean_rms_center)
qc_names.append('mean_rms_center')
qc_logic.append('mean_rms_center > {0:.2f}'
''.format(p['QC_LOC_RMSMAX_CENTER']))
# ----------------------------------------------------------------------
# check that the rms in center fit is lower than qc threshold
if mean_rms_fwhm > p['QC_LOC_RMSMAX_FWHM']:
fmsg = 'too high rms on profile fwhm fitting ({0:.2f} > {1:.2f})'
fail_msg.append(fmsg.format(mean_rms_fwhm, p['QC_LOC_RMSMAX_CENTER']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(mean_rms_fwhm)
qc_names.append('mean_rms_fwhm')
qc_logic.append('mean_rms_fwhm > {0:.2f}'
''.format(p['QC_LOC_RMSMAX_CENTER']))
# ----------------------------------------------------------------------
# check for abnormal number of identified orders
if rorder_num != p['QC_LOC_NBO']:
fmsg = ('abnormal number of identified orders (found {0:.2f} '
'expected {1:.2f})')
fail_msg.append(fmsg.format(rorder_num, p['QC_LOC_NBO']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(rorder_num)
qc_names.append('rorder_num')
qc_logic.append('rorder_num != {0:.2f}'.format(p['QC_LOC_NBO']))
# ----------------------------------------------------------------------
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ----------------------------------------------------------------------
# Save and record of image of localization with order center and keywords
# ----------------------------------------------------------------------
raw_loco_file = os.path.basename(p['FITSFILENAME'])
# construct filename
locofits, tag2 = spirouConfig.Constants.LOC_LOCO_FILE(p)
locofitsname = os.path.split(locofits)[-1]
# log that we are saving localization file
WLOG(p, '', ('Saving localization information '
'in file: {0}').format(locofitsname))
# add keys from original header file
hdict = spirouImage.CopyOriginalKeys(hdr)
# define new keys to add
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=raw_loco_file)
hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_SIGDET'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_CONAD'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOCO_BCKGRD'],
value=loc['MEAN_BACKGRD'])
hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_NBO'], value=rorder_num)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOCO_DEG_C'],
value=p['IC_LOCDFITC'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOCO_DEG_W'],
value=p['IC_LOCDFITW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_E'])
hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DELTA'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_MAXFLX'],
value=float(loc['MAX_SIGNAL']))
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_SMAXPTS_CTR'],
value=np.nansum(loc['MAX_RMPTS_POS']))
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_SMAXPTS_WID'],
value=np.nansum(loc['MAX_RMPTS_WID']))
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_RMS_CTR'],
value=mean_rms_center)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_RMS_WID'],
value=mean_rms_fwhm)
# write 2D list of position fit coefficients
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_LOCO_CTR_COEFF'],
values=loc['ACC'][0:rorder_num])
# write 2D list of width fit coefficients
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_LOCO_FWHM_COEFF'],
values=loc['ASS'][0:rorder_num])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# write center fits and add header keys (via hdict)
center_fits = spirouLOCOR.CalcLocoFits(loc['ACC'], data2.shape[1])
p = spirouImage.WriteImage(p, locofits, center_fits, hdict)
# ----------------------------------------------------------------------
# Save and record of image of sigma
# ----------------------------------------------------------------------
# construct filename
locofits2, tag3 = spirouConfig.Constants.LOC_LOCO_FILE2(p)
locofits2name = os.path.split(locofits2)[-1]
# log that we are saving localization file
wmsg = 'Saving FWHM information in file: {0}'
WLOG(p, '', wmsg.format(locofits2name))
# add keys from original header file
hdict = spirouImage.CopyOriginalKeys(hdr)
# define new keys to add
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBACK'], value=p['BKGRDFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='file',
values=p['ARG_FILE_NAMES'])
# add outputs
hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_SIGDET'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_CONAD'])
hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_NBO'], value=rorder_num)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOCO_DEG_C'],
value=p['IC_LOCDFITC'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOCO_DEG_W'],
value=p['IC_LOCDFITW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_E'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_MAXFLX'],
value=float(loc['MAX_SIGNAL']))
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_SMAXPTS_CTR'],
value=np.nansum(loc['MAX_RMPTS_POS']))
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_SMAXPTS_WID'],
value=np.nansum(loc['MAX_RMPTS_WID']))
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_RMS_CTR'],
value=mean_rms_center)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_RMS_WID'],
value=mean_rms_fwhm)
# write 2D list of position fit coefficients
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_LOCO_CTR_COEFF'],
values=loc['ACC'][0:rorder_num])
# write 2D list of width fit coefficients
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_LOCO_FWHM_COEFF'],
values=loc['ASS'][0:rorder_num])
# add quality control
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
# write image and add header keys (via hdict)
width_fits = spirouLOCOR.CalcLocoFits(loc['ASS'], data2.shape[1])
p = spirouImage.WriteImage(p, locofits2, width_fits, hdict)
# ----------------------------------------------------------------------
# Save and Record of image of localization
# ----------------------------------------------------------------------
if p['IC_LOCOPT1']:
# construct filename
locofits3, tag4 = spirouConfig.Constants.LOC_LOCO_FILE3(p)
locofits3name = os.path.split(locofits3)[-1]
# log that we are saving localization file
wmsg1 = 'Saving localization image with superposition of orders in '
wmsg2 = 'file: {0}'.format(locofits3name)
WLOG(p, '', [wmsg1, wmsg2])
# superpose zeros over the fit in the image
data4 = spirouLOCOR.ImageLocSuperimp(data2o, loc['ACC'][0:rorder_num])
# save this image to file
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_FIBER'], value=p['FIBER'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBDARK'],
value=p['DARKFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBBAD'],
value=p['BADPFILE'])
p = spirouImage.WriteImage(p, locofits3, data4, hdict)
# ----------------------------------------------------------------------
# Update the calibration database
# ----------------------------------------------------------------------
if p['QC'] == 1:
keydb = 'LOC_' + p['FIBER']
# copy localisation file to the calibDB folder
spirouDB.PutCalibFile(p, locofits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, locofitsname, hdr)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
"""
cal_SLIT_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_SLIT_spirou.py [night_directory] [files]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files)
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# set the fiber type
p['FIB_TYP'] = 'AB'
p.set_source('FIB_TYP', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add')
# ----------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
data = spirouImage.FixNonPreProcess(p, data)
# ----------------------------------------------------------------------
# Get basic image properties
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# ----------------------------------------------------------------------
# Correction of DARK
# ----------------------------------------------------------------------
p, datac = spirouImage.CorrectForDark(p, data, hdr)
# ----------------------------------------------------------------------
# Resize image
# ----------------------------------------------------------------------
# rotate the image and convert from ADU/s to e-
data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p)
# convert NaN to zeros
data0 = np.where(~np.isfinite(data), np.zeros_like(data), data)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'],
getshape=False)
data2 = spirouImage.ResizeImage(p, data0, **bkwargs)
# log change in data size
WLOG(p, '', ('Image format changed to '
'{0}x{1}').format(*data2.shape))
# ----------------------------------------------------------------------
# Correct for the BADPIX mask (set all bad pixels to zero)
# ----------------------------------------------------------------------
p, data2 = spirouImage.CorrectForBadPix(p, data2, hdr)
# ----------------------------------------------------------------------
# Background computation
# ----------------------------------------------------------------------
if p['IC_DO_BKGR_SUBTRACTION']:
# log that we are doing background measurement
WLOG(p, '', 'Doing background measurement on raw frame')
# get the bkgr measurement
bargs = [p, data2, hdr]
# background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs)
p, background = spirouBACK.MeasureBackgroundMap(*bargs)
else:
background = np.zeros_like(data2)
p['BKGRDFILE'] = 'None'
p.set_source('BKGRDFILE', __NAME__ + '.main()')
# correct data2 with background
data2 = data2 - background
# ----------------------------------------------------------------------
# Log the number of dead pixels
# ----------------------------------------------------------------------
# get the number of bad pixels
n_bad_pix = np.nansum(~np.isfinite(data2))
n_bad_pix_frac = n_bad_pix * 100 / np.product(data2.shape)
# Log number
wmsg = 'Nb dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# ----------------------------------------------------------------------
# Log the number of dead pixels
# ----------------------------------------------------------------------
loc = ParamDict()
# ----------------------------------------------------------------------
# Loop around fiber types
# ----------------------------------------------------------------------
# set fiber
p['FIBER'] = p['FIB_TYP']
# ------------------------------------------------------------------
# Get localisation coefficients
# ------------------------------------------------------------------
# original there is a loop but it is not used --> removed
p = spirouImage.FiberParams(p, p['FIBER'], merge=True)
# get localisation fit coefficients
p, loc = spirouLOCOR.GetCoeffs(p, hdr, loc)
# ------------------------------------------------------------------
# Calculating the tilt
# ------------------------------------------------------------------
# get the tilt by extracting the AB fibers and correlating them
loc = spirouImage.GetTilt(p, loc, data2)
# fit the tilt with a polynomial
loc = spirouImage.FitTilt(p, loc)
# log the tilt dispersion
wmsg = 'Tilt dispersion = {0:.3f} deg'
WLOG(p, 'info', wmsg.format(loc['RMS_TILT']))
# ------------------------------------------------------------------
# Plotting
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# plots setup: start interactive plot
sPlt.start_interactive_session(p)
# plot image with selected order shown
sPlt.slit_sorder_plot(p, loc, data2)
# plot slit tilt angle and fit
sPlt.slit_tilt_angle_and_fit_plot(p, loc)
# end interactive section
sPlt.end_interactive_session(p)
# ------------------------------------------------------------------
# Replace tilt by the global fit
# ------------------------------------------------------------------
loc['TILT'] = loc['YFIT_TILT']
oldsource = loc.get_source('tilt')
loc.set_source('TILT', oldsource + '+{0}/main()'.format(__NAME__))
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# check that tilt rms is below required
if loc['RMS_TILT'] > p['QC_SLIT_RMS']:
# add failed message to fail message list
fmsg = 'abnormal RMS of SLIT angle ({0:.2f} > {1:.2f} deg)'
fail_msg.append(fmsg.format(loc['RMS_TILT'], p['QC_SLIT_RMS']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(loc['RMS_TILT'])
qc_names.append('RMS_TILT')
qc_logic.append('RMS_TILT > {0:.2f}'.format(p['QC_SLIT_RMS']))
# ----------------------------------------------------------------------
# check that tilt is less than max tilt required
max_tilt = np.max(loc['TILT'])
if max_tilt > p['QC_SLIT_MAX']:
# add failed message to fail message list
fmsg = 'abnormal SLIT angle ({0:.2f} > {1:.2f} deg)'
fail_msg.append(fmsg.format(max_tilt, p['QC_SLIT_MAX']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(max_tilt)
qc_names.append('max_tilt')
qc_logic.append('max_tilt > {0:.2f}'.format(p['QC_SLIT_MAX']))
# ----------------------------------------------------------------------
# check that tilt is greater than min tilt required
min_tilt = np.min(loc['TILT'])
if min_tilt < p['QC_SLIT_MIN']:
# add failed message to fail message list
fmsg = 'abnormal SLIT angle ({0:.2f} < {1:.2f} deg)'
fail_msg.append(fmsg.format(max_tilt, p['QC_SLIT_MIN']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(min_tilt)
qc_names.append('min_tilt')
qc_logic.append('min_tilt > {0:.2f}'.format(p['QC_SLIT_MIN']))
# ----------------------------------------------------------------------
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ----------------------------------------------------------------------
# Save and record of tilt table
# ----------------------------------------------------------------------
# copy the tilt along the orders
tiltima = np.ones((int(loc['NUMBER_ORDERS']/2), data2.shape[1]))
tiltima *= loc['TILT'][:, None]
# get the raw tilt file name
raw_tilt_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
tiltfits, tag = spirouConfig.Constants.SLIT_TILT_FILE(p)
tiltfitsname = os.path.basename(tiltfits)
# Log that we are saving tilt file
wmsg = 'Saving tilt information in file: {0}'
WLOG(p, '', wmsg.format(tiltfitsname))
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(hdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'],
value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'],
value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBACK'],
value=p['BKGRDFILE'])
hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file',
values=p['ARG_FILE_NAMES'])
# add qc parameters
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# add tilt parameters as 1d list
hdict = spirouImage.AddKey1DList(p, hdict, p['KW_TILT'], values=loc['TILT'])
# write tilt file to file
p = spirouImage.WriteImage(p, tiltfits, tiltima, hdict)
# ----------------------------------------------------------------------
# Update the calibration data base
# ----------------------------------------------------------------------
if p['QC']:
keydb = 'TILT'
# copy localisation file to the calibDB folder
spirouDB.PutCalibFile(p, tiltfits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, tiltfitsname, hdr)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
"""
cal_shape_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_shape_spirou.py [night_directory] [fitsfilename]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files)
p = spirouStartup.InitialFileSetup(p)
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add')
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# ----------------------------------------------------------------------
# Get basic image properties for FP file
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# ----------------------------------------------------------------------
# Correction of reference FP
# ----------------------------------------------------------------------
# set the number of frames
p['NBFRAMES'] = 1
p.set_source('NBFRAMES', __NAME__ + '.main()')
# Correction of DARK
p, datac = spirouImage.CorrectForDark(p, data, hdr)
# Resize hc data
# rotate the image and convert from ADU/s to e-
data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'],
getshape=False)
data1 = spirouImage.ResizeImage(p, data, **bkwargs)
# log change in data size
WLOG(p, '',
('FPref Image format changed to {0}x{1}').format(*data1.shape))
# Correct for the BADPIX mask (set all bad pixels to zero)
bargs = [p, data1, hdr]
p, data1 = spirouImage.CorrectForBadPix(*bargs)
p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True)
# log progress
WLOG(p, '', 'Cleaning FPref hot pixels')
# correct hot pixels
data1 = spirouEXTOR.CleanHotpix(data1, badpixmask)
# Log the number of dead pixels
# get the number of bad pixels
with warnings.catch_warnings(record=True) as _:
n_bad_pix = np.nansum(data1 <= 0)
n_bad_pix_frac = n_bad_pix * 100 / np.product(data1.shape)
# Log number
wmsg = 'Nb FPref dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# ----------------------------------------------------------------------
# Get master FP file
# ----------------------------------------------------------------------
# log progress
WLOG(p, '', 'Getting FP Master from calibDB')
# get master fp
p, masterfp = spirouImage.GetFPMaster(p, hdr)
# ----------------------------------------------------------------------
# Get transform parameters
# ----------------------------------------------------------------------
# log progress
wargs = [p['ARG_FILE_NAMES'][0], p['FPMASTERFILE']]
WLOG(p, 'info', 'Calculating transforming for {0} onto {1}'.format(*wargs))
gout = spirouImage.GetLinearTransformParams(p, masterfp, data1)
transform, xres, yres = gout
# ------------------------------------------------------------------
# Need to straighten the fp data for debug
# ------------------------------------------------------------------
p, shapem_x = spirouImage.GetShapeX(p, hdr)
p, shapem_y = spirouImage.GetShapeY(p, hdr)
data2 = spirouImage.EATransform(data1, transform, dxmap=shapem_x,
dymap=shapem_y)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# if transform is None means the fp image quality was too poor
if transform is None:
fail_msg.append('FP Image quality too poor (sigma clip failed)')
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
qc_values.append('None')
qc_names.append('Image Quality')
qc_logic.append('Image too poor')
# ----------------------------------------------------------------------
# get residual qc parameter
qc_res = p['SHAPE_QC_LINEAR_TRANS_RES_THRES']
# assess quality of x residuals
if xres > qc_res:
fail_msg.append('x-resdiuals too high {0} > {1}'.format(xres, qc_res))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
qc_values.append(xres)
qc_names.append('XRES')
qc_logic.append('XRES > {0}'.format(qc_res))
# assess quality of x residuals
if yres > qc_res:
fail_msg.append('y-resdiuals too high {0} > {1}'.format(yres, qc_res))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
qc_values.append(yres)
qc_names.append('YRES')
qc_logic.append('YRES > {0}'.format(qc_res))
# ----------------------------------------------------------------------
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Writing shape to file
# ------------------------------------------------------------------
# get the raw tilt file name
raw_shape_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
shapefits, tag = spirouConfig.Constants.SLIT_SHAPE_LOCAL_FILE(p)
shapefitsname = os.path.basename(shapefits)
# Log that we are saving tilt file
wmsg = 'Saving shape information in file: {0}'
WLOG(p, '', wmsg.format(shapefitsname))
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(hdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEX'],
value=p['SHAPEXFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEY'],
value=p['SHAPEYFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFPMASTER'],
value=p['FPMASTERFILE'])
hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file',
values=p['ARG_FILE_NAMES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# add the transform parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_DX'], value=transform[0])
hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_DY'], value=transform[1])
hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_A'], value=transform[2])
hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_B'], value=transform[3])
hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_C'], value=transform[4])
hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_D'], value=transform[5])
# write tilt file to file
p = spirouImage.WriteImage(p, shapefits, [transform], hdict)
# ----------------------------------------------------------------------
# Move to calibDB and update calibDB
# ----------------------------------------------------------------------
if p['QC']:
# add shape
keydb = 'SHAPE'
# copy shape file to the calibDB folder
spirouDB.PutCalibFile(p, shapefits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, shapefitsname, hdr)
# ------------------------------------------------------------------
# Writing sanity check files
# ------------------------------------------------------------------
if p['SHAPE_DEBUG_OUTPUTS']:
# log
WLOG(p, '', 'Saving debug sanity check files')
# construct file names
dargs = [p, p['ARG_FILE_NAMES'][0]]
out1 = spirouConfig.Constants.SLIT_SHAPE_IN_FP_FILE(*dargs)
input_fp_file, tag1= out1
out2 = spirouConfig.Constants.SLIT_SHAPE_OUT_FP_FILE(*dargs)
output_fp_file, tag2 = out2
# write input fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
p = spirouImage.WriteImage(p, input_fp_file, data1, hdict)
# write output fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
p = spirouImage.WriteImage(p, output_fp_file, data2, hdict)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
"""
cal_SLIT_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_SLIT_spirou.py [night_directory] [files]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files)
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# set the fiber type
p['FIBER'] = 'AB'
p.set_source('FIBER', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add')
# ----------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
data = spirouImage.FixNonPreProcess(p, data)
# ----------------------------------------------------------------------
# Get basic image properties
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# ----------------------------------------------------------------------
# Correction of DARK
# ----------------------------------------------------------------------
p, datac = spirouImage.CorrectForDark(p, data, hdr)
datac = data
# ----------------------------------------------------------------------
# Resize image
# ----------------------------------------------------------------------
# rotate the image and convert from ADU/s to e-
data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p)
# convert NaN to zeros
data0 = np.where(~np.isfinite(data), np.zeros_like(data), data)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'],
getshape=False)
data2 = spirouImage.ResizeImage(p, data0, **bkwargs)
# log change in data size
WLOG(p, '', ('Image format changed to '
'{0}x{1}').format(*data2.shape))
# ----------------------------------------------------------------------
# Correct for the BADPIX mask (set all bad pixels to zero)
# ----------------------------------------------------------------------
p, data2 = spirouImage.CorrectForBadPix(p, data2, hdr)
p, badpixmap = spirouImage.CorrectForBadPix(p, data2, hdr, return_map=True)
# ----------------------------------------------------------------------
# Background computation
# ----------------------------------------------------------------------
if p['IC_DO_BKGR_SUBTRACTION']:
# log that we are doing background measurement
WLOG(p, '', 'Doing background measurement on raw frame')
# get the bkgr measurement
bargs = [p, data2, hdr, badpixmap]
# background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs)
p, background = spirouBACK.MeasureBackgroundMap(*bargs)
else:
background = np.zeros_like(data2)
p['BKGRDFILE'] = 'None'
p.set_source('BKGRDFILE', __NAME__ + '.main()')
# apply background correction to data
data2 = data2 - background
# save data to loc
loc = ParamDict()
loc['DATA'] = data2
loc.set_source('DATA', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Log the number of dead pixels
# ----------------------------------------------------------------------
# get the number of bad pixels
n_bad_pix = np.nansum(data2 <= 0)
n_bad_pix_frac = n_bad_pix * 100 / np.product(data2.shape)
# Log number
wmsg = 'Nb dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# ------------------------------------------------------------------
# Get localisation coefficients
# ------------------------------------------------------------------
# original there is a loop but it is not used --> removed
p = spirouImage.FiberParams(p, p['FIBER'], merge=True)
# get localisation fit coefficients
p, loc = spirouLOCOR.GetCoeffs(p, hdr, loc)
# ------------------------------------------------------------------
# Calculate shape map
# ------------------------------------------------------------------
loc = spirouImage.GetShapeMap(p, loc)
# ------------------------------------------------------------------
# Plotting
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# plots setup: start interactive plot
sPlt.start_interactive_session(p)
# plot the shape process for each order
sPlt.slit_shape_angle_plot(p, loc)
# end interactive section
sPlt.end_interactive_session(p)
# ------------------------------------------------------------------
# Writing to file
# ------------------------------------------------------------------
# get the raw tilt file name
raw_shape_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
shapefits, tag = spirouConfig.Constants.SLIT_XSHAPE_FILE(p)
shapefitsname = os.path.basename(shapefits)
# Log that we are saving tilt file
wmsg = 'Saving shape information in file: {0}'
WLOG(p, '', wmsg.format(shapefitsname))
# Copy keys from fits file
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(hdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPE'], value=raw_shape_file)
# write tilt file to file
p = spirouImage.WriteImage(p, shapefits, loc['DXMAP'], hdict)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# TODO: Decide on some quality control criteria?
# set passed variable and fail message list
passed, fail_msg = True, []
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Move to calibDB and update calibDB
# ----------------------------------------------------------------------
if p['QC']:
keydb = 'SHAPE'
# copy shape file to the calibDB folder
spirouDB.PutCalibFile(p, shapefits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, shapefitsname, hdr)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, hcfile=None, fpfiles=None):
"""
cal_SLIT_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_SLIT_spirou.py [night_directory] [files]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
if hcfile is None or fpfiles is None:
names, types = ['hcfile', 'fpfiles'], [str, str]
customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1],
types,
names,
last_multi=True)
else:
customargs = dict(hcfile=hcfile, fpfiles=fpfiles)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsfile='fpfiles')
# ----------------------------------------------------------------------
# Construct reference filename and get fiber type
# ----------------------------------------------------------------------
p, hcfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['HCFILE'])
p, fpfilenames = spirouStartup.MultiFileSetup(p, files=p['FPFILES'])
# set fiber (it doesn't matter with the 2D image but we need this to get
# the lamp type for FPFILES and HCFILES, AB == C
p['FIBER'] = 'AB'
p['FIB_TYP'] = [p['FIBER']]
fsource = __NAME__ + '/main()'
p.set_sources(['FIBER', 'FIB_TYP'], fsource)
# set the hcfilename to the first hcfilenames
fpfitsfilename = fpfilenames[0]
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# add a force plot off
p['PLOT_PER_ORDER'] = PLOT_PER_ORDER
p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()')
# ----------------------------------------------------------------------
# Read FP and HC files
# ----------------------------------------------------------------------
# read and combine all FP files except the first (fpfitsfilename)
rargs = [p, 'add', fpfitsfilename, fpfilenames[1:]]
p, fpdata, fphdr = spirouImage.ReadImageAndCombine(*rargs)
# read first file (hcfitsfilename)
hcdata, hchdr, _, _ = spirouImage.ReadImage(p, hcfitsfilename)
# add data and hdr to loc
loc = ParamDict()
loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr
loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr
# set the source
sources = ['HCDATA', 'HCHDR']
loc.set_sources(sources, 'spirouImage.ReadImageAndCombine()')
sources = ['FPDATA', 'FPHDR']
loc.set_sources(sources, 'spirouImage.ReadImage()')
# ---------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
hcdata = spirouImage.FixNonPreProcess(p, hcdata)
fpdata = spirouImage.FixNonPreProcess(p, fpdata)
# ----------------------------------------------------------------------
# Get basic image properties for reference file
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, fphdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, fphdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, fphdr, name='gain')
# get lamp parameters
p = spirouTHORCA.GetLampParams(p, hchdr)
# ----------------------------------------------------------------------
# Correction of DARK
# ----------------------------------------------------------------------
# p, hcdatac = spirouImage.CorrectForDark(p, hcdata, hchdr)
hcdatac = hcdata
p['DARKFILE'] = 'None'
# p, fpdatac = spirouImage.CorrectForDark(p, fpdata, fphdr)
fpdatac = fpdata
# ----------------------------------------------------------------------
# Resize hc data
# ----------------------------------------------------------------------
# rotate the image and convert from ADU/s to e-
hcdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, hcdatac), p=p)
# convert NaN to zeros
hcdata0 = np.where(~np.isfinite(hcdata), np.zeros_like(hcdata), hcdata)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'],
xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'],
yhigh=p['IC_CCDY_HIGH'],
getshape=False)
hcdata2 = spirouImage.ResizeImage(p, hcdata0, **bkwargs)
# log change in data size
WLOG(p, '', ('HC Image format changed to '
'{0}x{1}').format(*hcdata2.shape))
# ----------------------------------------------------------------------
# Resize fp data
# ----------------------------------------------------------------------
# rotate the image and convert from ADU/s to e-
fpdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, fpdatac), p=p)
# convert NaN to zeros
fpdata0 = np.where(~np.isfinite(fpdata), np.zeros_like(fpdata), fpdata)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'],
xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'],
yhigh=p['IC_CCDY_HIGH'],
getshape=False)
fpdata2 = spirouImage.ResizeImage(p, fpdata0, **bkwargs)
# log change in data size
WLOG(p, '', ('FP Image format changed to '
'{0}x{1}').format(*fpdata2.shape))
# ----------------------------------------------------------------------
# Correct for the BADPIX mask (set all bad pixels to zero)
# ----------------------------------------------------------------------
# p, hcdata2 = spirouImage.CorrectForBadPix(p, hcdata2, hchdr)
# p, fpdata2 = spirouImage.CorrectForBadPix(p, fpdata2, fphdr)
p['BADPFILE'] = 'None'
# save data to loc
loc['HCDATA'] = hcdata2
loc.set_source('HCDATA', __NAME__ + '/main()')
# save data to loc
loc['FPDATA'] = fpdata2
loc.set_source('FPDATA', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Log the number of dead pixels
# ----------------------------------------------------------------------
# get the number of bad pixels
n_bad_pix = np.nansum(hcdata2 <= 0)
n_bad_pix_frac = n_bad_pix * 100 / np.product(hcdata2.shape)
# Log number
wmsg = 'Nb HC dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# ----------------------------------------------------------------------
# Log the number of dead pixels
# ----------------------------------------------------------------------
# get the number of bad pixels
n_bad_pix = np.nansum(fpdata2 <= 0)
n_bad_pix_frac = n_bad_pix * 100 / np.product(fpdata2.shape)
# Log number
wmsg = 'Nb FP dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# ------------------------------------------------------------------
# Get localisation coefficients
# ------------------------------------------------------------------
# original there is a loop but it is not used --> removed
p = spirouImage.FiberParams(p, p['FIBER'], merge=True)
# get localisation fit coefficients
p, loc = spirouLOCOR.GetCoeffs(p, fphdr, loc)
# ------------------------------------------------------------------
# Get master wave solution map
# ------------------------------------------------------------------
# get master wave map
masterwavefile = spirouDB.GetDatabaseMasterWave(p)
# log process
wmsg1 = 'Getting master wavelength grid'
wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile))
WLOG(p, '', [wmsg1, wmsg2])
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# read master wave map
wout = spirouImage.GetWaveSolution(p,
filename=masterwavefile,
return_wavemap=True,
quiet=True,
return_header=True,
fiber=wave_fiber)
loc['MASTERWAVEP'], loc['MASTERWAVE'] = wout[:2]
loc['MASTERWAVEHDR'], loc['WSOURCE'] = wout[2:]
# set sources
wsource = ['MASTERWAVEP', 'MASTERWAVE', 'MASTERWAVEHDR']
loc.set_sources(wsource, 'spirouImage.GetWaveSolution()')
# ----------------------------------------------------------------------
# Read UNe solution
# ----------------------------------------------------------------------
wave_u_ne, amp_u_ne = spirouImage.ReadLineList(p)
loc['LL_LINE'], loc['AMPL_LINE'] = wave_u_ne, amp_u_ne
source = __NAME__ + '.main() + spirouImage.ReadLineList()'
loc.set_sources(['LL_LINE', 'AMPL_LINE'], source)
# ----------------------------------------------------------------------
# Read cavity length file
# ----------------------------------------------------------------------
loc['CAVITY_LEN_COEFFS'] = spirouImage.ReadCavityLength(p)
source = __NAME__ + '.main() + spirouImage.ReadCavityLength()'
loc.set_source('CAVITY_LEN_COEFFS', source)
# ------------------------------------------------------------------
# Calculate shape map
# ------------------------------------------------------------------
loc = spirouImage.GetShapeMap(p, loc)
# ------------------------------------------------------------------
# Plotting
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# plots setup: start interactive plot
sPlt.start_interactive_session(p)
# plot the shape process for one order
sPlt.slit_shape_angle_plot(p, loc)
# end interactive section
sPlt.end_interactive_session(p)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# TODO: Decide on some quality control criteria?
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# add to qc header lists
qc_values.append('None')
qc_names.append('None')
qc_logic.append('None')
qc_pass.append(1)
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Writing DXMAP to file
# ------------------------------------------------------------------
# get the raw tilt file name
raw_shape_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
shapefits, tag = spirouConfig.Constants.SLIT_XSHAPE_FILE(p)
shapefitsname = os.path.basename(shapefits)
# Log that we are saving tilt file
wmsg = 'Saving shape information in file: {0}'
WLOG(p, '', wmsg.format(shapefitsname))
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(fphdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='hcfile',
values=p['HCFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE2'],
dim1name='fpfile',
values=p['FPFILES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# write tilt file to file
p = spirouImage.WriteImage(p, shapefits, loc['DXMAP'], hdict)
# ------------------------------------------------------------------
# Writing sanity check files
# ------------------------------------------------------------------
if p['SHAPE_DEBUG_OUTPUTS']:
# log
WLOG(p, '', 'Saving debug sanity check files')
# construct file names
input_fp_file, tag1 = spirouConfig.Constants.SLIT_SHAPE_IN_FP_FILE(p)
output_fp_file, tag2 = spirouConfig.Constants.SLIT_SHAPE_OUT_FP_FILE(p)
input_hc_file, tag3 = spirouConfig.Constants.SLIT_SHAPE_IN_HC_FILE(p)
output_hc_file, tag4 = spirouConfig.Constants.SLIT_SHAPE_OUT_HC_FILE(p)
overlap_file, tag5 = spirouConfig.Constants.SLIT_SHAPE_OVERLAP_FILE(p)
# write input fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
p = spirouImage.WriteImage(p, input_fp_file, loc['FPDATA'], hdict)
# write output fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
p = spirouImage.WriteImage(p, output_fp_file, loc['FPDATA2'], hdict)
# write input fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
p = spirouImage.WriteImage(p, input_hc_file, loc['HCDATA'], hdict)
# write output fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
p = spirouImage.WriteImage(p, output_hc_file, loc['HCDATA2'], hdict)
# write overlap file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag5)
p = spirouImage.WriteImage(p, overlap_file, loc['ORDER_OVERLAP'],
hdict)
# ----------------------------------------------------------------------
# Move to calibDB and update calibDB
# ----------------------------------------------------------------------
if p['QC']:
keydb = 'SHAPE'
# copy shape file to the calibDB folder
spirouDB.PutCalibFile(p, shapefits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, shapefitsname, fphdr)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, hcfile=None, fpfiles=None):
"""
cal_SLIT_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_SLIT_spirou.py [night_directory] [files]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
if hcfile is None or fpfiles is None:
names, types = ['hcfile', 'fpfiles'], [str, str]
customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1],
types,
names,
last_multi=True)
else:
customargs = dict(hcfile=hcfile, fpfile=fpfiles)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsfile='fpfiles')
# ----------------------------------------------------------------------
# Construct reference filename and get fiber type
# ----------------------------------------------------------------------
p, hcfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['HCFILE'])
p, fpfitsfiles = spirouStartup.MultiFileSetup(p, files=p['FPFILES'])
# set fiber (it doesn't matter with the 2D image but we need this to get
# the lamp type for FPFILES and HCFILES, AB == C
p['FIBER'] = 'AB'
p['FIB_TYP'] = [p['FIBER']]
fsource = __NAME__ + '/main()'
p.set_sources(['FIBER', 'FIB_TYP'], fsource)
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# add a force plot off
p['PLOT_PER_ORDER'] = PLOT_PER_ORDER
p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()')
# ----------------------------------------------------------------------
# Read FP and HC files
# ----------------------------------------------------------------------
# read input fp and hc data
rkwargs = dict(filename=fpfitsfiles[0],
filenames=fpfitsfiles[1:],
framemath='add')
p, fpdata, fphdr = spirouImage.ReadImageAndCombine(p, **rkwargs)
hcdata, hchdr, _, _ = spirouImage.ReadImage(p, hcfitsfilename)
# add data and hdr to loc
loc = ParamDict()
loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr
loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr
# set the source
sources = ['HCDATA', 'HCHDR']
loc.set_sources(sources, 'spirouImage.ReadImage()')
sources = ['FPDATA', 'FPHDR']
loc.set_sources(sources, 'spirouImage.ReadImage()')
# ---------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
hcdata = spirouImage.FixNonPreProcess(p, hcdata)
fpdata = spirouImage.FixNonPreProcess(p, fpdata)
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# add a force plot off
p['PLOT_PER_ORDER'] = PLOT_PER_ORDER
p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()')
# ----------------------------------------------------------------------
# Get basic image properties for reference file
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, fphdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, fphdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, fphdr, name='gain')
# get lamp parameters
p = spirouTHORCA.GetLampParams(p, hchdr)
# get FP_FP DPRTYPE
p = spirouImage.ReadParam(p, fphdr, 'KW_DPRTYPE', 'DPRTYPE', dtype=str)
# ----------------------------------------------------------------------
# Correction of reference FP
# ----------------------------------------------------------------------
# set the number of frames
p['NBFRAMES'] = len(fpfitsfiles)
p.set_source('NBFRAMES', __NAME__ + '.main()')
# Correction of DARK
p, fpdatac = spirouImage.CorrectForDark(p, fpdata, fphdr)
# Resize hc data
# rotate the image and convert from ADU/s to e-
fpdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, fpdatac), p=p)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'],
xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'],
yhigh=p['IC_CCDY_HIGH'],
getshape=False)
fpdata1 = spirouImage.ResizeImage(p, fpdata, **bkwargs)
# log change in data size
WLOG(p, '',
('FPref Image format changed to {0}x{1}').format(*fpdata1.shape))
# Correct for the BADPIX mask (set all bad pixels to zero)
bargs = [p, fpdata1, fphdr]
p, fpdata1 = spirouImage.CorrectForBadPix(*bargs)
p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True)
# log progress
WLOG(p, '', 'Cleaning FPref hot pixels')
# correct hot pixels
fpdata1 = spirouEXTOR.CleanHotpix(fpdata1, badpixmask)
# add to loc
loc['FPDATA1'] = fpdata1
loc.set_source('FPDATA1', __NAME__ + '.main()')
# Log the number of dead pixels
# get the number of bad pixels
with warnings.catch_warnings(record=True) as _:
n_bad_pix = np.nansum(fpdata1 <= 0)
n_bad_pix_frac = n_bad_pix * 100 / np.product(fpdata1.shape)
# Log number
wmsg = 'Nb FPref dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# ----------------------------------------------------------------------
# Correction of HC
# ----------------------------------------------------------------------
# set the number of frames
p['NBFRAMES'] = 1
p.set_source('NBFRAMES', __NAME__ + '.main()')
# Correction of DARK
p, hcdatac = spirouImage.CorrectForDark(p, hcdata, hchdr)
# Resize hc data
# rotate the image and convert from ADU/s to e-
hcdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, hcdatac), p=p)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'],
xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'],
yhigh=p['IC_CCDY_HIGH'],
getshape=False)
hcdata1 = spirouImage.ResizeImage(p, hcdata, **bkwargs)
# log change in data size
WLOG(p, '', ('HC Image format changed to {0}x{1}').format(*hcdata1.shape))
# Correct for the BADPIX mask (set all bad pixels to zero)
bargs = [p, hcdata1, hchdr]
p, hcdata1 = spirouImage.CorrectForBadPix(*bargs)
p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True)
# log progress
WLOG(p, '', 'Cleaning HC hot pixels')
# correct hot pixels
hcdata1 = spirouEXTOR.CleanHotpix(hcdata1, badpixmask)
# add to loc
loc['HCDATA1'] = hcdata1
loc.set_source('HCDATA1', __NAME__ + '.main()')
# Log the number of dead pixels
# get the number of bad pixels
with warnings.catch_warnings(record=True) as _:
n_bad_pix = np.nansum(hcdata1 <= 0)
n_bad_pix_frac = n_bad_pix * 100 / np.product(hcdata1.shape)
# Log number
wmsg = 'Nb HC dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# -------------------------------------------------------------------------
# get all FP_FP files
# -------------------------------------------------------------------------
fpfilenames = spirouImage.FindFiles(p,
filetype=p['DPRTYPE'],
allowedtypes=p['ALLOWED_FP_TYPES'])
# convert filenames to a numpy array
fpfilenames = np.array(fpfilenames)
# julian date to know which file we need to
# process together
fp_time = np.zeros(len(fpfilenames))
basenames, fp_exp, fp_pp_version, nightnames = [], [], [], []
# log progress
WLOG(p, '', 'Reading all fp file headers')
# looping through the file headers
for it in range(len(fpfilenames)):
# log progress
wmsg = '\tReading file {0} / {1}'
WLOG(p, 'info', wmsg.format(it + 1, len(fpfilenames)))
# get fp filename
fpfilename = fpfilenames[it]
# get night name
night_name = os.path.dirname(fpfilenames[it]).split(p['TMP_DIR'])[-1]
# read data
data_it, hdr_it, _, _ = spirouImage.ReadImage(p, fpfilename)
# get header
hdr = spirouImage.ReadHeader(p, filepath=fpfilenames[it])
# add MJDATE to dark times
fp_time[it] = float(hdr[p['KW_ACQTIME'][0]])
# add other keys (for tabular output)
basenames.append(os.path.basename(fpfilenames[it]))
nightnames.append(night_name)
fp_exp.append(float(hdr[p['KW_EXPTIME'][0]]))
fp_pp_version.append(hdr[p['KW_PPVERSION'][0]])
# -------------------------------------------------------------------------
# match files by date
# -------------------------------------------------------------------------
# log progress
wmsg = 'Matching FP files by observation time (+/- {0} hrs)'
WLOG(p, '', wmsg.format(p['DARK_MASTER_MATCH_TIME']))
# get the time threshold
time_thres = p['FP_MASTER_MATCH_TIME']
# get items grouped by time
matched_id = spirouImage.GroupFilesByTime(p, fp_time, time_thres)
# -------------------------------------------------------------------------
# construct the master fp file (+ correct for dark/badpix)
# -------------------------------------------------------------------------
cargs = [fpdata1, fpfilenames, matched_id]
fpcube, transforms = spirouImage.ConstructMasterFP(p, *cargs)
# log process
wmsg1 = 'Master FP construction complete.'
wmsg2 = '\tAdding {0} group images to form FP master image'
WLOG(p, 'info', [wmsg1, wmsg2.format(len(fpcube))])
# sum the cube to make fp data
masterfp = np.sum(fpcube, axis=0)
# add to loc
loc['MASTERFP'] = masterfp
loc.set_source('MASTERFP', __NAME__ + '.main()')
# ------------------------------------------------------------------
# Get localisation coefficients
# ------------------------------------------------------------------
# original there is a loop but it is not used --> removed
p = spirouImage.FiberParams(p, p['FIBER'], merge=True)
# get localisation fit coefficients
p, loc = spirouLOCOR.GetCoeffs(p, fphdr, loc)
# ------------------------------------------------------------------
# Get master wave solution map
# ------------------------------------------------------------------
# get master wave map
masterwavefile = spirouDB.GetDatabaseMasterWave(p)
# log process
wmsg1 = 'Getting master wavelength grid'
wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile))
WLOG(p, '', [wmsg1, wmsg2])
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# read master wave map
wout = spirouImage.GetWaveSolution(p,
filename=masterwavefile,
return_wavemap=True,
quiet=True,
return_header=True,
fiber=wave_fiber)
loc['MASTERWAVEP'], loc['MASTERWAVE'] = wout[:2]
loc['MASTERWAVEHDR'], loc['WSOURCE'] = wout[2:]
# set sources
wsource = ['MASTERWAVEP', 'MASTERWAVE', 'MASTERWAVEHDR']
loc.set_sources(wsource, 'spirouImage.GetWaveSolution()')
# ----------------------------------------------------------------------
# Read UNe solution
# ----------------------------------------------------------------------
wave_u_ne, amp_u_ne = spirouImage.ReadLineList(p)
loc['LL_LINE'], loc['AMPL_LINE'] = wave_u_ne, amp_u_ne
source = __NAME__ + '.main() + spirouImage.ReadLineList()'
loc.set_sources(['LL_LINE', 'AMPL_LINE'], source)
# ----------------------------------------------------------------------
# Read cavity length file
# ----------------------------------------------------------------------
loc['CAVITY_LEN_COEFFS'] = spirouImage.ReadCavityLength(p)
source = __NAME__ + '.main() + spirouImage.ReadCavityLength()'
loc.set_source('CAVITY_LEN_COEFFS', source)
# ----------------------------------------------------------------------
# Calculate shape map
# ----------------------------------------------------------------------
# calculate dx map
loc = spirouImage.GetXShapeMap(p, loc)
# if dx map is None we shouldn't continue
if loc['DXMAP'] is None:
fargs = [
loc['MAXDXMAPINFO'][0], loc['MAXDXMAPINFO'][1], loc['MAXDXMAPSTD'],
p['SHAPE_QC_DXMAP_STD']
]
fmsg = ('The std of the dxmap for order {0} y-pixel {1} is too large.'
' std = {2} (limit = {3})'.format(*fargs))
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(fmsg))
WLOG(p, 'warning', 'Cannot continue. Exiting.')
# End Message
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
# calculate dymap
loc = spirouImage.GetYShapeMap(p, loc, fphdr)
# ------------------------------------------------------------------
# Need to straighten the dxmap
# ------------------------------------------------------------------
# copy it first
loc['DXMAP0'] = np.array(loc['DXMAP'])
# straighten it
loc['DXMAP'] = spirouImage.EATransform(loc['DXMAP'], dymap=loc['DYMAP'])
# ------------------------------------------------------------------
# Need to straighten the hc data and fp data for debug
# ------------------------------------------------------------------
# log progress
WLOG(p, '', 'Shape finding complete. Applying transforms.')
# apply very last update of the debananafication
tkwargs = dict(dxmap=loc['DXMAP'], dymap=loc['DYMAP'])
loc['HCDATA2'] = spirouImage.EATransform(loc['HCDATA1'], **tkwargs)
loc['FPDATA2'] = spirouImage.EATransform(loc['FPDATA1'], **tkwargs)
loc.set_sources(['HCDATA2', 'FPDATA2'], __NAME__ + '.main()')
# ------------------------------------------------------------------
# Plotting
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# plots setup: start interactive plot
sPlt.start_interactive_session(p)
# plot the shape process for one order
sPlt.slit_shape_angle_plot(p, loc)
# end interactive section
sPlt.end_interactive_session(p)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# TODO: Decide on some quality control criteria?
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# add to qc header lists
qc_values.append(loc['MAXDXMAPSTD'])
qc_names.append('DXMAP STD')
qc_logic.append('DXMAP STD < {0}'.format(p['SHAPE_QC_DXMAP_STD']))
qc_pass.append(1)
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Writing FP big table
# ------------------------------------------------------------------
# construct big fp table
colnames = [
'FILENAME', 'NIGHT', 'MJDATE', 'EXPTIME', 'PVERSION', 'GROUPID',
'DXREF', 'DYREF', 'A', 'B', 'C', 'D'
]
values = [
basenames, nightnames, fp_time, fp_exp, fp_pp_version, matched_id,
transforms[:, 0], transforms[:, 1], transforms[:, 2], transforms[:, 3],
transforms[:, 4], transforms[:, 5]
]
fptable = spirouImage.MakeTable(p, colnames, values)
# ------------------------------------------------------------------
# Writing DXMAP to file
# ------------------------------------------------------------------
# get the raw tilt file name
raw_shape_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
shapexfits, tag = spirouConfig.Constants.SLIT_XSHAPE_FILE(p)
shapexfitsname = os.path.basename(shapexfits)
# Log that we are saving tilt file
wmsg = 'Saving shape x information in file: {0}'
WLOG(p, '', wmsg.format(shapexfitsname))
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(fphdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='hcfile',
values=p['HCFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE2'],
dim1name='fpfile',
values=p['FPFILES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# write tilt file to file
p = spirouImage.WriteImageTable(p,
shapexfits,
image=loc['DXMAP'],
table=fptable,
hdict=hdict)
# ------------------------------------------------------------------
# Writing DYMAP to file
# ------------------------------------------------------------------
# get the raw tilt file name
raw_shape_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
shapeyfits, tag = spirouConfig.Constants.SLIT_YSHAPE_FILE(p)
shapeyfitsname = os.path.basename(shapeyfits)
# Log that we are saving tilt file
wmsg = 'Saving shape y information in file: {0}'
WLOG(p, '', wmsg.format(shapeyfitsname))
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(fphdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='hcfile',
values=p['HCFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE2'],
dim1name='fpfile',
values=p['FPFILES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# write tilt file to file
p = spirouImage.WriteImageTable(p,
shapeyfits,
image=loc['DYMAP'],
table=fptable,
hdict=hdict)
# ------------------------------------------------------------------
# Writing Master FP to file
# ------------------------------------------------------------------
# get the raw tilt file name
raw_shape_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
fpmasterfits, tag = spirouConfig.Constants.SLIT_MASTER_FP_FILE(p)
fpmasterfitsname = os.path.basename(fpmasterfits)
# Log that we are saving tilt file
wmsg = 'Saving master FP file: {0}'
WLOG(p, '', wmsg.format(fpmasterfitsname))
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(fphdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='hcfile',
values=p['HCFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE2'],
dim1name='fpfile',
values=p['FPFILES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# write tilt file to file
p = spirouImage.WriteImageTable(p,
fpmasterfits,
image=masterfp,
table=fptable,
hdict=hdict)
# ------------------------------------------------------------------
# Writing sanity check files
# ------------------------------------------------------------------
if p['SHAPE_DEBUG_OUTPUTS']:
# log
WLOG(p, '', 'Saving debug sanity check files')
# construct file names
input_fp_file, tag1 = spirouConfig.Constants.SLIT_SHAPE_IN_FP_FILE(p)
output_fp_file, tag2 = spirouConfig.Constants.SLIT_SHAPE_OUT_FP_FILE(p)
input_hc_file, tag3 = spirouConfig.Constants.SLIT_SHAPE_IN_HC_FILE(p)
output_hc_file, tag4 = spirouConfig.Constants.SLIT_SHAPE_OUT_HC_FILE(p)
bdxmap_file, tag5 = spirouConfig.Constants.SLIT_SHAPE_BDXMAP_FILE(p)
# write input fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
p = spirouImage.WriteImage(p, input_fp_file, loc['FPDATA1'], hdict)
# write output fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
p = spirouImage.WriteImage(p, output_fp_file, loc['FPDATA2'], hdict)
# write input fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
p = spirouImage.WriteImage(p, input_hc_file, loc['HCDATA1'], hdict)
# write output fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
p = spirouImage.WriteImage(p, output_hc_file, loc['HCDATA2'], hdict)
# write overlap file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag5)
p = spirouImage.WriteImage(p, bdxmap_file, loc['DXMAP0'], hdict)
# ----------------------------------------------------------------------
# Move to calibDB and update calibDB
# ----------------------------------------------------------------------
if p['QC']:
# add shape x
keydb = 'SHAPEX'
# copy shape file to the calibDB folder
spirouDB.PutCalibFile(p, shapexfits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, shapexfitsname, fphdr)
# add shape y
keydb = 'SHAPEY'
# copy shape file to the calibDB folder
spirouDB.PutCalibFile(p, shapeyfits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, shapeyfitsname, fphdr)
# add fp master
keydb = 'FPMASTER'
# copy shape file to the calibDB folder
spirouDB.PutCalibFile(p, fpmasterfits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, fpmasterfitsname, fphdr)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, flatfile=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# deal with arguments being None (i.e. get from sys.argv)
name, lname = ['flatfile'], ['Reference file']
req, call, call_priority = [True], [flatfile], [True]
# now get custom arguments
customargs = spirouStartup.GetCustomFromRuntime(p, [0], [str], name, req,
call, call_priority, lname)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsfile='flatfile')
# ----------------------------------------------------------------------
# Construct reference filename and get fiber type
# ----------------------------------------------------------------------
p, reffile = spirouStartup.SingleFileSetup(p, filename=p['FLATFILE'])
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# ----------------------------------------------------------------------
# Get the required fiber type from the constants file
# ----------------------------------------------------------------------
# get the fiber type (set to AB)
p['FIBER'] = p['EM_FIB_TYPE']
p['FIBER_TYPES'] = [p['EM_FIB_TYPE']]
# ----------------------------------------------------------------------
# Read flat image file
# ----------------------------------------------------------------------
# read the image data (for the header only)
image, hdr, ny, nx = spirouImage.ReadData(p, reffile)
# ----------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
image = spirouImage.FixNonPreProcess(p, image)
# ----------------------------------------------------------------------
# Get basic image properties
# ----------------------------------------------------------------------
# create loc
loc = ParamDict()
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# ----------------------------------------------------------------------
# Resize flat image
# ----------------------------------------------------------------------
# rotate the image and convert from ADU/s to e-
image2 = spirouImage.ConvertToE(spirouImage.FlipImage(p, image), p=p)
# convert NaN to zeros
image2 = np.where(~np.isfinite(image2), np.zeros_like(image2), image2)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'],
xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'],
yhigh=p['IC_CCDY_HIGH'],
getshape=False)
image2 = spirouImage.ResizeImage(p, image2, **bkwargs)
# save flat to to loc and set source
loc['IMAGE'] = image2
loc.set_sources(['image'], __NAME__ + '/main()')
# log change in data size
wmsg = 'Image format changed to {0}x{1}'
WLOG(p, '', wmsg.format(*image2.shape))
# ----------------------------------------------------------------------
# Read shape or tilt slit angle
# ----------------------------------------------------------------------
# set source of tilt file
tsource = __NAME__ + '/main() + /spirouImage.ReadTiltFile'
if p['IC_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES:
# log progress
WLOG(p, '', 'Debananafying (straightening) image')
# get the shape map
p, loc['SHAPE'] = spirouImage.ReadShapeMap(p, hdr)
loc.set_source('SHAPE', tsource)
else:
# get tilts
p, loc['TILT'] = spirouImage.ReadTiltFile(p, hdr)
loc.set_source('TILT', tsource)
# ----------------------------------------------------------------------
# Read blaze
# ----------------------------------------------------------------------
# get tilts
p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hdr)
loc.set_source('BLAZE', __NAME__ + '/main() + /spirouImage.ReadBlazeFile')
# set number of orders from blaze file
loc['NBO'] = loc['BLAZE'].shape[0]
loc.set_source('NBO', __NAME__ + '/main()')
# ------------------------------------------------------------------
# Read wavelength solution
# ------------------------------------------------------------------
# set source of wave file
wsource = __NAME__ + '/main() + /spirouImage.GetWaveSolution'
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# get wave image
wout = spirouImage.GetWaveSolution(p,
hdr=hdr,
return_wavemap=True,
return_filename=True,
fiber=wave_fiber)
loc['WAVEPARAMS'], loc['WAVE'], loc['WAVEFILE'], loc['WSOURCE'] = wout
loc.set_sources(['WAVEPARAMS', 'WAVE', 'WAVEFILE', 'WSOURCE'], wsource)
# ------------------------------------------------------------------
# Get localisation coefficients
# ------------------------------------------------------------------
# storage for fiber parameters
loc['ALL_ACC'] = OrderedDict()
loc['ALL_ASS'] = OrderedDict()
# get this fibers parameters
for fiber in p['FIBER_TYPES']:
p = spirouImage.FiberParams(p, fiber, merge=True)
# get localisation fit coefficients
p, loc = spirouLOCOR.GetCoeffs(p, hdr, loc=loc)
# save all fibers
loc['ALL_ACC'][fiber] = loc['ACC']
loc['ALL_ASS'][fiber] = loc['ASS']
# ------------------------------------------------------------------
# Get telluric and telluric mask and add to loc
# ------------------------------------------------------------------
# log process
wmsg = 'Loading telluric model and locating "good" tranmission'
WLOG(p, '', wmsg)
# load telluric and get mask (add to loc)
loc = spirouExM.get_telluric(p, loc)
# ------------------------------------------------------------------
# Make 2D map of orders
# ------------------------------------------------------------------
# log progress
WLOG(p, '', 'Making 2D map of order locations')
# make the 2D wave-image
loc = spirouExM.order_profile(p, loc)
# ------------------------------------------------------------------
# Make 2D map of wavelengths accounting for shape / tilt
# ------------------------------------------------------------------
# log progress
WLOG(p, '', 'Mapping pixels on to wavelength grid')
# make the 2D map of wavelength
loc = spirouExM.create_wavelength_image(p, loc)
# ------------------------------------------------------------------
# Use spectra wavelength to create 2D image from wave-image
# ------------------------------------------------------------------
if p['EM_SAVE_MASK_MAP'] or p['EM_SAVE_TELL_SPEC']:
# log progress
WLOG(p, '', 'Creating image from wave-image interpolation')
# create image from waveimage
wkwargs = dict(x=loc['TELL_X'], y=loc['TELL_Y'])
loc = spirouExM.create_image_from_waveimage(loc, **wkwargs)
else:
loc['SPE'] = np.zeros_like(image2).astype(float)
# ------------------------------------------------------------------
# Create 2D mask (min to max lambda + transmission threshold)
# ------------------------------------------------------------------
if p['EM_SAVE_MASK_MAP']:
# log progress
WLOG(p, '', 'Creating wavelength/tranmission mask')
# create mask
loc = spirouExM.create_mask(p, loc)
else:
loc['TELL_MASK_2D'] = np.zeros_like(image2).astype(bool)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# TODO: Needs doing
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# add to qc header lists
qc_values.append('None')
qc_names.append('None')
qc_logic.append('None')
qc_pass.append(1)
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Construct parameters for header
# ------------------------------------------------------------------
hdict = OrderedDict()
# set the version
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
# set the input files
if loc['SHAPE'] is not None:
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBSHAPE'],
value=p['SHAPFILE'])
else:
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBTILT'],
value=p['TILTFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=loc['WAVEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=loc['WSOURCE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='file',
values=p['FLATFILE'])
# add name of the TAPAS y data
hdict = spirouImage.AddKey(p,
hdict,
p['KW_EM_TELLY'],
value=loc['TELLSPE'])
# add name of the localisation fits file used
hfile = os.path.basename(loc['LOCO_CTR_FILE'])
hdict = spirouImage.AddKey(p, hdict, p['kw_EM_LOCFILE'], value=hfile)
# add the max and min wavelength threshold
hdict = spirouImage.AddKey(p,
hdict,
p['kw_EM_MINWAVE'],
value=p['EM_MIN_LAMBDA'])
hdict = spirouImage.AddKey(p,
hdict,
p['kw_EM_MAXWAVE'],
value=p['EM_MAX_LAMBDA'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# add the transmission cut
hdict = spirouImage.AddKey(p,
hdict,
p['kw_EM_TRASCUT'],
value=p['EM_TELL_THRESHOLD'])
# ------------------------------------------------------------------
# Deal with output preferences
# ------------------------------------------------------------------
# add bad pixel map (if required)
if p['EM_COMBINED_BADPIX']:
# get bad pix mask (True where bad)
badpixmask, bhdr, badfile = spirouImage.GetBadPixMap(p, hdr)
goodpixels = badpixmask == 0
# apply mask (multiply)
loc['TELL_MASK_2D'] = loc['TELL_MASK_2D'] & goodpixels.astype(bool)
else:
badfile = 'None'
# add to hdict
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=badfile)
# convert waveimage mask into float array
loc['TELL_MASK_2D'] = loc['TELL_MASK_2D'].astype('float')
# check EM_OUTPUT_TYPE and deal with set to "all"
if p['EM_OUTPUT_TYPE'] not in ["drs", "raw", "preprocess", "all"]:
emsg1 = '"EM_OUTPUT_TYPE" not understood'
emsg2 = ' must be either "drs", "raw" or "preprocess"'
emsg3 = ' currently EM_OUTPUT_TYPE="{0}"'.format(p['EM_OUTPUT_TYPE'])
WLOG(p, 'error', [emsg1, emsg2, emsg3])
outputs = []
elif p['EM_OUTPUT_TYPE'] != 'all':
outputs = [str(p['EM_OUTPUT_TYPE'])]
else:
outputs = ["drs", "raw", "preprocess"]
# ----------------------------------------------------------------------
# loop around output types
# ----------------------------------------------------------------------
for output in outputs:
# log progress
WLOG(p, '', 'Processing {0} outputs'.format(output))
# change EM_OUTPUT_TYPE
p['EM_OUTPUT_TYPE'] = output
# copy arrays
out_spe = np.array(loc['SPE'])
out_wave = np.array(loc['WAVEIMAGE'])
out_mask = np.array(loc['TELL_MASK_2D'])
# change image size if needed
if output in ["raw", "preprocess"]:
kk = dict(xsize=image.shape[1], ysize=image.shape[0])
if p['EM_SAVE_TELL_SPEC']:
WLOG(p, '', 'Resizing/Flipping SPE')
out_spe = spirouExM.unresize(p, out_spe, **kk)
WLOG(p, '', 'Rescaling SPE')
out_spe = out_spe / (p['GAIN'] * p['EXPTIME'])
if p['EM_SAVE_WAVE_MAP']:
WLOG(p, '', 'Resizing/Flipping WAVEIMAGE')
out_wave = spirouExM.unresize(p, out_wave, **kk)
if p['EM_SAVE_MASK_MAP']:
WLOG(p, '', 'Resizing/Flipping TELL_MASK_2D')
out_mask = spirouExM.unresize(p, out_mask, **kk)
# if raw need to rotate (undo pre-processing)
if output == "raw":
if p['EM_SAVE_TELL_SPEC']:
WLOG(p, '', 'Rotating SPE')
out_spe = np.rot90(out_spe, 1)
if p['EM_SAVE_WAVE_MAP']:
WLOG(p, '', 'Rotating WAVEIMAGE')
out_wave = np.rot90(out_wave, 1)
if p['EM_SAVE_MASK_MAP']:
WLOG(p, '', 'Rotating TELL_MASK_2D')
out_mask = np.rot90(out_mask, 1)
# ----------------------------------------------------------------------
# save 2D spectrum, wavelength image and mask to file
# ----------------------------------------------------------------------
# save telluric spectrum
if p['EM_SAVE_TELL_SPEC']:
# construct spectrum filename
specfitsfile, tag = spirouConfig.Constants.EM_SPE_FILE(p)
specfilename = os.path.split(specfitsfile)[-1]
# set the version
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
# log progress
wmsg = 'Writing spectrum to file {0}'
WLOG(p, '', wmsg.format(specfilename))
# write to file
p = spirouImage.WriteImage(p, specfitsfile, out_spe, hdict=hdict)
# ----------------------------------------------------------------------
# save wave map
if p['EM_SAVE_WAVE_MAP']:
# construct waveimage filename
wavefitsfile, tag = spirouConfig.Constants.EM_WAVE_FILE(p)
wavefilename = os.path.split(wavefitsfile)[-1]
# set the version
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
# log progress
wmsg = 'Writing wave image to file {0}'
WLOG(p, '', wmsg.format(wavefilename))
# write to file
p = spirouImage.WriteImage(p, wavefitsfile, out_wave, hdict=hdict)
# ----------------------------------------------------------------------
# save mask file
if p['EM_SAVE_MASK_MAP']:
# construct tell mask 2D filename
maskfitsfile, tag = spirouConfig.Constants.EM_MASK_FILE(p)
maskfilename = os.path.split(maskfitsfile)[-1]
# set the version
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
# log progress
wmsg = 'Writing telluric mask to file {0}'
WLOG(p, '', wmsg.format(maskfilename))
# convert boolean mask to integers
writablemask = np.array(out_mask, dtype=float)
# write to file
p = spirouImage.WriteImage(p,
maskfitsfile,
writablemask,
hdict=hdict)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())