def main(night_name=None, files=None, fiber_type=None, **kwargs):
"""
cal_DRIFT_E2DS_spirou.py main function, if night_name and files are
None uses arguments from run time i.e.:
cal_DRIFT_E2DS_spirou.py [night_directory] [files]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:param fiber_type: string, if None does all fiber types (defined in
constants_SPIROU FIBER_TYPES (default is AB, A, B, C
if defined then only does this fiber type (but must
be in FIBER_TYPES)
:param kwargs: any keyword to overwrite constant in param dict "p"
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files)
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# deal with fiber type
if fiber_type is None:
fiber_type = p['FIBER_TYPES']
if type(fiber_type) == str:
if fiber_type.upper() == 'ALL':
fiber_type = p['FIBER_TYPES']
elif fiber_type in p['FIBER_TYPES']:
fiber_type = [fiber_type]
else:
emsg = 'fiber_type="{0}" not understood'
WLOG(p, 'error', emsg.format(fiber_type))
# set fiber type
p['FIB_TYPE'] = fiber_type
p.set_source('FIB_TYPE', __NAME__ + '__main__()')
# Overwrite keys from source
for kwarg in kwargs:
p[kwarg] = kwargs[kwarg]
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add')
# ----------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
data = spirouImage.FixNonPreProcess(p, data)
# ----------------------------------------------------------------------
# Get basic image properties
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# set sigdet and conad keywords (sigdet is changed later)
p['KW_CCD_SIGDET'][1] = p['SIGDET']
p['KW_CCD_CONAD'][1] = p['GAIN']
# now change the value of sigdet if require
if p['IC_EXT_SIGDET'] > 0:
p['SIGDET'] = float(p['IC_EXT_SIGDET'])
# get DPRTYPE from header (Will have it if valid)
p = spirouImage.ReadParam(p, hdr, 'KW_DPRTYPE', required=False, dtype=str)
# check the DPRTYPE is not None
if (p['DPRTYPE'] == 'None') or (['DPRTYPE'] is None):
emsg1 = 'Error: {0} is not set in header for file {1}'
eargs = [p['KW_DPRTYPE'][0], p['FITSFILENAME']]
emsg2 = '\tPlease run pre-processing on file.'
emsg3 = ('\tIf pre-processing fails or skips file, file is not '
'currrently as valid DRS fits file.')
WLOG(p, 'error', [emsg1.format(*eargs), emsg2, emsg3])
else:
p['DPRTYPE'] = p['DPRTYPE'].strip()
# ----------------------------------------------------------------------
# Correction of DARK
# ----------------------------------------------------------------------
p, datac = spirouImage.CorrectForDark(p, data, hdr)
# ----------------------------------------------------------------------
# Resize image
# ----------------------------------------------------------------------
# rotate the image and convert from ADU/s to ADU
data = spirouImage.ConvertToADU(spirouImage.FlipImage(p, datac), p=p)
# convert NaN to zeros
data0 = np.where(~np.isfinite(data), np.zeros_like(data), data)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'],
xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'],
yhigh=p['IC_CCDY_HIGH'],
getshape=False)
data1 = spirouImage.ResizeImage(p, data0, **bkwargs)
# log change in data size
wmsg = 'Image format changed to {1}x{0}'
WLOG(p, '', wmsg.format(*data1.shape))
# ----------------------------------------------------------------------
# Correct for the BADPIX mask (set all bad pixels to zero)
# ----------------------------------------------------------------------
p, data1 = spirouImage.CorrectForBadPix(p, data1, hdr)
# ----------------------------------------------------------------------
# Log the number of dead pixels
# ----------------------------------------------------------------------
# get the number of bad pixels
n_bad_pix = np.sum(~np.isfinite(data1))
n_bad_pix_frac = n_bad_pix * 100 / np.product(data1.shape)
# Log number
wmsg = 'Nb dead pixels = {0} / {1:.4f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# ----------------------------------------------------------------------
# Get the miny, maxy and max_signal for the central column
# ----------------------------------------------------------------------
# get the central column
y = data1[p['IC_CENT_COL'], :]
# get the min max and max signal using box smoothed approach
miny, maxy, max_signal, diff_maxmin = spirouBACK.MeasureMinMaxSignal(p, y)
# Log max average flux/pixel
wmsg = 'Maximum average flux/pixel in the spectrum: {0:.1f} [ADU]'
WLOG(p, 'info', wmsg.format(max_signal / p['NBFRAMES']))
# ----------------------------------------------------------------------
# Background computation
# ----------------------------------------------------------------------
if p['IC_DO_BKGR_SUBTRACTION']:
# log that we are doing background measurement
WLOG(p, '', 'Doing background measurement on raw frame')
# get the bkgr measurement
bargs = [p, data1, hdr]
# background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs)
p, background = spirouBACK.MeasureBackgroundMap(*bargs)
else:
background = np.zeros_like(data1)
p['BKGRDFILE'] = 'None'
p.set_source('BKGRDFILE', __NAME__ + '.main()')
# apply background correction to data (and set to zero where negative)
data1 = data1 - background
# ----------------------------------------------------------------------
# Read tilt slit angle
# ----------------------------------------------------------------------
# define loc storage parameter dictionary
loc = ParamDict()
# get tilts (if the mode requires it)
if p['IC_EXTRACT_TYPE'] not in EXTRACT_SHAPE_TYPES:
p, loc['TILT'] = spirouImage.ReadTiltFile(p, hdr)
loc.set_source('TILT',
__NAME__ + '/main() + /spirouImage.ReadTiltFile')
else:
loc['TILT'] = None
loc.set_source('TILT', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Earth Velocity calculation
# ----------------------------------------------------------------------
if p['IC_IMAGE_TYPE'] == 'H4RG':
p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, hdr)
# ----------------------------------------------------------------------
# Get all fiber data (for all fibers)
# ----------------------------------------------------------------------
# TODO: This is temp solution for options 5a and 5b
loc_fibers = spirouLOCOR.GetFiberData(p, hdr)
# ------------------------------------------------------------------
# Deal with debananafication
# ------------------------------------------------------------------
# if mode 4a or 4b we need to straighten in x only
if p['IC_EXTRACT_TYPE'] in ['4a', '4b']:
# get the shape parameters
p, shapem_x = spirouImage.GetShapeX(p, hdr)
p, shape_local = spirouImage.GetShapeLocal(p, hdr)
# log progress
WLOG(p, '', 'Debananafying (straightening) image')
# apply shape transforms
targs = dict(lin_transform_vect=shape_local, dxmap=shapem_x)
data2 = spirouImage.EATransform(data1, **targs)
# if mode 5a or 5b we need to straighten in x and y using the
# polynomial fits for location
elif p['IC_EXTRACT_TYPE'] in ['5a', '5b']:
# get the shape parameters
p, shapem_x = spirouImage.GetShapeX(p, hdr)
p, shapem_y = spirouImage.GetShapeY(p, hdr)
p, shape_local = spirouImage.GetShapeLocal(p, hdr)
p, fpmaster = spirouImage.GetFPMaster(p, hdr)
# get the bad pixel map
bkwargs = dict(return_map=True, quiet=True)
p, badpix = spirouImage.CorrectForBadPix(p, data1, hdr, **bkwargs)
# log progress
WLOG(p, '', 'Cleaning image')
# clean the image
data1 = spirouEXTOR.CleanHotpix(data1, badpix)
# log progress
WLOG(p, '', 'Debananafying (straightening) image')
# apply shape transforms
targs = dict(lin_transform_vect=shape_local,
dxmap=shapem_x,
dymap=shapem_y)
data2 = spirouImage.EATransform(data1, **targs)
# in any other mode we do not straighten
else:
data2 = np.array(data1)
# ----------------------------------------------------------------------
# Fiber loop
# ----------------------------------------------------------------------
# loop around fiber types
for fiber in p['FIB_TYPE']:
# set fiber
p['FIBER'] = fiber
p.set_source('FIBER', __NAME__ + '/main()()')
# ------------------------------------------------------------------
# Read wavelength solution
# ------------------------------------------------------------------
# set source of wave file
wsource = __NAME__ + '/main() + /spirouImage.GetWaveSolution'
# Force A and B to AB solution
if fiber in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = fiber
# get wave image
wkwargs = dict(hdr=hdr,
return_wavemap=True,
return_filename=True,
return_header=True,
fiber=wave_fiber)
wout = spirouImage.GetWaveSolution(p, **wkwargs)
loc['WAVEPARAMS'], loc['WAVE'], loc['WAVEFILE'] = wout[:3]
loc['WAVEHDR'], loc['WSOURCE'] = wout[3:]
source_names = ['WAVE', 'WAVEFILE', 'WAVEPARAMS', 'WAVEHDR']
loc.set_sources(source_names, wsource)
# get dates
loc['WAVE_ACQTIMES'] = spirouDB.GetTimes(p, loc['WAVEHDR'])
loc.set_source('WAVE_ACQTIMES', __NAME__ + '.main()')
# get the recipe that produced the wave solution
if 'WAVECODE' in loc['WAVEHDR']:
loc['WAVE_CODE'] = loc['WAVEHDR']['WAVECODE']
else:
loc['WAVE_CODE'] = 'UNKNOWN'
loc.set_source('WAVE_CODE', __NAME__ + '.main()')
# ----------------------------------------------------------------------
# Get WFP keys
# ----------------------------------------------------------------------
# Read the WFP keys - if they don't exist set to None and deal
# with later
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_DRIFT',
name='WFP_DRIFT',
required=False)
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_FWHM',
name='WFP_FWHM',
required=False)
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_CONTRAST',
name='WFP_CONTRAST',
required=False)
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_MAXCPP',
name='WFP_MAXCPP',
required=False)
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_MASK',
name='WFP_MASK',
required=False)
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_LINES',
name='WFP_LINES',
required=False)
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_TARG_RV',
name='WFP_TARG_RV',
required=False)
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_WIDTH',
name='WFP_WIDTH',
required=False)
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_STEP',
name='WFP_STEP',
required=False)
# ----------------------------------------------------------------------
# Read Flat file
# ----------------------------------------------------------------------
fout = spirouImage.ReadFlatFile(p, hdr, return_header=True)
p, loc['FLAT'], flathdr = fout
loc.set_source('FLAT',
__NAME__ + '/main() + /spirouImage.ReadFlatFile')
# get flat extraction mode
if p['KW_E2DS_EXTM'][0] in flathdr:
flat_ext_mode = flathdr[p['KW_E2DS_EXTM'][0]]
else:
flat_ext_mode = None
# ------------------------------------------------------------------
# Check extraction method is same as flat extraction method
# ------------------------------------------------------------------
# get extraction method and function
extmethod, extfunc = spirouEXTOR.GetExtMethod(p, p['IC_EXTRACT_TYPE'])
if not DEBUG:
# compare flat extraction mode to extraction mode
spirouEXTOR.CompareExtMethod(p, flat_ext_mode, extmethod, 'FLAT',
'EXTRACTION')
# ------------------------------------------------------------------
# Read Blaze file
# ------------------------------------------------------------------
p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hdr)
blazesource = __NAME__ + '/main() + /spirouImage.ReadBlazeFile'
loc.set_source('BLAZE', blazesource)
# ------------------------------------------------------------------
# Get fiber specific parameters from loc_fibers
# ------------------------------------------------------------------
# get this fibers parameters
p = spirouImage.FiberParams(p, p['FIBER'], merge=True)
# get localisation parameters
for key in loc_fibers[fiber]:
loc[key] = loc_fibers[fiber][key]
loc.set_source(key, loc_fibers[fiber].sources[key])
# get locofile source
p['LOCOFILE'] = loc['LOCOFILE']
p.set_source('LOCOFILE', loc.sources['LOCOFILE'])
# get the order_profile
order_profile = loc_fibers[fiber]['ORDER_PROFILE']
# ------------------------------------------------------------------
# Set up Extract storage
# ------------------------------------------------------------------
# Create array to store extraction (for each order and each pixel
# along order)
loc['E2DS'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
loc['E2DSFF'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
loc['E2DSLL'] = []
loc['SPE1'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
loc['SPE3'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
loc['SPE4'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
loc['SPE5'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
# Create array to store the signal to noise ratios for each order
loc['SNR'] = np.zeros(loc['NUMBER_ORDERS'])
# ------------------------------------------------------------------
# Extract orders
# ------------------------------------------------------------------
# source for parameter dictionary
source = __NAME__ + '/main()'
# get limits of order extraction
valid_orders = spirouEXTOR.GetValidOrders(p, loc)
# loop around each order
for order_num in valid_orders:
# -------------------------------------------------------------
# IC_EXTRACT_TYPE decides the extraction routine
# -------------------------------------------------------------
eargs = [p, loc, data2, order_num]
ekwargs = dict(mode=p['IC_EXTRACT_TYPE'],
order_profile=order_profile)
with warnings.catch_warnings(record=True) as w:
eout = spirouEXTOR.Extraction(*eargs, **ekwargs)
# deal with different return
if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES:
e2ds, e2dsll, cpt = eout
else:
e2ds, cpt = eout
e2dsll = None
# -------------------------------------------------------------
# calculate the noise
range1, range2 = p['IC_EXT_RANGE1'], p['IC_EXT_RANGE2']
# set the noise
noise = p['SIGDET'] * np.sqrt(range1 + range2)
# get window size
blaze_win1 = int(data2.shape[0] / 2) - p['IC_EXTFBLAZ']
blaze_win2 = int(data2.shape[0] / 2) + p['IC_EXTFBLAZ']
# get average flux per pixel
flux = np.nansum(
e2ds[blaze_win1:blaze_win2]) / (2 * p['IC_EXTFBLAZ'])
# calculate signal to noise ratio = flux/sqrt(flux + noise^2)
snr = flux / np.sqrt(flux + noise**2)
# log the SNR RMS
wmsg = 'On fiber {0} order {1}: S/N= {2:.1f} Nbcosmic= {3}'
wargs = [p['FIBER'], order_num, snr, cpt]
WLOG(p, '', wmsg.format(*wargs))
# add calculations to storage
loc['E2DS'][order_num] = e2ds
loc['E2DSFF'][order_num] = e2ds / loc['FLAT'][order_num]
loc['SNR'][order_num] = snr
# save the longfile
if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES:
loc['E2DSLL'].append(e2dsll)
# set sources
loc.set_sources(['e2ds', 'SNR'], source)
# Log if saturation level reached
satvalue = (flux / p['GAIN']) / (range1 + range2)
if satvalue > (p['QC_LOC_FLUMAX'] * p['NBFRAMES']):
wmsg = 'SATURATION LEVEL REACHED on Fiber {0} order={1}'
WLOG(p, 'warning', wmsg.format(fiber, order_num))
# ------------------------------------------------------------------
# Thermal correction
# ------------------------------------------------------------------
# get fiber type
if fiber in ['AB', 'A', 'B']:
fibertype = p['DPRTYPE'].split('_')[0]
else:
fibertype = p['DPRTYPE'].split('_')[1]
# apply thermal correction based on fiber type
if fibertype in p['THERMAL_CORRECTION_TYPE1']:
# log progress
wmsg = 'Correcting thermal background for {0}={1} mode={2}'
wargs = [fiber, fibertype, 1]
WLOG(p, 'info', wmsg.format(*wargs))
# correct E2DS
tkwargs = dict(image=loc['E2DS'], mode=1, fiber=fiber, hdr=hdr)
p, loc['E2DS'] = spirouBACK.ThermalCorrect(p, **tkwargs)
# correct E2DSFF
tkwargs = dict(image=loc['E2DSFF'],
mode=1,
fiber=fiber,
hdr=hdr,
flat=loc['FLAT'])
p, loc['E2DSFF'] = spirouBACK.ThermalCorrect(p, **tkwargs)
elif fibertype in p['THERMAL_CORRECTION_TYPE2']:
# log progress
wmsg = 'Correcting thermal background for {0}={1} mode={2}'
wargs = [fiber, fibertype, 2]
WLOG(p, 'info', wmsg.format(*wargs))
# correct E2DS
tkwargs = dict(image=loc['E2DS'], mode=2, fiber=fiber, hdr=hdr)
p, loc['E2DS'] = spirouBACK.ThermalCorrect(p, **tkwargs)
# correct E2DSFF
tkwargs = dict(image=loc['E2DSFF'],
mode=2,
fiber=fiber,
hdr=hdr,
flat=loc['FLAT'])
p, loc['E2DSFF'] = spirouBACK.ThermalCorrect(p, **tkwargs)
else:
# log progress
wmsg = 'Not correcting thermal background for {0}={1}'
wargs = [fiber, fibertype]
WLOG(p, 'info', wmsg.format(*wargs))
# set filename for output
outfile = 'THERMALFILE_{0}'.format(fiber)
p[outfile] = 'None'
p.set_source(outfile, __NAME__ + '.main()')
# ------------------------------------------------------------------
# Plots
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# start interactive session if needed
sPlt.start_interactive_session(p)
# plot all orders or one order
if p['IC_FF_PLOT_ALL_ORDERS']:
# plot image with all order fits (slower)
sPlt.ext_aorder_fit(p, loc, data1, max_signal / 10.)
else:
# plot image with selected order fit and edge fit (faster)
sPlt.ext_sorder_fit(p, loc, data1, max_signal / 10.)
# plot e2ds against wavelength
sPlt.ext_spectral_order_plot(p, loc)
if p['IC_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES:
sPlt.ext_debanana_plot(p, loc, data2, max_signal / 10.)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# if array is completely NaNs it shouldn't pass
if np.sum(np.isfinite(loc['E2DS'])) == 0:
fail_msg.append('E2DS image is all NaNs')
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append('NaN')
qc_names.append('image')
qc_logic.append('image is all NaN')
# ----------------------------------------------------------------------
# saturation check: check that the max_signal is lower than
# qc_max_signal
max_qcflux = p['QC_MAX_SIGNAL'] * p['NBFRAMES']
if max_signal > max_qcflux:
fmsg = 'Too much flux in the image ({0:.2f} > {1:.2f})'
fail_msg.append(fmsg.format(max_signal, max_qcflux))
passed = False
# Question: Why is this test ignored?
# For some reason this test is ignored in old code
passed = True
WLOG(p, 'info', fail_msg[-1])
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(max_signal)
qc_names.append('max_signal')
qc_logic.append('QC_MAX_SIGNAL > {0:.3f}'.format(max_qcflux))
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Store extraction in file(s)
# ------------------------------------------------------------------
raw_ext_file = os.path.basename(p['FITSFILENAME'])
# construct filename
e2dsfits, tag1 = spirouConfig.Constants.EXTRACT_E2DS_FILE(p)
e2dsfitsname = os.path.split(e2dsfits)[-1]
e2dsfffits, tag2 = spirouConfig.Constants.EXTRACT_E2DSFF_FILE(p)
e2dsfffitsname = os.path.split(e2dsfffits)[-1]
e2dsllfits, tag4 = spirouConfig.Constants.EXTRACT_E2DSLL_FILE(p)
e2dsfllitsname = os.path.split(e2dsllfits)[-1]
# log that we are saving E2DS spectrum
wmsg = 'Saving E2DS spectrum of Fiber {0} in {1}'
WLOG(p, '', wmsg.format(p['FIBER'], e2dsfitsname))
wmsg = 'Saving E2DSFF spectrum of Fiber {0} in {1}'
WLOG(p, '', wmsg.format(p['FIBER'], e2dsfffitsname))
# add keys from original header file
hdict = spirouImage.CopyOriginalKeys(hdr)
# set the version
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_FIBER'], value=p['FIBER'])
# set the input files
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBDARK'],
value=p['DARKFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBBAD'],
value=p['BADPFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBLOCO'],
value=p['LOCOFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBBACK'],
value=p['BKGRDFILE'])
if p['IC_EXTRACT_TYPE'] not in EXTRACT_SHAPE_TYPES:
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBTILT'],
value=p['TILTFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBBLAZE'],
value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBFLAT'],
value=p['FLATFILE'])
if p['IC_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES:
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBSHAPEX'],
value=p['SHAPEXFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBSHAPEY'],
value=p['SHAPEYFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBSHAPE'],
value=p['SHAPEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBFPMASTER'],
value=p['FPMASTERFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBTHERMAL'],
value=p['THERMALFILE_{0}'.format(fiber)])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=loc['WAVEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=loc['WSOURCE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='file',
values=p['ARG_FILE_NAMES'])
# construct loco filename
locofile, _ = spirouConfig.Constants.EXTRACT_LOCO_FILE(p)
locofilename = os.path.basename(locofile)
# add barycentric keys to header
hdict = spirouImage.AddKey(p, hdict, p['KW_BERV'], value=loc['BERV'])
hdict = spirouImage.AddKey(p, hdict, p['KW_BJD'], value=loc['BJD'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_BERV_MAX'],
value=loc['BERV_MAX'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_B_OBS_HOUR'],
value=loc['BERVHOUR'])
# add barycentric estimate keys to header
hdict = spirouImage.AddKey(p,
hdict,
p['KW_BERV_EST'],
value=loc['BERV_EST'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_BJD_EST'],
value=loc['BJD_EST'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_BERV_MAX_EST'],
value=loc['BERV_MAX_EST'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_BERV_SOURCE'],
value=loc['BERV_SOURCE'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# copy extraction method and function to header
# (for reproducibility)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_E2DS_EXTM'],
value=extmethod)
hdict = spirouImage.AddKey(p, hdict, p['KW_E2DS_FUNC'], value=extfunc)
# add localization file name to header
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOCO_FILE'],
value=locofilename)
# add wave solution date
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_TIME1'],
value=loc['WAVE_ACQTIMES'][0])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_TIME2'],
value=loc['WAVE_ACQTIMES'][1])
# add wave solution number of orders
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_ORD_N'],
value=loc['WAVEPARAMS'].shape[0])
# add wave solution degree of fit
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_LL_DEG'],
value=loc['WAVEPARAMS'].shape[1] - 1)
# -------------------------------------------------------------------------
# add keys of the wave solution FP CCF
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_FILE'],
value=loc['WAVEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_DRIFT'],
value=p['WFP_DRIFT'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_FWHM'],
value=p['WFP_FWHM'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_CONTRAST'],
value=p['WFP_CONTRAST'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_MAXCPP'],
value=p['WFP_MAXCPP'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_MASK'],
value=p['WFP_MASK'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_LINES'],
value=p['WFP_LINES'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_TARG_RV'],
value=p['WFP_TARG_RV'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_WIDTH'],
value=p['WFP_WIDTH'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_STEP'],
value=p['WFP_STEP'])
# write 1D list of the SNR
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_E2DS_SNR'],
values=loc['SNR'])
# add localization file keys to header
root = p['KW_ROOT_DRS_LOC'][0]
hdict = spirouImage.CopyRootKeys(p, hdict, locofile, root=root)
# add wave solution coefficients
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_WAVE_PARAM'],
values=loc['WAVEPARAMS'])
# Save E2DS file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_EXT_TYPE'],
value=p['DPRTYPE'])
p = spirouImage.WriteImage(p, e2dsfits, loc['E2DS'], hdict)
# Save E2DSFF file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_EXT_TYPE'],
value=p['DPRTYPE'])
p = spirouImage.WriteImage(p, e2dsfffits, loc['E2DSFF'], hdict)
# Save E2DSLL file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_EXT_TYPE'],
value=p['DPRTYPE'])
if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES:
llstack = np.vstack(loc['E2DSLL'])
p = spirouImage.WriteImage(p, e2dsllfits, llstack, hdict)
# ------------------------------------------------------------------
# 1-dimension spectral S1D (uniform in wavelength)
# ------------------------------------------------------------------
# get arguments for E2DS to S1D
e2dsargs = [loc['WAVE'], loc['E2DSFF'], loc['BLAZE']]
# get 1D spectrum
xs1d1, ys1d1 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='wave')
# Plot the 1D spectrum
if p['DRS_PLOT'] > 0:
sPlt.ext_1d_spectrum_plot(p, xs1d1, ys1d1)
# construct file name
s1dfile1, tag3 = spirouConfig.Constants.EXTRACT_S1D_FILE1(p)
s1dfilename1 = os.path.basename(s1dfile1)
# add header keys
# set the version
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_EXT_TYPE'],
value=p['DPRTYPE'])
# log writing to file
wmsg = 'Saving 1D spectrum (uniform in wavelength) for Fiber {0} in {1}'
WLOG(p, '', wmsg.format(p['FIBER'], s1dfilename1))
# Write to file
columns = ['wavelength', 'flux', 'eflux']
values = [xs1d1, ys1d1, np.zeros_like(ys1d1)]
units = ['nm', None, None]
s1d1 = spirouImage.MakeTable(p, columns, values, units=units)
spirouImage.WriteTable(p, s1d1, s1dfile1, header=hdict)
# ------------------------------------------------------------------
# 1-dimension spectral S1D (uniform in velocity)
# ------------------------------------------------------------------
# get arguments for E2DS to S1D
e2dsargs = [loc['WAVE'], loc['E2DSFF'], loc['BLAZE']]
# get 1D spectrum
xs1d2, ys1d2 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='velocity')
# Plot the 1D spectrum
if p['DRS_PLOT'] > 0:
sPlt.ext_1d_spectrum_plot(p, xs1d2, ys1d2)
# construct file name
s1dfile2, tag4 = spirouConfig.Constants.EXTRACT_S1D_FILE2(p)
s1dfilename2 = os.path.basename(s1dfile2)
# add header keys
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_EXT_TYPE'],
value=p['DPRTYPE'])
# log writing to file
wmsg = 'Saving 1D spectrum (uniform in velocity) for Fiber {0} in {1}'
WLOG(p, '', wmsg.format(p['FIBER'], s1dfilename2))
# Write to file
columns = ['wavelength', 'flux', 'eflux']
values = [xs1d2, ys1d2, np.zeros_like(ys1d2)]
units = ['nm', None, None]
s1d2 = spirouImage.MakeTable(p, columns, values, units=units)
spirouImage.WriteTable(p, s1d2, s1dfile2, header=hdict)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None,
e2dsfile=None,
mask=None,
rv=None,
width=None,
step=None):
"""
cal_CCF_E2DS_spirou.py main function, if arguments are None uses
arguments from run time i.e.:
cal_CCF_E2DS_spirou.py [night_directory] [E2DSfilename] [mask] [RV]
[width] [step]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param e2dsfile: string, the E2DS file to use
:param mask: string, the mask file to use (i.e. "UrNe.mas")
:param rv: float, the target RV to use
:param width: float, the CCF width to use
:param step: float, the CCF step to use
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# deal with arguments being None (i.e. get from sys.argv)
pos = [0, 1, 2, 3, 4]
fmt = [str, str, float, float, float]
name = ['e2dsfile', 'ccf_mask', 'target_rv', 'ccf_width', 'ccf_step']
lname = ['input_file', 'CCF_mask', 'RV', 'CCF_width', 'CCF_step']
req = [True, True, True, False, False]
call = [e2dsfile, mask, rv, width, step]
call_priority = [True, True, True, True, True]
# now get custom arguments
customargs = spirouStartup.GetCustomFromRuntime(p, pos, fmt, name, req,
call, call_priority, lname)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsfile='e2dsfile',
mainfitsdir='reduced')
# ----------------------------------------------------------------------
# Construct reference filename and get fiber type
# ----------------------------------------------------------------------
p, e2dsfilename = spirouStartup.SingleFileSetup(p, filename=p['E2DSFILE'])
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# ----------------------------------------------------------------------
# Deal with optional run time arguments
# ----------------------------------------------------------------------
# define default arguments (if ccf_width and ccf_step are not defined
# in function call or run time arguments
if 'ccf_width' not in p:
p['CCF_WIDTH'] = p['IC_CCF_WIDTH']
if 'ccf_step' not in p:
p['CCF_STEP'] = p['IC_CCF_STEP']
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
e2ds, hdr, nbo, nx = spirouImage.ReadData(p, e2dsfilename)
# add to loc
loc = ParamDict()
loc['E2DS'] = e2ds
loc['NUMBER_ORDERS'] = nbo
loc.set_sources(['E2DS', 'number_orders'], __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Get basic image properties for reference file
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# get acquisition time
p = spirouImage.GetAcqTime(p, hdr, name='acqtime', kind='julian')
# get obj name
p = spirouImage.ReadParam(p, hdr, 'KW_OBJNAME', name='OBJNAME', dtype=str)
bjdref = p['ACQTIME']
# set sigdet and conad keywords (sigdet is changed later)
p['KW_CCD_SIGDET'][1] = p['SIGDET']
p['KW_CCD_CONAD'][1] = p['GAIN']
# ----------------------------------------------------------------------
# Earth Velocity calculation
# ----------------------------------------------------------------------
if p['IC_IMAGE_TYPE'] == 'H4RG':
p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, hdr)
# ----------------------------------------------------------------------
# Read wavelength solution
# ----------------------------------------------------------------------
# log
WLOG(p, '', 'Reading wavelength solution ')
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# get wave image
wout = spirouImage.GetWaveSolution(p,
hdr=hdr,
return_wavemap=True,
return_filename=True,
fiber=wave_fiber)
param_ll, wave_ll, wavefile, wsource = wout
# save to storage
loc['PARAM_LL'], loc['WAVE_LL'], loc['WAVEFILE'], loc['WSOURCE'] = wout
source = __NAME__ + '/main() + spirouTHORCA.GetWaveSolution()'
loc.set_sources(['WAVE_LL', 'PARAM_LL', 'WAVEFILE', 'WSOURCE'], source)
# ----------------------------------------------------------------------
# Read Flat file
# ----------------------------------------------------------------------
# TODO We do not need to correct FLAT
# log
# WLOG(p, '', 'Reading Flat-Field ')
# get flat
# loc['FLAT'] = spirouImage.ReadFlatFile(p, hdr)
# loc.set_source('FLAT', __NAME__ + '/main() + /spirouImage.ReadFlatFile')
# get all values in flat that are zero to 1
# loc['FLAT'] = np.where(loc['FLAT'] == 0, 1.0, loc['FLAT'])
# get blaze
# p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hdr)
p, blaze0 = spirouImage.ReadBlazeFile(p, hdr)
# ----------------------------------------------------------------------
# Preliminary set up = no flat, no blaze
# ----------------------------------------------------------------------
# reset flat to all ones
# loc['FLAT'] = np.ones((nbo, nx))
# set blaze to all ones (if not bug in correlbin !!!
# TODO Check why Blaze makes bugs in correlbin
loc['BLAZE'] = np.ones((nbo, nx))
# set sources
# loc.set_sources(['flat', 'blaze'], __NAME__ + '/main()')
loc.set_sources(['blaze'], __NAME__ + '/main()')
# Modification of E2DS array with N.A.N
if np.isnan(np.sum(e2ds)):
WLOG(p, 'warning', 'NaN values found in e2ds, converting process')
# First basic approach Replacing N.A.N by zeros
# e2ds[np.isnan(e2ds)] = 0
# Second approach replacing N.A.N by the Adjusted Blaze
e2dsb = e2ds / blaze0
for i in np.arange(len(e2ds)):
with warnings.catch_warnings(record=True) as _:
rap = np.mean(e2dsb[i][np.isfinite(e2dsb[i])])
if np.isnan(rap):
rap = 0.0
e2ds[i] = np.where(np.isfinite(e2dsb[i]), e2ds[i], blaze0[i] * rap)
# ----------------------------------------------------------------------
# correct extracted image for flat
# ----------------------------------------------------------------------
# loc['E2DSFF'] = e2ds/loc['FLAT']
# loc['E2DSFF'] = e2ds*1.
loc['E2DSFF'] = e2ds
loc.set_source('E2DSFF', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Compute photon noise uncertainty for reference file
# ----------------------------------------------------------------------
# set up the arguments for DeltaVrms2D
dargs = [loc['E2DS'], loc['WAVE_LL']]
dkwargs = dict(sigdet=p['IC_DRIFT_NOISE'],
size=p['IC_DRIFT_BOXSIZE'],
threshold=p['IC_DRIFT_MAXFLUX'])
# run DeltaVrms2D
dvrmsref, wmeanref = spirouRV.DeltaVrms2D(*dargs, **dkwargs)
# save to loc
loc['DVRMSREF'], loc['WMEANREF'] = dvrmsref, wmeanref
loc.set_sources(['dvrmsref', 'wmeanref'], __NAME__ + '/main()()')
# log the estimated RV uncertainty
# wmsg = 'On fiber {0} estimated RV uncertainty on spectrum is {1:.3f} m/s'
# WLOG(p, 'info', wmsg.format(p['FIBER'], wmeanref))
wmsg = 'On fiber estimated RV uncertainty on spectrum is {0:.3f} m/s'
WLOG(p, 'info', wmsg.format(wmeanref))
# TEST N.A.N
# loc['E2DSFF'][20:22,2000:3000]=np.nan
# e2ds[20:30,1000:3000]=np.nan
# ----------------------------------------------------------------------
# Reference plots
# ----------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# start interactive session if needed
sPlt.start_interactive_session(p)
# plot FP spectral order
sPlt.drift_plot_selected_wave_ref(p,
loc,
x=loc['WAVE_LL'],
y=loc['E2DS'])
# plot photon noise uncertainty
sPlt.drift_plot_photon_uncertainty(p, loc)
# ----------------------------------------------------------------------
# Get template RV (from ccf_mask)
# ----------------------------------------------------------------------
# get the CCF mask from file (check location of mask)
loc = spirouRV.GetCCFMask(p, loc)
# check and deal with mask in microns (should be in nm)
if np.mean(loc['LL_MASK_CTR']) < 2.0:
loc['LL_MASK_CTR'] *= 1000.0
loc['LL_MASK_D'] *= 1000.0
# ----------------------------------------------------------------------
# Do correlation
# ----------------------------------------------------------------------
# calculate and fit the CCF
loc = spirouRV.Coravelation(p, loc)
# ----------------------------------------------------------------------
# Correlation stats
# ----------------------------------------------------------------------
# get the maximum number of orders to use
nbmax = p['CCF_NUM_ORDERS_MAX']
# get the average ccf
loc['AVERAGE_CCF'] = np.nansum(loc['CCF'][:nbmax], axis=0)
# normalize the average ccf
normalized_ccf = loc['AVERAGE_CCF'] / np.max(loc['AVERAGE_CCF'])
# get the fit for the normalized average ccf
ccf_res, ccf_fit = spirouRV.FitCCF(p,
loc['RV_CCF'],
normalized_ccf,
fit_type=0)
loc['CCF_RES'] = ccf_res
loc['CCF_FIT'] = ccf_fit
# get the max cpp
loc['MAXCPP'] = np.nansum(loc['CCF_MAX']) / np.nansum(
loc['PIX_PASSED_ALL'])
# get the RV value from the normalised average ccf fit center location
loc['RV'] = float(ccf_res[1])
# get the contrast (ccf fit amplitude)
loc['CONTRAST'] = np.abs(100 * ccf_res[0])
# get the FWHM value
loc['FWHM'] = ccf_res[2] * spirouCore.spirouMath.fwhm()
# ----------------------------------------------------------------------
# set the source
keys = [
'average_ccf', 'maxcpp', 'rv', 'contrast', 'fwhm', 'ccf_res', 'ccf_fit'
]
loc.set_sources(keys, __NAME__ + '/main()')
# ----------------------------------------------------------------------
# log the stats
wmsg = ('Correlation: C={0:.1f}[%] RV={1:.5f}[km/s] '
'FWHM={2:.4f}[km/s] maxcpp={3:.1f}')
wargs = [loc['CONTRAST'], loc['RV'], loc['FWHM'], loc['MAXCPP']]
WLOG(p, 'info', wmsg.format(*wargs))
# ----------------------------------------------------------------------
# rv ccf plot
# ----------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# Plot rv vs ccf (and rv vs ccf_fit)
sPlt.ccf_rv_ccf_plot(p, loc['RV_CCF'], normalized_ccf, ccf_fit)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# TODO: Needs doing
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# add to qc header lists
qc_values.append('None')
qc_names.append('None')
qc_logic.append('None')
qc_pass.append(1)
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ----------------------------------------------------------------------
# archive ccf to table
# ----------------------------------------------------------------------
# construct filename
res_table_file = spirouConfig.Constants.CCF_TABLE_FILE(p)
# log progress
WLOG(p, '', 'Archiving CCF on file {0}'.format(res_table_file))
# define column names
columns = ['order', 'maxcpp', 'nlines', 'contrast', 'RV', 'sig']
# define values for each column
values = [
loc['ORDERS'], loc['CCF_MAX'] / loc['PIX_PASSED_ALL'], loc['TOT_LINE'],
np.abs(100 * loc['CCF_ALL_RESULTS'][:, 0]),
loc['CCF_ALL_RESULTS'][:, 1], loc['CCF_ALL_RESULTS'][:, 2]
]
# define the format for each column
formats = ['2.0f', '5.0f', '4.0f', '4.1f', '9.4f', '7.4f']
# construct astropy table from column names, values and formats
table = spirouImage.MakeTable(p, columns, values, formats)
# save table to file
spirouImage.WriteTable(p, table, res_table_file, fmt='ascii')
# ----------------------------------------------------------------------
# archive ccf to fits file
# ----------------------------------------------------------------------
raw_infile = os.path.basename(p['E2DSFILE'])
# construct folder and filename
corfile, tag = spirouConfig.Constants.CCF_FITS_FILE(p)
corfilename = os.path.split(corfile)[-1]
# log that we are archiving the CCF on file
WLOG(p, '', 'Archiving CCF on file {0}'.format(corfilename))
# get constants from p
mask = p['CCF_MASK']
# if file exists remove it
if os.path.exists(corfile):
os.remove(corfile)
# add the average ccf to the end of ccf
data = np.vstack([loc['CCF'], loc['AVERAGE_CCF']])
# add drs keys
hdict = spirouImage.CopyOriginalKeys(hdr)
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
# set the input files
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=loc['WAVEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=loc['WSOURCE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='file',
values=p['E2DSFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_INCCFMASK'],
value=p['CCF_MASK'])
hdict = spirouImage.AddKey(p, hdict, p['KW_INRV'], value=p['TARGET_RV'])
hdict = spirouImage.AddKey(p, hdict, p['KW_INWIDTH'], value=p['CCF_WIDTH'])
hdict = spirouImage.AddKey(p, hdict, p['KW_INSTEP'], value=p['CCF_STEP'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# add CCF keys
hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CTYPE'], value='km/s')
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CCF_CRVAL'],
value=loc['RV_CCF'][0])
# the rv step
rvstep = np.abs(loc['RV_CCF'][0] - loc['RV_CCF'][1])
hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CDELT'], value=rvstep)
# add ccf stats
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CCF_RV'],
value=loc['CCF_RES'][1])
hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_RVC'], value=loc['RV'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_FWHM'], value=loc['FWHM'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CCF_WMREF'],
value=loc['WMEANREF'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CCF_CONTRAST'],
value=loc['CONTRAST'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CCF_MAXCPP'],
value=loc['MAXCPP'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_MASK'], value=p['CCF_MASK'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CCF_LINES'],
value=np.nansum(loc['TOT_LINE']))
# add berv values
hdict = spirouImage.AddKey(p, hdict, p['KW_BERV'], value=loc['BERV'])
hdict = spirouImage.AddKey(p, hdict, p['KW_BJD'], value=loc['BJD'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_BERV_MAX'],
value=loc['BERV_MAX'])
# write image and add header keys (via hdict)
p = spirouImage.WriteImage(p, corfile, data, hdict)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p,
night_name,
files,
mainfitsdir='reduced')
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# set up function name
main_name = __NAME__ + '.main()'
# ------------------------------------------------------------------
# Load first file
# ------------------------------------------------------------------
loc = ParamDict()
rd = spirouImage.ReadImage(p, p['FITSFILENAME'])
loc['DATA'], loc['DATAHDR'], loc['YDIM'], loc['XDIM'] = rd
loc.set_sources(['DATA', 'DATAHDR', 'XDIM', 'YDIM'], main_name)
# ------------------------------------------------------------------
# Get the wave solution
# ------------------------------------------------------------------
masterwavefile = spirouDB.GetDatabaseMasterWave(p)
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# read master wave map
wout = spirouImage.GetWaveSolution(p,
filename=masterwavefile,
return_wavemap=True,
quiet=True,
return_header=True,
fiber=wave_fiber)
_, loc['WAVE'], loc['WAVEFILE'], _ = wout
loc.set_sources(['WAVE', 'WAVEFILE'], main_name)
# get the wave keys
loc = spirouImage.GetWaveKeys(p, loc, loc['DATAHDR'])
# ------------------------------------------------------------------
# Construct convolution kernels (used in GetMolecularTellLines)
# ------------------------------------------------------------------
loc = spirouTelluric.ConstructConvKernel1(p, loc)
# ------------------------------------------------------------------
# Get molecular telluric lines
# ------------------------------------------------------------------
loc = spirouTelluric.GetMolecularTellLines(p, loc)
# if TAPAS FNAME is not None we generated a new file so should add to tellDB
if loc['TAPAS_FNAME'] is not None:
# add to the telluric database
spirouDB.UpdateDatabaseTellConv(p, loc['TAPAS_FNAME'], loc['DATAHDR'])
# put file in telluDB
spirouDB.PutTelluFile(p, loc['TAPAS_ABSNAME'])
# ----------------------------------------------------------------------
# load the expected atmospheric transmission
# ----------------------------------------------------------------------
# read filename from telluDB
tapas_file_names = spirouDB.GetDatabaseTellConv(p)
tapas_file_name = tapas_file_names[-1]
# load atmospheric transmission
sp_tapas = np.load(tapas_file_name)
loc['TAPAS_ALL_SPECIES'] = sp_tapas
# extract the water and other line-of-sight optical depths
loc['TAPAS_WATER'] = sp_tapas[1, :]
loc['TAPAS_OTHERS'] = np.prod(sp_tapas[2:, :], axis=0)
loc.set_sources(['TAPAS_ALL_SPECIES', 'TAPAS_WATER', 'TAPAS_OTHERS'],
main_name)
# ------------------------------------------------------------------
# Get master wave solution map
# ------------------------------------------------------------------
# get master wave map
masterwavefile = spirouDB.GetDatabaseMasterWave(p)
# log process
wmsg1 = 'Shifting transmission map on to master wavelength grid'
wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile))
WLOG(p, '', [wmsg1, wmsg2])
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# read master wave map
mout = spirouImage.GetWaveSolution(p,
filename=masterwavefile,
return_wavemap=True,
quiet=True,
return_header=True,
fiber=wave_fiber)
masterwavep, masterwave, masterwaveheader, mwsource = mout
# get wave acqtimes
master_acqtimes = spirouDB.GetTimes(p, masterwaveheader)
# ------------------------------------------------------------------
# Loop around the files
# ------------------------------------------------------------------
# construct extension
tellu_ext = '{0}_{1}.fits'
# get current telluric maps from telluDB
# tellu_db_data = spirouDB.GetDatabaseTellMap(p, required=False)
# tellu_db_files = tellu_db_data[0]
# storage for valid output files
loc['OUTPUTFILES'] = []
# loop around the files
for basefilename in p['ARG_FILE_NAMES']:
# ------------------------------------------------------------------
# Get absolute path of filename
# ------------------------------------------------------------------
filename = os.path.join(p['ARG_FILE_DIR'], basefilename)
# ------------------------------------------------------------------
# Read obj telluric file and correct blaze (per order)
# ------------------------------------------------------------------
# get image
sp, shdr, _, _ = spirouImage.ReadImage(p, filename)
# ------------------------------------------------------------------
# check that file has valid DPRTYPE
# ------------------------------------------------------------------
# get FP_FP DPRTYPE
p = spirouImage.ReadParam(p, shdr, 'KW_DPRTYPE', 'DPRTYPE', dtype=str)
# if dprtype is incorrect skip
if p['DPRTYPE'] not in p['ALLOWED_TELLURIC_DPRTYPES']:
wmsg1 = 'Skipping file (DPRTYPE incorrect)'
wmsg2 = '\t DPRTYPE = {0}'.format(p['DPRTYPE'])
WLOG(p, 'warning', [wmsg1, wmsg2])
continue
# get blaze
p, blaze = spirouImage.ReadBlazeFile(p, shdr)
# get the blaze percentile
blaze_p = p['MKTELLU_BLAZE_PERCENTILE']
# loop through blaze orders, normalize blaze by its peak amplitude
for order_num in range(sp.shape[0]):
# normalize the spectrum
spo, bzo = sp[order_num], blaze[order_num]
sp[order_num] = spo / np.nanpercentile(spo, blaze_p)
# normalize the blaze
blaze[order_num] = bzo / np.nanpercentile(bzo, blaze_p)
# find where the blaze is bad
with warnings.catch_warnings(record=True) as _:
badblaze = blaze < p['MKTELLU_CUT_BLAZE_NORM']
# set bad blaze to NaN
blaze[badblaze] = np.nan
# set to NaN values where spectrum is zero
zeromask = sp == 0
sp[zeromask] = np.nan
# divide spectrum by blaze
with warnings.catch_warnings(record=True) as _:
sp = sp / blaze
# add sp to loc
loc['SP'] = sp
loc.set_source('SP', main_name)
# ----------------------------------------------------------------------
# Get object name, airmass and berv
# ----------------------------------------------------------------------
# Get object name
loc['OBJNAME'] = spirouImage.GetObjName(p, shdr)
# Get the airmass
loc['AIRMASS'] = spirouImage.GetAirmass(p, shdr)
# Get the Barycentric correction from header
p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, shdr)
# set sources
source = main_name + '+ spirouImage.ReadParams()'
loc.set_sources(['OBJNAME', 'AIRMASS'], source)
# ------------------------------------------------------------------
# get output transmission filename
outfile, tag1 = spirouConfig.Constants.TELLU_TRANS_MAP_FILE(
p, filename)
outfilename = os.path.basename(outfile)
loc['OUTPUTFILES'].append(outfile)
# ----------------------------------------------------------------------
# Load template (if available)
# ----------------------------------------------------------------------
# read filename from telluDB
template_file = spirouDB.GetDatabaseObjTemp(p,
loc['OBJNAME'],
required=False)
# if we don't have a template flag it
if template_file is None:
loc['FLAG_TEMPLATE'] = False
loc['TEMPLATE'] = None
# construct progres string
pstring = 'No template found.'
else:
loc['FLAG_TEMPLATE'] = True
# load template
template, _, _, _ = spirouImage.ReadImage(p, template_file)
# add to loc
loc['TEMPLATE'] = template
# construct progres string
template_bfile = os.path.basename(template_file)
pstring = 'Using template {0}'.format(template_bfile)
# set the source for flag and template
loc.set_sources(['FLAG_TEMPLATE', 'TEMPLATE'], main_name)
# ------------------------------------------------------------------
# log processing file
wmsg = 'Processing file {0}. {1}'
WLOG(p, '', [wmsg.format(outfilename, pstring)])
# ------------------------------------------------------------------
# Check that basefile is not in blacklist
# ------------------------------------------------------------------
blacklist_check = spirouTelluric.CheckBlackList(loc['OBJNAME'])
if blacklist_check:
# log black list file found
wmsg = 'File {0} is blacklisted (OBJNAME={1}). Skipping'
wargs = [basefilename, loc['OBJNAME']]
WLOG(p, 'warning', wmsg.format(*wargs))
# skip this file
continue
# ------------------------------------------------------------------
# deal with applying template to spectrum
# ------------------------------------------------------------------
# Requires from loc:
# TEMPLATE (None or template loaded from file)
# FLAG_TEMPLATE
# WAVE
# SP
# BERV
#
# Returns:
# SP (modified if template was used)
# TEMPLATE
# WCONV
loc = spirouTelluric.ApplyTemplate(p, loc)
# ------------------------------------------------------------------
# calcullate telluric absorption (with a sigma clip loop)
# ------------------------------------------------------------------
# Requires from loc:
# AIRMASS
# WAVE
# SP
# WCONV
# Returns:
# PASSED [Bool] True or False
# SP_OUT
# SED_OUT
# RECOV_AIRMASS
# RECOV_WATER
loc = spirouTelluric.CalcTelluAbsorption(p, loc)
# calculate tranmission map from sp and sed
transmission_map = loc['SP_OUT'] / loc['SED_OUT']
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# if array is completely NaNs it shouldn't pass
if np.sum(np.isfinite(transmission_map)) == 0:
fail_msg.append('transmission map is all NaNs')
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append('NaN')
qc_names.append('image')
qc_logic.append('image is all NaN')
# ----------------------------------------------------------------------
# get SNR for each order from header
nbo = loc['DATA'].shape[0]
snr_order = p['QC_MK_TELLU_SNR_ORDER']
snr = spirouImage.Read1Dkey(p, shdr, p['kw_E2DS_SNR'][0], nbo)
# check that SNR is high enough
if snr[snr_order] < p['QC_MK_TELLU_SNR_MIN']:
fmsg = 'low SNR in order {0}: ({1:.2f} < {2:.2f})'
fargs = [snr_order, snr[snr_order], p['QC_MK_TELLU_SNR_MIN']]
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(snr[snr_order])
qc_name_str = 'SNR[{0}]'.format(snr_order)
qc_names.append(qc_name_str)
qc_logic.append('{0} < {1:.2f}'.format(qc_name_str,
p['QC_MK_TELLU_SNR_ORDER']))
# ----------------------------------------------------------------------
# check that the file passed the CalcTelluAbsorption sigma clip loop
if not loc['PASSED']:
fmsg = 'File {0} did not converge on a solution in function: {1}'
fargs = [basefilename, 'spirouTelluric.CalcTelluAbsorption()']
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(basefilename)
qc_names.append('FILE')
qc_logic.append('FILE did not converge')
# ----------------------------------------------------------------------
# check that the airmass is not too different from input airmass
airmass_diff = np.abs(loc['RECOV_AIRMASS'] - loc['AIRMASS'])
fargs = [
loc['RECOV_AIRMASS'], loc['AIRMASS'], p['QC_MKTELLU_AIRMASS_DIFF']
]
if airmass_diff > p['QC_MKTELLU_AIRMASS_DIFF']:
fmsg = ('Recovered airmass to de-similar than input airmass.'
'Recovered: {0:.3f}. Input: {1:.3f}. QC limit = {2}')
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(airmass_diff)
qc_names.append('airmass_diff')
qc_logic.append('airmass_diff > {0:.2f}'
''.format(p['QC_MKTELLU_AIRMASS_DIFF']))
# ----------------------------------------------------------------------
# check that the water vapor is within limits
water_cond1 = loc['RECOV_WATER'] < p['MKTELLU_TRANS_MIN_WATERCOL']
water_cond2 = loc['RECOV_WATER'] > p['MKTELLU_TRANS_MAX_WATERCOL']
fargs = [
p['MKTELLU_TRANS_MIN_WATERCOL'], p['MKTELLU_TRANS_MAX_WATERCOL']
]
if water_cond1 or water_cond2:
fmsg = ('Recovered water vapor optical depth not between {0:.3f} '
'and {1:.3f}')
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(loc['RECOV_WATER'])
qc_names.append('RECOV_WATER')
qc_logic.append('RECOV_WATER not between {0:.3f} and {1:.3f}'
''.format(*fargs))
# ----------------------------------------------------------------------
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
continue
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Save transmission map to file
# ------------------------------------------------------------------
# get raw file name
raw_in_file = os.path.basename(p['FITSFILENAME'])
# copy original keys
hdict = spirouImage.CopyOriginalKeys(loc['DATAHDR'])
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
# set the input files
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBBLAZE'],
value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=os.path.basename(masterwavefile))
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=mwsource)
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='file',
values=p['ARG_FILE_NAMES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# add wave solution date
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_TIME1'],
value=master_acqtimes[0])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_TIME2'],
value=master_acqtimes[1])
# add wave solution number of orders
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_ORD_N'],
value=masterwavep.shape[0])
# add wave solution degree of fit
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_LL_DEG'],
value=masterwavep.shape[1] - 1)
# add wave solution coefficients
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_WAVE_PARAM'],
values=masterwavep)
# add telluric keys
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TELLU_AIRMASS'],
value=loc['RECOV_AIRMASS'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TELLU_WATER'],
value=loc['RECOV_WATER'])
# write to file
p = spirouImage.WriteImage(p, outfile, transmission_map, hdict)
# ------------------------------------------------------------------
# Add transmission map to telluDB
# ------------------------------------------------------------------
if p['QC']:
# copy tellu file to the telluDB folder
spirouDB.PutTelluFile(p, outfile)
# update the master tellu DB file with transmission map
targs = [
p, outfilename, loc['OBJNAME'], loc['RECOV_AIRMASS'],
loc['RECOV_WATER']
]
spirouDB.UpdateDatabaseTellMap(*targs)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, e2dsfiles=None):
"""
cal_CCF_E2DS_spirou.py main function, if arguments are None uses
arguments from run time i.e.:
cal_CCF_E2DS_spirou.py [night_directory] [E2DSfilename] [mask] [RV]
[width] [step]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param e2dsfiles: list of string, the E2DS files to use
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# need custom args (to accept full path or wild card
if e2dsfiles is None:
names, types = ['e2dsfiles'], [str]
customargs = spirouStartup.GetCustomFromRuntime(p, [0],
types,
names,
last_multi=True)
else:
customargs = dict(e2dsfiles=e2dsfiles)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsdir='reduced')
# ----------------------------------------------------------------------
# Process files (including wildcards)
# ----------------------------------------------------------------------
try:
e2dsfiles = spirouFile.Paths(p['E2DSFILES'],
root=p['ARG_FILE_DIR']).abs_paths
except PathException as e:
WLOG(p, 'error', e)
# loop around files
for it, e2dsfile in enumerate(e2dsfiles):
# get the base file name
e2dsfilename = os.path.basename(e2dsfile)
# log the file process
wargs = [e2dsfilename, it + 1, len(e2dsfiles)]
wmsg = ' * Processing file {0} ({1} of {2})'.format(*wargs)
WLOG(p, '', spirouStartup.spirouStartup.HEADER)
WLOG(p, '', wmsg)
WLOG(p, '', spirouStartup.spirouStartup.HEADER)
# ------------------------------------------------------------------
# Check that we can process file
# ------------------------------------------------------------------
# check if ufile exists
if not os.path.exists(e2dsfile):
wmsg = 'File {0} does not exist... skipping'
WLOG(p, 'warning', wmsg.format(e2dsfilename))
continue
elif ('e2ds' not in e2dsfilename) and ('e2dsff' not in e2dsfilename):
wmsg = 'File {0} not a valid E2DS or E2DSFF file'
WLOG(p, 'warning', wmsg.format(e2dsfilename))
continue
elif '.fits' not in e2dsfilename:
wmsg = 'File {0} not a fits file... skipping'
WLOG(p, 'warning', wmsg.format(e2dsfilename))
continue
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
e2ds, hdr, nbo, nx = spirouImage.ReadData(p, e2dsfile)
# add to loc
loc = ParamDict()
loc['E2DS'] = e2ds
loc['NUMBER_ORDERS'] = nbo
loc.set_sources(['E2DS', 'number_orders'], __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Get basic image properties for reference file
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# get acquisition time
p = spirouImage.GetAcqTime(p, hdr, name='acqtime', kind='julian')
# ----------------------------------------------------------------------
# Read star parameters
# ----------------------------------------------------------------------
p = spirouImage.ReadParam(p, hdr, 'KW_OBJRA', dtype=str)
p = spirouImage.ReadParam(p, hdr, 'KW_OBJDEC', dtype=str)
p = spirouImage.ReadParam(p, hdr, 'KW_OBJEQUIN')
p = spirouImage.ReadParam(p, hdr, 'KW_OBJRAPM')
p = spirouImage.ReadParam(p, hdr, 'KW_OBJDECPM')
p = spirouImage.ReadParam(p, hdr, 'KW_DATE_OBS', dtype=str)
p = spirouImage.ReadParam(p, hdr, 'KW_UTC_OBS', dtype=str)
# -----------------------------------------------------------------------
# Earth Velocity calculation
# -----------------------------------------------------------------------
if p['IC_IMAGE_TYPE'] == 'H4RG':
loc = spirouImage.EarthVelocityCorrection(p,
loc,
method=p['CCF_BERVMODE'])
else:
loc['BERV'], loc['BJD'] = 0.0, 0.0
loc['BERV_MAX'], loc['BERV_SOURCE'] = 0.0, 'None'
loc.set_sources(['BERV', 'BJD', 'BERV_MAX'], __NAME__ + '.main()')
# ----------------------------------------------------------------------
# archive ccf to fits file
# ----------------------------------------------------------------------
outfilename = str(e2dsfile)
# add keys
hdict = spirouImage.CopyOriginalKeys(hdr)
# add berv values
hdict = spirouImage.AddKey(p, hdict, p['KW_BERV'], value=loc['BERV'])
hdict = spirouImage.AddKey(p, hdict, p['KW_BJD'], value=loc['BJD'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_BERV_MAX'],
value=loc['BERV_MAX'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_BERV_SOURCE'],
value=loc['BERV_SOURCE'])
# write image and add header keys (via hdict)
p = spirouImage.WriteImage(p, outfilename, e2ds, hdict)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p,
night_name,
files,
mainfitsdir='reduced')
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# set up function name
main_name = __NAME__ + '.main()'
# ----------------------------------------------------------------------
# load up first file
# ----------------------------------------------------------------------
# construct loc parameter dictionary for storing data
loc = ParamDict()
# read first file and assign values
rdata = spirouImage.ReadImage(p, p['FITSFILENAME'])
loc['DATA'], loc['DATAHDR'] = rdata[:2]
# Get object name and airmass
loc['OBJNAME'] = spirouImage.GetObjName(p, loc['DATAHDR'])
loc.set_source('OBJNAME', main_name)
loc['AIRMASS'] = spirouImage.GetAirmass(p, loc['DATAHDR'])
# set source
source = main_name + '+ spirouImage.ReadParams()'
loc.set_sources(['OBJNAME', 'AIRMASS'], source)
# ----------------------------------------------------------------------
# Get and Normalise the blaze
# ----------------------------------------------------------------------
p, loc = spirouTelluric.GetNormalizedBlaze(p, loc, loc['DATAHDR'])
# ----------------------------------------------------------------------
# Get database files
# ----------------------------------------------------------------------
# get current telluric maps from telluDB
tellu_db_data = spirouDB.GetDatabaseTellObj(p, required=False)
tellu_db_files, tellu_db_names = tellu_db_data[0], tellu_db_data[1]
# sort files by name
# sortmask = spirouImage.SortByName(tellu_db_files)
# tellu_db_files = np.array(tellu_db_files)[sortmask]
# tellu_db_names = np.array(tellu_db_names)[sortmask]
# filter by object name (only keep OBJNAME objects) and only keep
# unique filenames
tell_files, tell_names = [], []
for it in range(len(tellu_db_files)):
# check that objname is correct
cond1 = loc['OBJNAME'] in tellu_db_names[it]
# check that filename is not already used
cond2 = tellu_db_files[it] not in tell_files
# append to file list if criteria correct
if cond1 and cond2:
tell_files.append(tellu_db_files[it])
tell_names.append(tellu_db_names[it])
# log if we have no files
if len(tell_files) == 0:
wmsg = 'No "TELL_OBJ" files found for object ="{0}" skipping'
WLOG(p, 'warning', wmsg.format(loc['OBJNAME']))
# End Message
wmsg = 'Recipe {0} has been successfully completed'
WLOG(p, 'info', wmsg.format(p['PROGRAM']))
# return a copy of locally defined variables in the memory
return dict(locals())
else:
# log how many found
wmsg = 'N={0} "TELL_OBJ" files found for object ="{1}"'
WLOG(p, 'info', wmsg.format(len(tell_files), loc['OBJNAME']))
# ----------------------------------------------------------------------
# Set up storage for cubes (NaN arrays)
# ----------------------------------------------------------------------
# set up flat size
dims = [loc['DATA'].shape[0], loc['DATA'].shape[1], len(tell_files)]
flatsize = np.product(dims)
# create NaN filled storage
big_cube = np.repeat([np.nan], flatsize).reshape(*dims)
big_cube0 = np.repeat([np.nan], flatsize).reshape(*dims)
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# get master wave map
loc['MASTERWAVEFILE'] = spirouDB.GetDatabaseMasterWave(p)
# read master wave map
mout = spirouImage.GetWaveSolution(p,
filename=loc['MASTERWAVEFILE'],
return_wavemap=True,
quiet=True,
fiber=wave_fiber)
loc['MASTERWAVEPARAMS'], loc['MASTERWAVE'], loc['WSOURCE'] = mout
keys = ['MASTERWAVEPARAMS', 'MASTERWAVE', 'MASTERWAVEFILE', 'WSOURCE']
loc.set_sources(keys, main_name)
# ----------------------------------------------------------------------
# Compile a median SNR for rejection of bad files
# ----------------------------------------------------------------------
snr_all = []
# choose snr to check
snr_order = p['QC_FIT_TELLU_SNR_ORDER']
# loop through files
for it, filename in enumerate(tell_files):
header = spirouImage.ReadHeader(p, filename)
# get the SNR from header
nbo = spirouImage.ReadParam(p,
header,
'KW_WAVE_ORD_N',
dtype=int,
return_value=True)
snr = spirouImage.Read1Dkey(p, header, p['kw_E2DS_SNR'][0], nbo)
# append snr_all
snr_all.append(snr[snr_order])
# work our bad snr (less than half the median SNR)
snr_thres = np.nanmedian(snr_all) / 2.0
bad_snr_objects = np.where(snr_all < snr_thres)[0]
# ----------------------------------------------------------------------
# Loop through input files
# ----------------------------------------------------------------------
# empty lists for storage
loc['BASE_ROWNUM'], loc['BASE_FILELIST'] = [], []
loc['BASE_OBJNAME'], loc['BASE_OBJECT'] = [], []
loc['BASE_BERVLIST'], loc['BASE_WAVELIST'] = [], []
loc['BASE_SNRLIST_{0}'.format(snr_order)] = []
loc['BASE_DATELIST'], loc['BASE_VERSION'] = [], []
loc['BASE_DARKFILE'], loc['BASE_BADFILE'] = [], []
loc['BASE_LOCOFILE'], loc['BASE_BLAZFILE'] = [], []
loc['BASE_FLATFILE'], loc['BASE_SHAPEFILE'] = [], []
# loop through files
for it, filename in enumerate(tell_files):
# get base filenmae
basefilename = os.path.basename(filename)
# ------------------------------------------------------------------
# skip if in bad snr objects
if it in bad_snr_objects:
wargs = [it + 1, len(tell_files), snr_all[it], snr_thres]
wmsg1 = ('Skipping file {0} of {1} due to bad SNR ({2:.3f} < '
'{3:.3f})'.format(*wargs))
wmsg2 = '\tFile = {0}'.format(basefilename)
WLOG(p, 'warning', [wmsg1, wmsg2])
continue
# ------------------------------------------------------------------
# append basename to file list
loc['BASE_FILELIST'].append(basefilename)
# ------------------------------------------------------------------
# create image for storage
image = np.repeat([np.nan], np.product(loc['DATA'].shape))
image = image.reshape(loc['DATA'].shape)
# ------------------------------------------------------------------
# Load the data for this file
tdata0, thdr, _, _ = spirouImage.ReadImage(p, filename)
nbo, npix = tdata0.shape
# Correct for the blaze
tdata = tdata0 / loc['NBLAZE']
# get berv and add to list
if p['KW_BERV'][0] in thdr:
loc['BASE_BERVLIST'].append('{0}'.format(thdr[p['KW_BERV'][0]]))
else:
loc['BASE_BERVLIST'].append('UNKNOWN')
# ------------------------------------------------------------------
# Get parameters from header
snr = snr_all[it]
dateobs = spirouImage.ReadParam(p,
thdr,
'KW_DATE_OBS',
dtype=str,
return_value=True)
utcobs = spirouImage.ReadParam(p,
thdr,
'KW_UTC_OBS',
dtype=str,
return_value=True)
tobjname = spirouImage.ReadParam(p,
thdr,
'KW_OBJNAME',
dtype=str,
return_value=True)
tobject = spirouImage.ReadParam(p,
thdr,
'KW_OBJECT',
dtype=str,
return_value=True)
tversion = spirouImage.ReadParam(p,
thdr,
'KW_version',
dtype=str,
return_value=True)
tdarkfile = spirouImage.ReadParam(p,
thdr,
'KW_CDBDARK',
dtype=str,
return_value=True)
tbadfile = spirouImage.ReadParam(p,
thdr,
'KW_CDBBAD',
dtype=str,
return_value=True)
tlocofile = spirouImage.ReadParam(p,
thdr,
'KW_CDBLOCO',
dtype=str,
return_value=True)
tblazfile = spirouImage.ReadParam(p,
thdr,
'KW_CDBBLAZE',
dtype=str,
return_value=True)
tflatfile = spirouImage.ReadParam(p,
thdr,
'KW_CDBFLAT',
dtype=str,
return_value=True)
tshapfile = spirouImage.ReadParam(p,
thdr,
'KW_CDBSHAPE',
dtype=str,
return_value=True)
# append to lists
loc['BASE_ROWNUM'].append(it)
loc['BASE_SNRLIST_{0}'.format(snr_order)].append(snr_all[it])
loc['BASE_DATELIST'].append('{0}_{1}'.format(dateobs, utcobs))
loc['BASE_OBJNAME'].append(tobjname)
loc['BASE_OBJECT'].append(tobject)
loc['BASE_VERSION'].append(tversion)
loc['BASE_DARKFILE'].append(tdarkfile)
loc['BASE_BADFILE'].append(tbadfile)
loc['BASE_LOCOFILE'].append(tlocofile)
loc['BASE_BLAZFILE'].append(tblazfile)
loc['BASE_FLATFILE'].append(tflatfile)
loc['BASE_SHAPEFILE'].append(tshapfile)
# ------------------------------------------------------------------
# Get the wave solution for this file
# ------------------------------------------------------------------
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# need to reload calibDB
# as we have custom arguments need to load the calibration database
calib_db = dict(p['CALIBDB'])
del p['CALIBDB']
p = spirouStartup.LoadCalibDB(p, header=thdr)
# get wave solution
wout = spirouImage.GetWaveSolution(p,
image=tdata,
hdr=thdr,
return_wavemap=True,
fiber=wave_fiber,
return_filename=True)
_, loc['WAVE'], loc['WAVEFILE'], _ = wout
loc.set_sources(['WAVE', 'WAVEFILE'], main_name)
# add wave to wave list
loc['BASE_WAVELIST'].append(loc['WAVEFILE'])
# readd original calibDB to p
p['CALIBDB'] = dict(calib_db)
# ------------------------------------------------------------------
# Get the Barycentric correction from header
dv, _, _ = spirouImage.GetBERV(p, thdr)
# ------------------------------------------------------------------
# log stats
wmsg = 'Processing file {0} of {1} file={2} dv={3}'
wargs = [it + 1, len(tell_files), basefilename, dv]
WLOG(p, 'info', wmsg.format(*wargs))
# ------------------------------------------------------------------
# shift to correct berv
dvshift = spirouMath.relativistic_waveshift(dv, units='km/s')
image = spirouTelluric.Wave2Wave(p, tdata, loc['WAVE'] * dvshift,
loc['MASTERWAVE'])
# ------------------------------------------------------------------
# loop around orders
for order_num in range(loc['DATA'].shape[0]):
# normalise the tdata
tdata[order_num, :] /= np.nanmedian(tdata[order_num, :])
image[order_num, :] /= np.nanmedian(image[order_num, :])
# ------------------------------------------------------------------
# add to cube storage
big_cube[:, :, it] = image
big_cube0[:, :, it] = tdata
# ----------------------------------------------------------------------
# log if we have no files
if len(loc['BASE_FILELIST']) == 0:
wmsg = 'No good files found for object ="{0}" skipping'
WLOG(p, 'warning', wmsg.format(loc['OBJNAME']))
# End Message
wmsg = 'Recipe {0} has been successfully completed'
WLOG(p, 'info', wmsg.format(p['PROGRAM']))
# return a copy of locally defined variables in the memory
return dict(locals())
# ----------------------------------------------------------------------
# make median image
with warnings.catch_warnings(record=True) as _:
big_cube_med = np.nanmedian(big_cube, axis=2)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# TODO: Needs doing
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# add to qc header lists
qc_values.append('None')
qc_names.append('None')
qc_logic.append('None')
qc_pass.append(1)
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ----------------------------------------------------------------------
# Write Cube median (the template) to file
# ----------------------------------------------------------------------
# get raw file name
raw_in_file = os.path.basename(p['FITSFILENAME'])
# construct filename
outfile, tag = spirouConfig.Constants.OBJTELLU_TEMPLATE_FILE(p, loc)
outfilename = os.path.basename(outfile)
# hdict is first file keys
hdict = spirouImage.CopyOriginalKeys(loc['DATAHDR'])
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
# set the input files
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=loc['MASTERWAVEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=loc['WSOURCE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='file',
values=p['ARG_FILE_NAMES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# add wave solution coefficients
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_WAVE_PARAM'],
values=loc['MASTERWAVEPARAMS'])
# write to file
p = spirouImage.WriteImage(p, outfile, big_cube_med, hdict)
# ----------------------------------------------------------------------
# Update the telluric database with the template
# ----------------------------------------------------------------------
spirouDB.UpdateDatabaseObjTemp(p, outfilename, loc['OBJNAME'],
loc['DATAHDR'])
# put file in telluDB
spirouDB.PutTelluFile(p, outfile)
# ----------------------------------------------------------------------
# Save cubes to file
# ----------------------------------------------------------------------
# make big cube table
big_table = spirouTelluric.ConstructBigTable(p, loc)
# construct file names
outfile1, tag1 = spirouConfig.Constants.OBJTELLU_TEMPLATE_CUBE_FILE1(
p, loc)
outfile2, tag2 = spirouConfig.Constants.OBJTELLU_TEMPLATE_CUBE_FILE2(
p, loc)
# log big cube 1
wmsg1 = 'Saving bigcube to file {0}'.format(os.path.basename(outfile1))
# save big cube 1
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
big_cube_s = np.swapaxes(big_cube, 1, 2)
p = spirouImage.WriteImageTable(p,
outfile1,
image=big_cube_s,
table=big_table,
hdict=hdict)
# log big cube 0
wmsg = 'Saving bigcube0 to file {0}'.format(os.path.basename(outfile2))
# save big cube 0
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
big_cube_s0 = np.swapaxes(big_cube0, 1, 2)
p = spirouImage.WriteImageTable(p,
outfile2,
image=big_cube_s0,
table=big_table,
hdict=hdict)
# # mega plot
# nfiles = big_cube_s0.shape[1]
# ncols = int(np.ceil(np.sqrt(nfiles)))
# nrows = int(np.ceil(nfiles/ncols))
# fig, frames = plt.subplots(ncols=ncols, nrows=nrows)
# for it in range(big_cube_s0.shape[1]):
# jt, kt = it // ncols, it % ncols
# frame = frames[jt][kt]
# frame.imshow(big_cube_s0[:, it, :], origin='lower')
# frame.set(xlim=(2030, 2060))
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, hcfile=None, fpfiles=None):
"""
cal_SLIT_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_SLIT_spirou.py [night_directory] [files]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
if hcfile is None or fpfiles is None:
names, types = ['hcfile', 'fpfiles'], [str, str]
customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1],
types,
names,
last_multi=True)
else:
customargs = dict(hcfile=hcfile, fpfile=fpfiles)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsfile='fpfiles')
# ----------------------------------------------------------------------
# Construct reference filename and get fiber type
# ----------------------------------------------------------------------
p, hcfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['HCFILE'])
p, fpfitsfiles = spirouStartup.MultiFileSetup(p, files=p['FPFILES'])
# set fiber (it doesn't matter with the 2D image but we need this to get
# the lamp type for FPFILES and HCFILES, AB == C
p['FIBER'] = 'AB'
p['FIB_TYP'] = [p['FIBER']]
fsource = __NAME__ + '/main()'
p.set_sources(['FIBER', 'FIB_TYP'], fsource)
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# add a force plot off
p['PLOT_PER_ORDER'] = PLOT_PER_ORDER
p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()')
# ----------------------------------------------------------------------
# Read FP and HC files
# ----------------------------------------------------------------------
# read input fp and hc data
rkwargs = dict(filename=fpfitsfiles[0],
filenames=fpfitsfiles[1:],
framemath='add')
p, fpdata, fphdr = spirouImage.ReadImageAndCombine(p, **rkwargs)
hcdata, hchdr, _, _ = spirouImage.ReadImage(p, hcfitsfilename)
# add data and hdr to loc
loc = ParamDict()
loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr
loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr
# set the source
sources = ['HCDATA', 'HCHDR']
loc.set_sources(sources, 'spirouImage.ReadImage()')
sources = ['FPDATA', 'FPHDR']
loc.set_sources(sources, 'spirouImage.ReadImage()')
# ---------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
hcdata = spirouImage.FixNonPreProcess(p, hcdata)
fpdata = spirouImage.FixNonPreProcess(p, fpdata)
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# add a force plot off
p['PLOT_PER_ORDER'] = PLOT_PER_ORDER
p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()')
# ----------------------------------------------------------------------
# Get basic image properties for reference file
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, fphdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, fphdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, fphdr, name='gain')
# get lamp parameters
p = spirouTHORCA.GetLampParams(p, hchdr)
# get FP_FP DPRTYPE
p = spirouImage.ReadParam(p, fphdr, 'KW_DPRTYPE', 'DPRTYPE', dtype=str)
# ----------------------------------------------------------------------
# Correction of reference FP
# ----------------------------------------------------------------------
# set the number of frames
p['NBFRAMES'] = len(fpfitsfiles)
p.set_source('NBFRAMES', __NAME__ + '.main()')
# Correction of DARK
p, fpdatac = spirouImage.CorrectForDark(p, fpdata, fphdr)
# Resize hc data
# rotate the image and convert from ADU/s to e-
fpdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, fpdatac), p=p)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'],
xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'],
yhigh=p['IC_CCDY_HIGH'],
getshape=False)
fpdata1 = spirouImage.ResizeImage(p, fpdata, **bkwargs)
# log change in data size
WLOG(p, '',
('FPref Image format changed to {0}x{1}').format(*fpdata1.shape))
# Correct for the BADPIX mask (set all bad pixels to zero)
bargs = [p, fpdata1, fphdr]
p, fpdata1 = spirouImage.CorrectForBadPix(*bargs)
p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True)
# log progress
WLOG(p, '', 'Cleaning FPref hot pixels')
# correct hot pixels
fpdata1 = spirouEXTOR.CleanHotpix(fpdata1, badpixmask)
# add to loc
loc['FPDATA1'] = fpdata1
loc.set_source('FPDATA1', __NAME__ + '.main()')
# Log the number of dead pixels
# get the number of bad pixels
with warnings.catch_warnings(record=True) as _:
n_bad_pix = np.nansum(fpdata1 <= 0)
n_bad_pix_frac = n_bad_pix * 100 / np.product(fpdata1.shape)
# Log number
wmsg = 'Nb FPref dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# ----------------------------------------------------------------------
# Correction of HC
# ----------------------------------------------------------------------
# set the number of frames
p['NBFRAMES'] = 1
p.set_source('NBFRAMES', __NAME__ + '.main()')
# Correction of DARK
p, hcdatac = spirouImage.CorrectForDark(p, hcdata, hchdr)
# Resize hc data
# rotate the image and convert from ADU/s to e-
hcdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, hcdatac), p=p)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'],
xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'],
yhigh=p['IC_CCDY_HIGH'],
getshape=False)
hcdata1 = spirouImage.ResizeImage(p, hcdata, **bkwargs)
# log change in data size
WLOG(p, '', ('HC Image format changed to {0}x{1}').format(*hcdata1.shape))
# Correct for the BADPIX mask (set all bad pixels to zero)
bargs = [p, hcdata1, hchdr]
p, hcdata1 = spirouImage.CorrectForBadPix(*bargs)
p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True)
# log progress
WLOG(p, '', 'Cleaning HC hot pixels')
# correct hot pixels
hcdata1 = spirouEXTOR.CleanHotpix(hcdata1, badpixmask)
# add to loc
loc['HCDATA1'] = hcdata1
loc.set_source('HCDATA1', __NAME__ + '.main()')
# Log the number of dead pixels
# get the number of bad pixels
with warnings.catch_warnings(record=True) as _:
n_bad_pix = np.nansum(hcdata1 <= 0)
n_bad_pix_frac = n_bad_pix * 100 / np.product(hcdata1.shape)
# Log number
wmsg = 'Nb HC dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# -------------------------------------------------------------------------
# get all FP_FP files
# -------------------------------------------------------------------------
fpfilenames = spirouImage.FindFiles(p,
filetype=p['DPRTYPE'],
allowedtypes=p['ALLOWED_FP_TYPES'])
# convert filenames to a numpy array
fpfilenames = np.array(fpfilenames)
# julian date to know which file we need to
# process together
fp_time = np.zeros(len(fpfilenames))
basenames, fp_exp, fp_pp_version, nightnames = [], [], [], []
# log progress
WLOG(p, '', 'Reading all fp file headers')
# looping through the file headers
for it in range(len(fpfilenames)):
# log progress
wmsg = '\tReading file {0} / {1}'
WLOG(p, 'info', wmsg.format(it + 1, len(fpfilenames)))
# get fp filename
fpfilename = fpfilenames[it]
# get night name
night_name = os.path.dirname(fpfilenames[it]).split(p['TMP_DIR'])[-1]
# read data
data_it, hdr_it, _, _ = spirouImage.ReadImage(p, fpfilename)
# get header
hdr = spirouImage.ReadHeader(p, filepath=fpfilenames[it])
# add MJDATE to dark times
fp_time[it] = float(hdr[p['KW_ACQTIME'][0]])
# add other keys (for tabular output)
basenames.append(os.path.basename(fpfilenames[it]))
nightnames.append(night_name)
fp_exp.append(float(hdr[p['KW_EXPTIME'][0]]))
fp_pp_version.append(hdr[p['KW_PPVERSION'][0]])
# -------------------------------------------------------------------------
# match files by date
# -------------------------------------------------------------------------
# log progress
wmsg = 'Matching FP files by observation time (+/- {0} hrs)'
WLOG(p, '', wmsg.format(p['DARK_MASTER_MATCH_TIME']))
# get the time threshold
time_thres = p['FP_MASTER_MATCH_TIME']
# get items grouped by time
matched_id = spirouImage.GroupFilesByTime(p, fp_time, time_thres)
# -------------------------------------------------------------------------
# construct the master fp file (+ correct for dark/badpix)
# -------------------------------------------------------------------------
cargs = [fpdata1, fpfilenames, matched_id]
fpcube, transforms = spirouImage.ConstructMasterFP(p, *cargs)
# log process
wmsg1 = 'Master FP construction complete.'
wmsg2 = '\tAdding {0} group images to form FP master image'
WLOG(p, 'info', [wmsg1, wmsg2.format(len(fpcube))])
# sum the cube to make fp data
masterfp = np.sum(fpcube, axis=0)
# add to loc
loc['MASTERFP'] = masterfp
loc.set_source('MASTERFP', __NAME__ + '.main()')
# ------------------------------------------------------------------
# Get localisation coefficients
# ------------------------------------------------------------------
# original there is a loop but it is not used --> removed
p = spirouImage.FiberParams(p, p['FIBER'], merge=True)
# get localisation fit coefficients
p, loc = spirouLOCOR.GetCoeffs(p, fphdr, loc)
# ------------------------------------------------------------------
# Get master wave solution map
# ------------------------------------------------------------------
# get master wave map
masterwavefile = spirouDB.GetDatabaseMasterWave(p)
# log process
wmsg1 = 'Getting master wavelength grid'
wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile))
WLOG(p, '', [wmsg1, wmsg2])
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# read master wave map
wout = spirouImage.GetWaveSolution(p,
filename=masterwavefile,
return_wavemap=True,
quiet=True,
return_header=True,
fiber=wave_fiber)
loc['MASTERWAVEP'], loc['MASTERWAVE'] = wout[:2]
loc['MASTERWAVEHDR'], loc['WSOURCE'] = wout[2:]
# set sources
wsource = ['MASTERWAVEP', 'MASTERWAVE', 'MASTERWAVEHDR']
loc.set_sources(wsource, 'spirouImage.GetWaveSolution()')
# ----------------------------------------------------------------------
# Read UNe solution
# ----------------------------------------------------------------------
wave_u_ne, amp_u_ne = spirouImage.ReadLineList(p)
loc['LL_LINE'], loc['AMPL_LINE'] = wave_u_ne, amp_u_ne
source = __NAME__ + '.main() + spirouImage.ReadLineList()'
loc.set_sources(['LL_LINE', 'AMPL_LINE'], source)
# ----------------------------------------------------------------------
# Read cavity length file
# ----------------------------------------------------------------------
loc['CAVITY_LEN_COEFFS'] = spirouImage.ReadCavityLength(p)
source = __NAME__ + '.main() + spirouImage.ReadCavityLength()'
loc.set_source('CAVITY_LEN_COEFFS', source)
# ----------------------------------------------------------------------
# Calculate shape map
# ----------------------------------------------------------------------
# calculate dx map
loc = spirouImage.GetXShapeMap(p, loc)
# if dx map is None we shouldn't continue
if loc['DXMAP'] is None:
fargs = [
loc['MAXDXMAPINFO'][0], loc['MAXDXMAPINFO'][1], loc['MAXDXMAPSTD'],
p['SHAPE_QC_DXMAP_STD']
]
fmsg = ('The std of the dxmap for order {0} y-pixel {1} is too large.'
' std = {2} (limit = {3})'.format(*fargs))
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(fmsg))
WLOG(p, 'warning', 'Cannot continue. Exiting.')
# End Message
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
# calculate dymap
loc = spirouImage.GetYShapeMap(p, loc, fphdr)
# ------------------------------------------------------------------
# Need to straighten the dxmap
# ------------------------------------------------------------------
# copy it first
loc['DXMAP0'] = np.array(loc['DXMAP'])
# straighten it
loc['DXMAP'] = spirouImage.EATransform(loc['DXMAP'], dymap=loc['DYMAP'])
# ------------------------------------------------------------------
# Need to straighten the hc data and fp data for debug
# ------------------------------------------------------------------
# log progress
WLOG(p, '', 'Shape finding complete. Applying transforms.')
# apply very last update of the debananafication
tkwargs = dict(dxmap=loc['DXMAP'], dymap=loc['DYMAP'])
loc['HCDATA2'] = spirouImage.EATransform(loc['HCDATA1'], **tkwargs)
loc['FPDATA2'] = spirouImage.EATransform(loc['FPDATA1'], **tkwargs)
loc.set_sources(['HCDATA2', 'FPDATA2'], __NAME__ + '.main()')
# ------------------------------------------------------------------
# Plotting
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# plots setup: start interactive plot
sPlt.start_interactive_session(p)
# plot the shape process for one order
sPlt.slit_shape_angle_plot(p, loc)
# end interactive section
sPlt.end_interactive_session(p)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# TODO: Decide on some quality control criteria?
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# add to qc header lists
qc_values.append(loc['MAXDXMAPSTD'])
qc_names.append('DXMAP STD')
qc_logic.append('DXMAP STD < {0}'.format(p['SHAPE_QC_DXMAP_STD']))
qc_pass.append(1)
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Writing FP big table
# ------------------------------------------------------------------
# construct big fp table
colnames = [
'FILENAME', 'NIGHT', 'MJDATE', 'EXPTIME', 'PVERSION', 'GROUPID',
'DXREF', 'DYREF', 'A', 'B', 'C', 'D'
]
values = [
basenames, nightnames, fp_time, fp_exp, fp_pp_version, matched_id,
transforms[:, 0], transforms[:, 1], transforms[:, 2], transforms[:, 3],
transforms[:, 4], transforms[:, 5]
]
fptable = spirouImage.MakeTable(p, colnames, values)
# ------------------------------------------------------------------
# Writing DXMAP to file
# ------------------------------------------------------------------
# get the raw tilt file name
raw_shape_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
shapexfits, tag = spirouConfig.Constants.SLIT_XSHAPE_FILE(p)
shapexfitsname = os.path.basename(shapexfits)
# Log that we are saving tilt file
wmsg = 'Saving shape x information in file: {0}'
WLOG(p, '', wmsg.format(shapexfitsname))
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(fphdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='hcfile',
values=p['HCFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE2'],
dim1name='fpfile',
values=p['FPFILES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# write tilt file to file
p = spirouImage.WriteImageTable(p,
shapexfits,
image=loc['DXMAP'],
table=fptable,
hdict=hdict)
# ------------------------------------------------------------------
# Writing DYMAP to file
# ------------------------------------------------------------------
# get the raw tilt file name
raw_shape_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
shapeyfits, tag = spirouConfig.Constants.SLIT_YSHAPE_FILE(p)
shapeyfitsname = os.path.basename(shapeyfits)
# Log that we are saving tilt file
wmsg = 'Saving shape y information in file: {0}'
WLOG(p, '', wmsg.format(shapeyfitsname))
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(fphdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='hcfile',
values=p['HCFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE2'],
dim1name='fpfile',
values=p['FPFILES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# write tilt file to file
p = spirouImage.WriteImageTable(p,
shapeyfits,
image=loc['DYMAP'],
table=fptable,
hdict=hdict)
# ------------------------------------------------------------------
# Writing Master FP to file
# ------------------------------------------------------------------
# get the raw tilt file name
raw_shape_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
fpmasterfits, tag = spirouConfig.Constants.SLIT_MASTER_FP_FILE(p)
fpmasterfitsname = os.path.basename(fpmasterfits)
# Log that we are saving tilt file
wmsg = 'Saving master FP file: {0}'
WLOG(p, '', wmsg.format(fpmasterfitsname))
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(fphdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='hcfile',
values=p['HCFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE2'],
dim1name='fpfile',
values=p['FPFILES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# write tilt file to file
p = spirouImage.WriteImageTable(p,
fpmasterfits,
image=masterfp,
table=fptable,
hdict=hdict)
# ------------------------------------------------------------------
# Writing sanity check files
# ------------------------------------------------------------------
if p['SHAPE_DEBUG_OUTPUTS']:
# log
WLOG(p, '', 'Saving debug sanity check files')
# construct file names
input_fp_file, tag1 = spirouConfig.Constants.SLIT_SHAPE_IN_FP_FILE(p)
output_fp_file, tag2 = spirouConfig.Constants.SLIT_SHAPE_OUT_FP_FILE(p)
input_hc_file, tag3 = spirouConfig.Constants.SLIT_SHAPE_IN_HC_FILE(p)
output_hc_file, tag4 = spirouConfig.Constants.SLIT_SHAPE_OUT_HC_FILE(p)
bdxmap_file, tag5 = spirouConfig.Constants.SLIT_SHAPE_BDXMAP_FILE(p)
# write input fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
p = spirouImage.WriteImage(p, input_fp_file, loc['FPDATA1'], hdict)
# write output fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
p = spirouImage.WriteImage(p, output_fp_file, loc['FPDATA2'], hdict)
# write input fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
p = spirouImage.WriteImage(p, input_hc_file, loc['HCDATA1'], hdict)
# write output fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
p = spirouImage.WriteImage(p, output_hc_file, loc['HCDATA2'], hdict)
# write overlap file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag5)
p = spirouImage.WriteImage(p, bdxmap_file, loc['DXMAP0'], hdict)
# ----------------------------------------------------------------------
# Move to calibDB and update calibDB
# ----------------------------------------------------------------------
if p['QC']:
# add shape x
keydb = 'SHAPEX'
# copy shape file to the calibDB folder
spirouDB.PutCalibFile(p, shapexfits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, shapexfitsname, fphdr)
# add shape y
keydb = 'SHAPEY'
# copy shape file to the calibDB folder
spirouDB.PutCalibFile(p, shapeyfits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, shapeyfitsname, fphdr)
# add fp master
keydb = 'FPMASTER'
# copy shape file to the calibDB folder
spirouDB.PutCalibFile(p, fpmasterfits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, fpmasterfitsname, fphdr)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p,
night_name,
files,
mainfitsdir='reduced')
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# set up function name
main_name = __NAME__ + '.main()'
# ----------------------------------------------------------------------
# Load first file
# ----------------------------------------------------------------------
loc = ParamDict()
rd = spirouImage.ReadImage(p, p['FITSFILENAME'])
loc['DATA'], loc['DATAHDR'], loc['XDIM'], loc['YDIM'] = rd
loc.set_sources(['DATA', 'DATAHDR', 'XDIM', 'YDIM'], main_name)
# ----------------------------------------------------------------------
# Get object name, airmass and berv
# ----------------------------------------------------------------------
# Get object name
loc['OBJNAME'] = spirouImage.GetObjName(p, loc['DATAHDR'])
# Get the airmass
loc['AIRMASS'] = spirouImage.GetAirmass(p, loc['DATAHDR'])
# Get the Barycentric correction from header
p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, loc['DATAHDR'])
# set sources
source = main_name + '+ spirouImage.ReadParams()'
loc.set_sources(['OBJNAME', 'AIRMASS'], source)
loc.set_sources(['OBJNAME', 'AIRMASS'], source)
# ----------------------------------------------------------------------
# Read wavelength solution
# ----------------------------------------------------------------------
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# used for plotting only
wout = spirouImage.GetWaveSolution(p,
image=loc['DATA'],
hdr=loc['DATAHDR'],
return_wavemap=True,
fiber=wave_fiber)
_, loc['WAVE'], _ = wout
loc.set_source('WAVE', main_name)
# ----------------------------------------------------------------------
# Get and Normalise the blaze
# ----------------------------------------------------------------------
p, loc = spirouTelluric.GetNormalizedBlaze(p, loc, loc['DATAHDR'])
# ----------------------------------------------------------------------
# Load transmission files
# ----------------------------------------------------------------------
transdata = spirouDB.GetDatabaseTellMap(p)
trans_files = transdata[0]
# make sure we have unique filenames for trans_files
trans_files = np.unique(trans_files)
# ----------------------------------------------------------------------
# Start plotting
# ----------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# start interactive plot
sPlt.start_interactive_session(p)
# ----------------------------------------------------------------------
# Load template (if available)
# ----------------------------------------------------------------------
# read filename from telluDB
template_file = spirouDB.GetDatabaseObjTemp(p,
loc['OBJNAME'],
required=False)
# if we don't have a template flag it
if template_file is None:
loc['FLAG_TEMPLATE'] = False
loc['TEMPLATE'] = None
else:
loc['FLAG_TEMPLATE'] = True
# load template
template, _, _, _ = spirouImage.ReadImage(p, template_file)
# add to loc
loc['TEMPLATE'] = template
# set the source for flag and template
loc.set_sources(['FLAG_TEMPLATE', 'TEMPLATE'], main_name)
# ----------------------------------------------------------------------
# load the expected atmospheric transmission
# ----------------------------------------------------------------------
# read filename from telluDB
tapas_file_names = spirouDB.GetDatabaseTellConv(p)
tapas_file_name = tapas_file_names[-1]
# load atmospheric transmission
loc['TAPAS_ALL_SPECIES'] = np.load(tapas_file_name)
loc.set_source('TAPAS_ALL_SPECIES', main_name)
# ----------------------------------------------------------------------
# Generate the absorption map
# ----------------------------------------------------------------------
# get number of files
nfiles = len(trans_files)
npc = p['TELLU_NUMBER_OF_PRINCIPLE_COMP']
# check that we have enough files (greater than number of principle
# components)
if nfiles <= npc:
emsg1 = 'Not enough "TELL_MAP" files in telluDB to run PCA analysis'
emsg2 = '\tNumber of files = {0}, number of PCA components = {1}'
emsg3 = '\tNumber of files > number of PCA components'
emsg4 = '\tAdd more files or reduce number of PCA components'
WLOG(p, 'error', [emsg1, emsg2.format(nfiles, npc), emsg3, emsg4])
# check whether we can used pre-saved abso
filetime = spirouImage.GetMostRecent(trans_files)
tout = spirouConfig.Constants.TELLU_ABSO_SAVE(p, filetime)
abso_save_file, absoprefix = tout
use_saved = os.path.exists(abso_save_file)
# noinspection PyBroadException
try:
# try loading from file
abso = np.load(abso_save_file)
# log progress
wmsg = 'Loaded abso from file {0}'.format(abso_save_file)
WLOG(p, '', wmsg)
except:
# set up storage for the absorption
abso = np.zeros([nfiles, np.product(loc['DATA'].shape)])
# loop around outputfiles and add them to abso
for it, filename in enumerate(trans_files):
# load data
data_it, _, _, _ = spirouImage.ReadImage(p, filename=filename)
# push data into array
abso[it, :] = data_it.reshape(np.product(loc['DATA'].shape))
# log progres
wmsg = 'Saving abso to file {0}'.format(abso_save_file)
WLOG(p, '', wmsg)
# remove all abso save files (only need most recent one)
afolder = os.path.dirname(abso_save_file)
afilelist = os.listdir(afolder)
for afile in afilelist:
if afile.startswith(absoprefix):
os.remove(os.path.join(afolder, afile))
# save to file for later use
np.save(abso_save_file, abso)
# filter out rows of all NaNs
# TODO: Why are we getting all NaN e2ds files?
abso_filtered = []
for row in range(len(abso)):
if np.sum(np.isnan(abso[row])) != len(abso[row]):
abso_filtered.append(abso[row])
else:
wargs = [trans_files[row]]
WLOG(p, '', 'Removing trans file {0} (all NaN)'.format(*wargs))
abso = np.array(abso_filtered)
# log the absorption cube
with warnings.catch_warnings(record=True) as w:
log_abso = np.log(abso)
# ----------------------------------------------------------------------
# Locate valid pixels for PCA
# ----------------------------------------------------------------------
# determining the pixels relevant for PCA construction
keep = np.isfinite(np.sum(abso, axis=0))
# log fraction of valid (non NaN) pixels
fraction = np.nansum(keep) / len(keep)
wmsg = 'Fraction of valid pixels (not NaNs) for PCA construction = {0:.3f}'
WLOG(p, '', wmsg.format(fraction))
# log fraction of valid pixels > 1 - (1/e)
with warnings.catch_warnings(record=True) as w:
keep &= np.min(log_abso, axis=0) > -1
fraction = np.nansum(keep) / len(keep)
wmsg = 'Fraction of valid pixels with transmission > 1 - (1/e) = {0:.3f}'
WLOG(p, '', wmsg.format(fraction))
# ----------------------------------------------------------------------
# Perform PCA analysis on the log of the telluric absorption map
# ----------------------------------------------------------------------
# Requires p:
# TELLU_NUMBER_OF_PRINCIPLE_COMP
# ADD_DERIV_PC
# FIT_DERIV_PC
# Requires loc:
# DATA
# Returns loc:
# PC
# NPC
# FIT_PC
loc = spirouTelluric.CalculateAbsorptionPCA(p, loc, log_abso, keep)
# Plot PCA components
# debug plot
if p['DRS_PLOT'] and (p['DRS_DEBUG'] > 1):
# plot the transmission map plot
sPlt.tellu_pca_comp_plot(p, loc)
# ----------------------------------------------------------------------
# Get master wavelength grid for shifting
# ----------------------------------------------------------------------
# get master wave map
loc['MASTERWAVEFILE'] = spirouDB.GetDatabaseMasterWave(p)
loc.set_source('MASTERWAVEFILE', main_name)
# log progress
wmsg1 = 'Getting master wavelength grid'
wmsg2 = '\tFile = {0}'.format(os.path.basename(loc['MASTERWAVEFILE']))
WLOG(p, '', [wmsg1, wmsg2])
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# read master wave map
mout = spirouImage.GetWaveSolution(p,
filename=loc['MASTERWAVEFILE'],
return_wavemap=True,
quiet=True,
fiber=wave_fiber)
_, loc['MASTERWAVE'], _ = mout
loc.set_source('MASTERWAVE', main_name)
# ----------------------------------------------------------------------
# Loop around telluric files
# ----------------------------------------------------------------------
for basefilename in p['ARG_FILE_NAMES']:
# ------------------------------------------------------------------
# Construct absolute file path
# ------------------------------------------------------------------
filename = os.path.join(p['ARG_FILE_DIR'], basefilename)
# ------------------------------------------------------------------
# Construct output file names
# ------------------------------------------------------------------
outfile1, tag1 = CONSTANTS.TELLU_FIT_OUT_FILE(p, filename)
outfilename1 = os.path.basename(outfile1)
outfile2, tag2 = CONSTANTS.TELLU_FIT_RECON_FILE(p, filename)
outfilename2 = os.path.basename(outfile2)
# ------------------------------------------------------------------
# Read filename
# ------------------------------------------------------------------
# read image
tdata, thdr, _, _ = spirouImage.ReadImage(p, filename)
# normalise with blaze function
loc['SP'] = tdata / loc['NBLAZE']
loc.set_source('SP', main_name)
# ------------------------------------------------------------------
# check that file has valid DPRTYPE
# ------------------------------------------------------------------
# get FP_FP DPRTYPE
p = spirouImage.ReadParam(p, thdr, 'KW_DPRTYPE', 'DPRTYPE', dtype=str)
# if dprtype is incorrect skip
if p['DPRTYPE'] not in p['ALLOWED_TELLURIC_DPRTYPES']:
wmsg1 = 'Skipping file (DPRTYPE incorrect)'
wmsg2 = '\t DPRTYPE = {0}'.format(p['DPRTYPE'])
WLOG(p, 'warning', [wmsg1, wmsg2])
continue
# ------------------------------------------------------------------
# Set storage
# ------------------------------------------------------------------
loc['RECON_ABSO'] = np.ones(np.product(loc['DATA'].shape))
loc['AMPS_ABSOL_TOTAL'] = np.zeros(loc['NPC'])
loc.set_sources(['RECON_ABSO', 'AMPS_ABSOL_TOTAL'], main_name)
# ------------------------------------------------------------------
# Read wavelength solution
# ------------------------------------------------------------------
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# get wavelength solution
wout = spirouImage.GetWaveSolution(p,
image=tdata,
hdr=thdr,
return_wavemap=True,
return_filename=True,
fiber=wave_fiber)
_, loc['WAVE_IT'], loc['WAVEFILE'], loc['WSOURCE'] = wout
loc.set_sources(['WAVE_IT', 'WAVEFILE', 'WSOURCE'], main_name)
# load wave keys
loc = spirouImage.GetWaveKeys(p, loc, thdr)
# ------------------------------------------------------------------
# Interpolate at shifted wavelengths (if we have a template)
# ------------------------------------------------------------------
if loc['FLAG_TEMPLATE']:
# Requires p:
# TELLU_FIT_KEEP_FRAC
# Requires loc:
# DATA
# TEMPLATE
# WAVE_IT
# Returns:
# TEMPLATE2
loc = spirouTelluric.BervCorrectTemplate(p, loc, thdr)
# debug plot
if p['DRS_PLOT'] and (p['DRS_DEBUG'] > 1):
# plot the transmission map plot
sPlt.tellu_fit_tellu_spline_plot(p, loc)
# store PC and TAPAS_ALL_SPECIES before shift
loc['PC_PRESHIFT'] = np.array(loc['PC'])
loc['TAPAS_ALL_PRESHIFT'] = np.array(loc['TAPAS_ALL_SPECIES'])
loc.set_sources(['PC_PRESHIFT', 'TAPAS_ALL_PRESHIFT'], main_name)
# ------------------------------------------------------------------
# Shift the pca components to correct frame
# ------------------------------------------------------------------
# log process
wmsg1 = 'Shifting PCA components from master wavelength grid'
wmsg2 = '\tFile = {0}'.format(os.path.basename(loc['MASTERWAVEFILE']))
WLOG(p, '', [wmsg1, wmsg2])
# shift pca components (one by one)
for comp in range(loc['NPC']):
wargs = [p, loc['PC'][:, comp], loc['MASTERWAVE'], loc['WAVE_IT']]
shift_pc = spirouTelluric.Wave2Wave(*wargs, reshape=True)
loc['PC'][:, comp] = shift_pc.reshape(wargs[1].shape)
wargs = [
p, loc['FIT_PC'][:, comp], loc['MASTERWAVE'], loc['WAVE_IT']
]
shift_fpc = spirouTelluric.Wave2Wave(*wargs, reshape=True)
loc['FIT_PC'][:, comp] = shift_fpc.reshape(wargs[1].shape)
# ------------------------------------------------------------------
# Shift the tapas spectrum to correct frame
# ------------------------------------------------------------------
# log process
wmsg1 = 'Shifting TAPAS spectrum from master wavelength grid'
wmsg2 = '\tFile = {0}'.format(os.path.basename(loc['MASTERWAVEFILE']))
WLOG(p, '', [wmsg1, wmsg2])
# shift tapas
for comp in range(len(loc['TAPAS_ALL_SPECIES'])):
wargs = [
p, loc['TAPAS_ALL_SPECIES'][comp], loc['MASTERWAVE'],
loc['WAVE_IT']
]
stapas = spirouTelluric.Wave2Wave(*wargs, reshape=True)
loc['TAPAS_ALL_SPECIES'][comp] = stapas.reshape(wargs[1].shape)
# Debug plot to test shifting
if p['DRS_PLOT'] and p['DRS_DEBUG'] > 1:
sPlt.tellu_fit_debug_shift_plot(p, loc)
# ------------------------------------------------------------------
# Calculate reconstructed absorption
# ------------------------------------------------------------------
# Requires p:
# TELLU_FIT_MIN_TRANSMISSION
# TELLU_FIT_NITER
# TELLU_LAMBDA_MIN
# TELLU_LAMBDA_MAX
# TELLU_FIT_VSINI
# TRANSMISSION_CUT
# FIT_DERIV_PC
# LOG_OPT
# Requires loc:
# FLAG_TEMPLATE
# TAPAS_ALL_SPECIES
# AMPS_ABSOL_TOTAL
# WAVE_IT
# TEMPLATE2
# FIT_PC
# NPC
# PC
# Returns loc:
# SP2
# TEMPLATE2
# RECON_ABSO
# AMPS_ABSOL_TOTAL
loc = spirouTelluric.CalcReconAbso(p, loc)
# debug plot
if p['DRS_PLOT'] > 0:
# plot the recon abso plot
sPlt.tellu_fit_recon_abso_plot(p, loc)
# ------------------------------------------------------------------
# Get molecular absorption
# ------------------------------------------------------------------
# Requires p:
# TELLU_FIT_LOG_LIMIT
# Requeres loc:
# RECON_ABSO
# TAPAS_ALL_SPECIES
# Returns loc:
# TAPAS_{molecule}
loc = spirouTelluric.CalcMolecularAbsorption(p, loc)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# get SNR for each order from header
nbo = loc['DATA'].shape[0]
snr_order = p['QC_FIT_TELLU_SNR_ORDER']
snr = spirouImage.Read1Dkey(p, thdr, p['kw_E2DS_SNR'][0], nbo)
# check that SNR is high enough
if snr[snr_order] < p['QC_FIT_TELLU_SNR_MIN']:
fmsg = 'low SNR in order {0}: ({1:.2f} < {2:.2f})'
fargs = [snr_order, snr[snr_order], p['QC_FIT_TELLU_SNR_MIN']]
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(snr[snr_order])
qc_name_str = 'SNR[{0}]'.format(snr_order)
qc_names.append(qc_name_str)
qc_logic.append('{0} < {1:.2f}'.format(qc_name_str,
p['QC_FIT_TELLU_SNR_ORDER']))
# ----------------------------------------------------------------------
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
continue
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Get components amplitudes for header
# ------------------------------------------------------------------
# get raw file name
raw_in_file = os.path.basename(p['FITSFILENAME'])
# copy original keys
hdict = spirouImage.CopyOriginalKeys(thdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
# set the input files
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBBLAZE'],
value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=loc['WAVEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=loc['WSOURCE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='file',
values=p['ARG_FILE_NAMES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# set tellu keys
npc = loc['NPC']
hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_NPC'], value=npc)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TELLU_FIT_DPC'],
value=p['FIT_DERIV_PC'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TELLU_ADD_DPC'],
value=p['ADD_DERIV_PC'])
if p['ADD_DERIV_PC']:
values = loc['AMPS_ABSOL_TOTAL'][:npc - 2]
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_TELLU_AMP_PC'],
values=values,
dim1name='amp')
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TELLU_DV_TELL1'],
value=loc['AMPS_ABSOL_TOTAL'][npc - 2])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TELLU_DV_TELL2'],
value=loc['AMPS_ABSOL_TOTAL'][npc - 1])
else:
values = loc['AMPS_ABSOL_TOTAL'][:npc]
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_TELLU_AMP_PC'],
values=values,
dim1name='PC')
# ------------------------------------------------------------------
# Write corrected spectrum to E2DS
# ------------------------------------------------------------------
# reform the E2DS
sp_out = loc['SP2'] / loc['RECON_ABSO']
sp_out = sp_out.reshape(loc['DATA'].shape)
# multiply by blaze
sp_out = sp_out * loc['NBLAZE']
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
# log progress
wmsg = 'Saving {0} to file'.format(outfilename1)
WLOG(p, '', wmsg)
# write sp_out to file
p = spirouImage.WriteImage(p, outfile1, sp_out, hdict)
# ------------------------------------------------------------------
# 1-dimension spectral S1D (uniform in wavelength)
# ------------------------------------------------------------------
# get arguments for E2DS to S1D
e2dsargs = [loc['WAVE'], sp_out, loc['BLAZE']]
# get 1D spectrum
xs1d1, ys1d1 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='wave')
# Plot the 1D spectrum
if p['DRS_PLOT'] > 0:
sPlt.ext_1d_spectrum_plot(p, xs1d1, ys1d1)
# construct file name
targs = [p, raw_in_file]
s1dfile1, tag3 = spirouConfig.Constants.TELLU_FIT_S1D_FILE1(*targs)
s1dfilename1 = os.path.basename(s1dfile1)
# add header keys
# set the version
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_EXT_TYPE'],
value=p['DPRTYPE'])
# log writing to file
wmsg = 'Saving 1D spectrum (uniform in wavelength) in {0}'
WLOG(p, '', wmsg.format(s1dfilename1))
# Write to file
columns = ['wavelength', 'flux', 'eflux']
values = [xs1d1, ys1d1, np.zeros_like(ys1d1)]
units = ['nm', None, None]
s1d1 = spirouImage.MakeTable(p, columns, values, units=units)
spirouImage.WriteTable(p, s1d1, s1dfile1, header=hdict)
# ------------------------------------------------------------------
# 1-dimension spectral S1D (uniform in velocity)
# ------------------------------------------------------------------
# get arguments for E2DS to S1D
e2dsargs = [loc['WAVE'], sp_out, loc['BLAZE']]
# get 1D spectrum
xs1d2, ys1d2 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='velocity')
# Plot the 1D spectrum
if p['DRS_PLOT'] > 0:
sPlt.ext_1d_spectrum_plot(p, xs1d2, ys1d2)
# construct file name
targs = [p, raw_in_file]
s1dfile2, tag4 = spirouConfig.Constants.TELLU_FIT_S1D_FILE2(*targs)
s1dfilename2 = os.path.basename(s1dfile2)
# add header keys
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_EXT_TYPE'],
value=p['DPRTYPE'])
# log writing to file
wmsg = 'Saving 1D spectrum (uniform in velocity) in {0}'
WLOG(p, '', wmsg.format(s1dfilename2))
# Write to file
columns = ['wavelength', 'flux', 'eflux']
values = [xs1d2, ys1d2, np.zeros_like(ys1d2)]
units = ['nm', None, None]
s1d2 = spirouImage.MakeTable(p, columns, values, units=units)
spirouImage.WriteTable(p, s1d2, s1dfile2, header=hdict)
# ------------------------------------------------------------------
# Write reconstructed absorption to E2DS
# ------------------------------------------------------------------
# set up empty storage
recon_abso2 = np.zeros_like(loc['DATA'])
# get dimensions of data
ydim, xdim = loc['DATA'].shape
# loop around orders
for order_num in range(ydim):
# get start and end points
start, end = xdim * order_num, xdim * order_num + xdim
# save to storage
recon_abso2[order_num, :] = loc['RECON_ABSO'][start:end]
# add molecular absorption to file
for it, molecule in enumerate(p['TELLU_ABSORBERS'][1:]):
# get molecule keyword store and key
molkey = '{0}_{1}'.format(p['KW_TELLU_ABSO'][0], molecule.upper())
molkws = [molkey, 0, 'Absorption in {0}'.format(molecule.upper())]
# load into hdict
hdict = spirouImage.AddKey(p, hdict, molkws, value=loc[molkey])
# add water col
if molecule == 'h2o':
loc['WATERCOL'] = loc[molkey]
# set source
loc.set_source('WATERCOL', main_name)
# add the tau keys
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TAU_H2O'],
value=loc['TAU_H2O'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TAU_REST'],
value=loc['TAU_REST'])
# log progress
wmsg = 'Saving {0} to file'.format(outfilename2)
WLOG(p, '', wmsg)
# write recon_abso to file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
p = spirouImage.WriteImage(p, outfile2, recon_abso2, hdict)
# ------------------------------------------------------------------
# Update the Telluric database
# ------------------------------------------------------------------
if p['QC']:
# add TELLU_OBJ to telluric database
oparams = dict(objname=loc['OBJNAME'],
berv=loc['BERV'],
airmass=loc['AIRMASS'],
watercol=loc['WATERCOL'])
spirouDB.UpdateDatabaseTellObj(p, outfilename1, **oparams)
# copy file to database
spirouDB.PutTelluFile(p, outfile1)
# add TELLU_RECON to telluric database
# add TELLU_OBJ to telluric database
oparams = dict(objname=loc['OBJNAME'],
berv=loc['BERV'],
airmass=loc['AIRMASS'],
watercol=loc['WATERCOL'])
spirouDB.UpdateDatabaseTellRecon(p, outfilename2, **oparams)
# copy file to database
spirouDB.PutTelluFile(p, outfile2)
# ----------------------------------------------------------------------
# End plotting
# ----------------------------------------------------------------------
# debug plot
if p['DRS_PLOT'] > 0:
# end interactive session
sPlt.end_interactive_session(p)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())