def main(night_name=None, oldfile=None, newfile=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# deal with arguments being None (i.e. get from sys.argv)
pos = [0, 1]
fmt = [str, str]
names = ['oldfile', 'newfile']
call = [oldfile, newfile]
# now get custom arguments
customargs = spirouStartup.GetCustomFromRuntime(p,
pos,
fmt,
names,
calls=call)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsfile='newfile')
# ----------------------------------------------------------------------
# Get files
# ----------------------------------------------------------------------
# check that path exists
emsg = '{0} file = {1} does not exist'
if not os.path.exists(p['OLDFILE']):
WLOG(p, 'error', emsg.format('old', p['OLDFILE']))
# check that paths exists
if not os.path.exists(p['NEWFILE']):
WLOG(p, 'error', emsg.format('new', p['NEWFILE']))
# load files
data1, hdr1, _, _ = spirouImage.ReadImage(p, filename=oldfile)
data2, hdr2, _, _ = spirouImage.ReadImage(p, filename=newfile)
# ----------------------------------------------------------------------
# Do difference image
# ----------------------------------------------------------------------
diff_image(p, data1, data2, 'old image', 'new image', scale=(1, 99))
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
wmsg = 'Recipe {0} has been successfully completed'
WLOG(p, 'info', wmsg.format(p['PROGRAM']))
# return a copy of locally defined variables in the memory
return dict(locals())
def get_full_flat(p):
# get parameters from p
filename = p['BADPIX_FULL_FLAT']
threshold = p['BADPIX_FULL_THRESHOLD']
# construct filepath
package = spirouConfig.Constants.PACKAGE()
relfolder = spirouConfig.Constants.BADPIX_DIR()
datadir = spirouConfig.GetAbsFolderPath(package, relfolder)
absfilename = os.path.join(datadir, filename)
# check that filepath exists
if not os.path.exists(absfilename):
emsg = 'badpix full flat ({0}) not found in {1}. Please correct.'
WLOG(p, 'error', emsg.format(filename, datadir))
# read image
mdata, _, _, _ = spirouImage.ReadImage(p, absfilename, kind='FULLFLAT')
# return image
return mdata
def find_hotpix_offset(p, filename, yhot, xhot):
# get the med_size
med_size = p['PP_CORRUPT_MED_SIZE']
# get data
try:
data, hdr, _, _ = spirouImage.ReadImage(p,
filename,
kind='None',
log=False)
except SystemExit:
return np.nan
if p['KW_EXPTIME'][0] in hdr:
exptime = hdr[p['KW_EXPTIME'][0]]
else:
exptime = np.nan
# get median hot pixel box
med_hotpix = np.zeros([2 * med_size + 1, 2 * med_size + 1])
# loop around x
for dx in range(-med_size, med_size + 1):
# loop around y
for dy in range(-med_size, med_size + 1):
# define position in median box
posx = dx + med_size
posy = dy + med_size
# get the hot pixel values at position in median box
data_hot = np.array(data[yhot + dx, xhot + dy])
# median the data_hot for this box position
med_hotpix[posx, posy] = np.nanmedian(data_hot)
# work out an rms
res = med_hotpix - np.nanmedian(med_hotpix)
rms = np.nanmedian(np.abs(res))
snr_hotpix = res[med_size, med_size] / rms
# return rms
return snr_hotpix, exptime
def main(night_name=None, fpfile=None, hcfiles=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
if hcfiles is None or fpfile is None:
names, types = ['fpfile', 'hcfiles'], [str, str]
customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1],
types,
names,
last_multi=True)
else:
customargs = dict(hcfiles=hcfiles, fpfile=fpfile)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsdir='reduced',
mainfitsfile='hcfiles')
# ----------------------------------------------------------------------
# Construct reference filename and get fiber type
# ----------------------------------------------------------------------
p, fpfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['FPFILE'])
fiber1 = str(p['FIBER'])
p, hcfilenames = spirouStartup.MultiFileSetup(p, files=p['HCFILES'])
fiber2 = str(p['FIBER'])
# set the hcfilename to the first hcfilenames
hcfitsfilename = hcfilenames[0]
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# ----------------------------------------------------------------------
# Have to check that the fibers match
# ----------------------------------------------------------------------
if fiber1 == fiber2:
p['FIBER'] = fiber1
fsource = __NAME__ + '/main() & spirouStartup.GetFiberType()'
p.set_source('FIBER', fsource)
else:
emsg = 'Fiber not matching for {0} and {1}, should be the same'
eargs = [hcfitsfilename, fpfitsfilename]
WLOG(p, 'error', emsg.format(*eargs))
# set the fiber type
p['FIB_TYP'] = [p['FIBER']]
p.set_source('FIB_TYP', __NAME__ + '/main()')
# set find line mode
find_lines_mode = p['HC_FIND_LINES_MODE']
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read and combine all HC files except the first (fpfitsfilename)
rargs = [p, 'add', hcfitsfilename, hcfilenames[1:]]
p, hcdata, hchdr = spirouImage.ReadImageAndCombine(*rargs)
# read first file (fpfitsfilename)
fpdata, fphdr, _, _ = spirouImage.ReadImage(p, fpfitsfilename)
# add data and hdr to loc
loc = ParamDict()
loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr
loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr
# set the source
sources = ['HCDATA', 'HCHDR']
loc.set_sources(sources, 'spirouImage.ReadImageAndCombine()')
sources = ['FPDATA', 'FPHDR']
loc.set_sources(sources, 'spirouImage.ReadImage()')
# ----------------------------------------------------------------------
# Get basic image properties for reference file
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hchdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hchdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hchdr, name='gain')
# get acquisition time
p = spirouImage.GetAcqTime(p, hchdr, name='acqtime', kind='julian')
bjdref = p['ACQTIME']
# set sigdet and conad keywords (sigdet is changed later)
p['KW_CCD_SIGDET'][1] = p['SIGDET']
p['KW_CCD_CONAD'][1] = p['GAIN']
# get lamp parameters
p = spirouTHORCA.GetLampParams(p, hchdr)
# ----------------------------------------------------------------------
# Obtain the flat
# ----------------------------------------------------------------------
# get the flat
p, loc = spirouFLAT.GetFlat(p, loc, hchdr)
# ----------------------------------------------------------------------
# Read blaze
# ----------------------------------------------------------------------
# get tilts
p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hchdr)
loc.set_source('BLAZE', __NAME__ + '/main() + /spirouImage.ReadBlazeFile')
# correct the data with the flat
# TODO: Should this be used?
# log
# WLOG(p, '', 'Applying flat correction')
# loc['HCDATA'] = loc['HCDATA']/loc['FLAT']
# loc['FPDATA'] = loc['FPDATA']/loc['FLAT']
# ----------------------------------------------------------------------
# Start plotting session
# ----------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# start interactive plot
sPlt.start_interactive_session(p)
# ----------------------------------------------------------------------
# loop around fiber type
# ----------------------------------------------------------------------
for fiber in p['FIB_TYP']:
# set fiber type for inside loop
p['FIBER'] = fiber
# ------------------------------------------------------------------
# Instrumental drift computation (if previous solution exists)
# ------------------------------------------------------------------
# get key
keydb = 'HCREF_{0}'.format(p['FIBER'])
# check for key in calibDB
if keydb in p['CALIBDB'].keys():
# log process
wmsg = ('Doing Instrumental drift computation from previous '
'calibration')
WLOG(p, '', wmsg)
# calculate instrument drift
loc = spirouTHORCA.CalcInstrumentDrift(p, loc)
# ------------------------------------------------------------------
# Wave solution
# ------------------------------------------------------------------
# log message for loop
wmsg = 'Processing Wavelength Calibration for Fiber {0}'
WLOG(p, 'info', wmsg.format(p['FIBER']))
# ------------------------------------------------------------------
# Part 1 of cal_HC
# ------------------------------------------------------------------
p, loc = cal_HC_E2DS_spirou.part1(p, loc, mode=find_lines_mode)
# ------------------------------------------------------------------
# FP solution
# ------------------------------------------------------------------
# log message
wmsg = 'Calculating FP wave solution'
WLOG(p, '', wmsg)
# calculate FP wave solution
# spirouTHORCA.FPWaveSolution(p, loc, mode=find_lines_mode)
spirouTHORCA.FPWaveSolutionNew(p, loc)
# ------------------------------------------------------------------
# FP solution plots
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# Plot the FP extracted spectrum against wavelength solution
sPlt.wave_plot_final_fp_order(p, loc, iteration=1)
# Plot the measured FP cavity width offset against line number
sPlt.wave_local_width_offset_plot(p, loc)
# Plot the FP line wavelength residuals
sPlt.wave_fp_wavelength_residuals(p, loc)
# ------------------------------------------------------------------
# Part 2 of cal_HC
# ------------------------------------------------------------------
# set params for part2
p['QC_RMS_LITTROW_MAX'] = p['QC_WAVE_RMS_LITTROW_MAX']
p['QC_DEV_LITTROW_MAX'] = p['QC_WAVE_DEV_LITTROW_MAX']
p['IC_HC_N_ORD_START_2'] = min(p['IC_HC_N_ORD_START_2'],
p['IC_FP_N_ORD_START'])
p['IC_HC_N_ORD_FINAL_2'] = max(p['IC_HC_N_ORD_FINAL_2'],
p['IC_FP_N_ORD_FINAL'])
# run part 2
# p, loc = part2test(p, loc)
p, loc = cal_HC_E2DS_spirou.part2(p, loc)
# ----------------------------------------------------------------------
# End plotting session
# ----------------------------------------------------------------------
# end interactive session
sPlt.end_interactive_session(p)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
customargs = spirouStartup.GetCustomFromRuntime(p, [0], [str], ['reffile'],
[True], [reffile])
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsfile='reffile',
mainfitsdir='reduced')
p['FIBER'] = 'AB'
# load the calibDB
p = spirouStartup.LoadCalibDB(p)
# load ref spectrum
e2ds, hdr, nx, ny = spirouImage.ReadImage(p)
# get blaze
blaze = spirouImage.ReadBlazeFile(p)
# get wave image
_, wave, _ = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True)
# get files
files = os.listdir('.')
fluxes = []
times = []
fig1, frame1 = plt.subplots(ncols=1, nrows=1)
# loop around files and sum flux
for filename in files:
def main(night_name=None, ufiles=None):
"""
cal_preprocess_spirou.py main function, if night_name and files are
None uses arguments from run time i.e.:
cal_preprocess_spirou.py [night_directory] [fitsfilename]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param ufiles: string, list or None, the list of files to process
Note can include wildcard i.e. "*.fits"
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# need custom args (to accept full path or wild card
if ufiles is None:
names, types = ['ufiles'], [str]
customargs = spirouStartup.GetCustomFromRuntime(p, [0], types, names,
last_multi=True)
else:
customargs = dict(ufiles=ufiles)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p, night_name, customargs=customargs,
mainfitsdir='raw')
# ----------------------------------------------------------------------
# Get hot pixels for corruption check
# ----------------------------------------------------------------------
hotpixels = spirouImage.PPGetHotPixels(p)
# ----------------------------------------------------------------------
# Process files (including wildcards)
# ----------------------------------------------------------------------
# get raw folder (assume all files are in the root directory)
rawdir = spirouConfig.Constants.RAW_DIR(p)
try:
ufiles = spirouFile.Paths(p['UFILES'], root=rawdir).abs_paths
except PathException as e:
WLOG(p, 'error', e)
# log how many files were found
wmsg = '{0} files found'
WLOG(p, '', wmsg.format(len(ufiles)))
# storage for output files
p['OUTPUT_NAMES'] = []
p.set_source('OUTPUT_NAMES', __NAME__ + '.main()')
# loop around files
for u_it, ufile in enumerate(ufiles):
# log the file process
wmsg = 'Processing file {0} ({1} of {2})'
WLOG(p, '', spirouStartup.spirouStartup.HEADER)
bfilename = os.path.basename(ufile)
WLOG(p, 'info', wmsg.format(bfilename, u_it+1, len(ufiles)))
WLOG(p, '', spirouStartup.spirouStartup.HEADER)
# ------------------------------------------------------------------
# Check that we can process file
# ------------------------------------------------------------------
# check if ufile exists
if not os.path.exists(ufile):
wmsg = 'File {0} does not exist... skipping'
WLOG(p, 'warning', wmsg.format(ufile))
continue
# skip processed files
elif p['PROCESSED_SUFFIX'] in bfilename:
wmsg = 'File {0} has been processed... skipping'
WLOG(p, 'warning', wmsg.format(ufile))
continue
# skip non-fits files
elif '.fits' not in bfilename:
wmsg = 'File {0} not a fits file... skipping'
WLOG(p, 'warning', wmsg.format(ufile))
continue
# skip index file
elif bfilename == spirouConfig.Constants.INDEX_OUTPUT_FILENAME():
wmsg = 'Skipping index fits file'
WLOG(p, 'warning', wmsg.format(ufile))
continue
# ------------------------------------------------------------------
# Read image file
# ------------------------------------------------------------------
# read the image data
rout = spirouImage.ReadImage(p, filename=ufile)
image, hdr, nx, ny = rout
# ------------------------------------------------------------------
# Identify file (and update filename, header and comments)
# ------------------------------------------------------------------
ufile, hdr = spirouImage.IdentifyUnProFile(p, ufile, hdr)
# ------------------------------------------------------------------
# correct image
# ------------------------------------------------------------------
# correct for the top and bottom reference pixels
WLOG(p, '', 'Correcting for top and bottom pixels')
image = spirouImage.PPCorrectTopBottom(p, image)
# correct by a median filter from the dark amplifiers
wmsg = 'Correcting by the median filter from dark amplifiers'
WLOG(p, '', wmsg)
image = spirouImage.PPMedianFilterDarkAmps(p, image)
# correct for the 1/f noise
wmsg = 'Correcting for the 1/f noise'
WLOG(p, '', wmsg)
image = spirouImage.PPMedianOneOverfNoise2(p, image)
# ------------------------------------------------------------------
# Quality control to check for corrupt files
# ------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# get pass condition
cout = spirouImage.PPTestForCorruptFile(p, image, hotpixels)
snr_hotpix, rms_list = cout
# print out SNR hotpix value
wmsg = 'Corruption check: SNR Hotpix value = {0:.5e}'
WLOG(p, '', wmsg.format(snr_hotpix))
#deal with printing corruption message
if snr_hotpix < p['PP_CORRUPT_SNR_HOTPIX']:
# add failed message to fail message list
fargs = [snr_hotpix, p['PP_CORRUPT_SNR_HOTPIX'],ufile ]
fmsg = ('File was found to be corrupted. (SNR_HOTPIX < threshold, '
'{0:.4e} < {1:.4e}). File will not be saved. '
'File = {2}'.format(*fargs))
fail_msg.append(fmsg)
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(snr_hotpix)
qc_names.append('snr_hotpix')
qc_logic.append('snr_hotpix < {0:.5e}'
''.format(p['PP_CORRUPT_SNR_HOTPIX']))
# ----------------------------------------------------------------------
if np.max(rms_list) > p['PP_CORRUPT_RMS_THRES']:
# add failed message to fail message list
fargs = [np.max(rms_list), p['PP_CORRUPT_RMS_THRES'], ufile]
fmsg = ('File was found to be corrupted. (RMS < threshold, '
'{0:.4e} > {1:.4e}). File will not be saved. '
'File = {0}'.format(*fargs))
fail_msg.append(fmsg)
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(np.max(rms_list))
qc_names.append('max(rms_list)')
qc_logic.append('max(rms_list) > {0:.4e}'
''.format(p['PP_CORRUPT_RMS_THRES']))
# ----------------------------------------------------------------------
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
WLOG(p, 'warning', '\tFile not written')
continue
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# rotate image
# ------------------------------------------------------------------
# rotation to match HARPS orientation (expected by DRS)
image = spirouImage.RotateImage(image, p['RAW_TO_PP_ROTATION'])
# ------------------------------------------------------------------
# Save rotated image
# ------------------------------------------------------------------
# construct rotated file name
outfits = spirouConfig.Constants.PP_FILE(p, bfilename)
outfitsname = os.path.basename(outfits)
# log that we are saving rotated image
WLOG(p, '', 'Saving Rotated Image in ' + outfitsname)
# add keys from original header file
hdict = spirouImage.CopyOriginalKeys(hdr)
# set the version
hdict = spirouImage.AddKey(p, hdict, p['KW_PPVERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
# set the inputs
hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'],
dim1name='file',
values=[os.path.basename(ufile)])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddKey1DList(p, hdict, p['KW_DRS_QC_NAME'],
values=qc_names)
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# set the DRS type (for file indexing)
p['DRS_TYPE'] = 'RAW'
p.set_source('DRS_TYPE', __NAME__ + '.main()')
# write to file
p = spirouImage.WriteImage(p, outfits, image, hdict)
# index this file
p = spirouStartup.End(p, outputs='pp', end=False)
# ------------------------------------------------------------------
# append to output storage in p
# ------------------------------------------------------------------
p['OUTPUT_NAMES'].append(outfitsname)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p, outputs=None)
# return a copy of locally defined variables in the memory
return dict(locals())
def sort_polar_files(p, polardict):
"""
Function to sort input data for polarimetry.
:param p: parameter dictionary, ParamDict containing constants
Must contain at least:
LOG_OPT: string, option for logging
REDUCED_DIR: string, directory path where reduced data are stored
ARG_FILE_NAMES: list, list of input filenames
KW_CMMTSEQ: string, FITS keyword where to find polarimetry
information
:param polardict: dictionary, ParamDict containing information on the
input data
:return polardict: dictionary, ParamDict containing information on the
input data
adds an entry for each filename, each entry is a
dictionary containing:
- basename, hdr, exposure, stokes, fiber, data
for each file
"""
func_name = __NAME__ + '.sort_polar_files()'
# get constants from p
rdir = p['REDUCED_DIR']
# set default properties
stokes, exposure, expstatus = 'UNDEF', 0, False
# loop over all input files
for filename in p['ARG_FILE_NAMES']:
# get full path of input file
filepath = os.path.join(rdir, filename)
# ------------------------------------------------------------------
# Check that we can process file
# check if file exists
if not os.path.exists(filepath):
wmsg = 'File {0} does not exist... skipping'
WLOG(p, 'warning', wmsg.format(filepath))
continue
# ------------------------------------------------------------------
# Read E2DS input file
data, hdr, nx, ny = spirouImage.ReadImage(p, filepath)
# ------------------------------------------------------------------
# get base file name
basename = os.path.basename(filename)
# ------------------------------------------------------------------
# try to get polarisation header key
if p['KW_CMMTSEQ'][0] in hdr and hdr[p['KW_CMMTSEQ'][0]] != "":
cmmtseq = hdr[p['KW_CMMTSEQ'][0]].split(" ")
stokes, exposure = cmmtseq[0], int(cmmtseq[2][0])
expstatus = True
else:
wmsg = 'File {0} has empty key="{1}", setting Stokes={2}'
wargs = [filename, p['KW_CMMTSEQ'][0], stokes]
# Question: stokes here will be set to the last file value?
WLOG(p, 'warning', wmsg.format(*wargs))
expstatus = False
# ------------------------------------------------------------------
# deal with fiber type
fout = deal_with_fiber(p, filename, expstatus, exposure)
fiber, expstatus, exposure, skip = fout
# deal with skip
if skip:
continue
# ------------------------------------------------------------------
# Question: Why add some basename and skip some stokes/fiber/data etc
# store data for this file
polardict[filename] = OrderedDict()
# set source
polardict.set_source(filename, func_name)
# add filename
polardict[filename]["basename"] = basename
# store header
polardict[filename]["hdr"] = hdr
# store exposure number
polardict[filename]["exposure"] = exposure
# store stokes parameter
polardict[filename]["stokes"] = stokes
# store fiber type
polardict[filename]["fiber"] = fiber
# store data
polardict[filename]["data"] = data
# set source of polar dict filename
polardict.set_source(filename, func_name)
# ------------------------------------------------------------------
# log file addition
wmsg = 'File {0}: fiber={1} Stokes={2} exposure={3}'
wargs = [filename, fiber, stokes, str(exposure)]
WLOG(p, 'info', wmsg.format(*wargs))
# return polarDict
return polardict
def main(night_name=None, files=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# get parameters from configuration files and run time arguments
customargs = spirouStartup.GetCustomFromRuntime(p, [0], [str], ['reffile'])
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsfile='reffile',
mainfitsdir='reduced')
# setup files and get fiber
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# set the fiber type
p['FIB_TYP'] = [p['FIBER']]
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
gfkwargs = dict(path=p['REDUCED_DIR'], filename=p['REFFILE'])
p['REFFILENAME'] = spirouStartup.GetFile(p, **gfkwargs)
p.set_source('REFFILENAME', __NAME__ + '/main()')
# get the fiber type
p['FIBER'] = 'AB'
e2ds, hdr, nx, ny = spirouImage.ReadImage(p)
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# get wave image
_, wave, _ = spirouImage.GetWaveSolution(p,
hdr=hdr,
return_wavemap=True,
fiber=wave_fiber)
blaze = spirouImage.ReadBlazeFile(p)
# ----------------------------------------------------------------------
# Get lamp params
# ----------------------------------------------------------------------
# get lamp parameters
p = spirouTHORCA.GetLampParams(p, hdr)
# ----------------------------------------------------------------------
# Get catalogue and fitted line list
# ----------------------------------------------------------------------
# load line file (from p['IC_LL_LINE_FILE'])
ll_line_cat, ampl_line_cat = spirouImage.ReadLineList(p)
# construct fitted lines table filename
wavelltbl = spirouConfig.Constants.WAVE_LINE_FILE(p)
WLOG(p, '', wavelltbl)
# read fitted lines
ll_ord, ll_line_fit, ampl_line_fit = np.genfromtxt(wavelltbl,
skip_header=4,
skip_footer=2,
unpack=True,
usecols=(0, 1, 3))
# ----------------------------------------------------------------------
# Plots
# ----------------------------------------------------------------------
# define line colours
col = ['magenta', 'purple']
# get order parity
ll_ord_par = np.mod(ll_ord, 2)
print(ll_ord_par)
col2 = [col[int(x)] for x in ll_ord_par]
# start interactive plot
sPlt.start_interactive_session(p)
plt.figure()
for order_num in np.arange(nx):
plt.plot(wave[order_num], e2ds[order_num])
# get heights
heights = []
for line in range(len(ll_line_cat)):
heights.append(200000 + np.max([np.min(e2ds), ampl_line_cat[line]]))
# plot ll_line_cat
plt.vlines(ll_line_cat,
0,
heights,
colors='darkgreen',
linestyles='dashed')
# get heights
heights = []
for line in range(len(ll_line_fit)):
heights.append(200000 + np.max([np.min(e2ds), ampl_line_fit[line]]))
# plot ll_line_fit
plt.vlines(ll_line_fit, 0, heights, colors=col2, linestyles='dashdot')
plt.xlabel('Wavelength [nm]')
plt.ylabel('Flux e-')
plt.title(p['REFFILENAME'])
# end interactive session
# sPlt.end_interactive_session()
# old code:
# plt.ion()
# plt.figure()
#
# for order_num in np.arange(nx):
# plt.plot(wave[order_num], e2ds[order_num])
#
# for line in range(len(ll_line_cat)):
# plt.vlines(ll_line_cat[line], 0, 200000 +
# max(np.min(e2ds), ampl_line_cat[line]),
# colors='darkgreen', linestyles='dashed')
#
# for line in range(len(ll_line_fit)):
# plt.vlines(ll_line_fit[line], 0, 200000 +
# max(np.min(e2ds), ampl_line_fit[line]),
# colors='magenta', linestyles='dashdot')
#
# plt.xlabel('Wavelength [nm]')
# plt.ylabel('Flux e-')
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p, outputs=None)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, reffile=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
customargs = spirouStartup.GetCustomFromRuntime(p, [0], [str], ['reffile'],
[True], [reffile])
p = spirouStartup.LoadArguments(p, night_name, customargs=customargs,
mainfitsfile='reffile',
mainfitsdir='reduced')
# load the calibDB
p = spirouStartup.LoadCalibDB(p)
# force plotting to 1
p['DRS_PLOT'] = 1
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
p, fpfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['REFFILE'])
# get the fiber type
fiber1 = str(p['FIBER'])
e2ds, hdr, nx, ny = spirouImage.ReadImage(p)
p, blaze = spirouImage.ReadBlazeFile(p)
# set source of wave file
wsource = __NAME__ + '/main() + /spirouImage.GetWaveSolution'
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# get wave image
_, wave, _ = spirouImage.GetWaveSolution(p, hdr=hdr, return_wavemap=True,
fiber=wave_fiber)
# ----------------------------------------------------------------------
# Get basic image properties
# ----------------------------------------------------------------------
plt.ion()
plt.figure()
for i in np.arange(nx):
plt.plot(wave[i], e2ds[i])
plt.xlabel('Wavelength [nm]')
plt.ylabel('Flux e-')
plt.title('Extracted spectra')
plt.figure()
for i in np.arange(nx):
plt.plot(wave[i], e2ds[i] / blaze[i])
plt.xlabel('Wavelength [nm]')
plt.ylabel('Relative Flux e-')
plt.title('Blaze corrected Extracted spectra')
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p, outputs=None)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(filetype='DARK_DARK'):
"""
cal_DARK_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_DARK_spirou.py [night_directory] [fitsfilename]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# set up function name
main_name = __NAME__ + '.main()'
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# now get custom arguments
pos, fmt = [0], [str]
names, call = ['filetype'], [filetype]
customargs = spirouStartup.GetCustomFromRuntime(p, pos, fmt, names,
calls=call,
require_night_name=False)
p = spirouStartup.LoadArguments(p, None, customargs=customargs,
mainfitsdir='tmp',
require_night_name=False)
# set the DRS type (for file indexing)
p['DRS_TYPE'] = 'RAW'
p.set_source('DRS_TYPE', __NAME__ + '.main()')
# -------------------------------------------------------------------------
# find all "filetype" objects
filenames = spirouImage.FindFiles(p, filetype=p['FILETYPE'],
allowedtypes=p['ALLOWED_DARK_TYPES'])
# convert filenames to a numpy array
filenames = np.array(filenames)
# -------------------------------------------------------------------------
# julian date to know which file we need to
# process together
dark_time = np.zeros(len(filenames))
dark_exp, dark_pp_version, dark_wt_temp = [], [], []
basenames, nightnames, dark_cass_temp, dark_humidity = [], [], [], []
# log progress
WLOG(p, '', 'Reading all dark file headers')
# looping through the file headers
for it in range(len(filenames)):
# get night name
night_name = os.path.dirname(filenames[it]).split(p['TMP_DIR'])[-1]
# get header
hdr = spirouImage.ReadHeader(p, filepath=filenames[it])
# add MJDATE to dark times
dark_time[it] = float(hdr[p['KW_ACQTIME'][0]])
# add other keys (for tabular output)
basenames.append(os.path.basename(filenames[it]))
nightnames.append(night_name)
dark_exp.append(float(hdr[p['KW_EXPTIME'][0]]))
dark_pp_version.append(hdr[p['KW_PPVERSION'][0]])
dark_wt_temp.append(float(hdr[p['KW_WEATHER_TOWER_TEMP'][0]]))
dark_cass_temp.append(float(hdr[p['KW_CASS_TEMP'][0]]))
dark_humidity.append(float(hdr[p['KW_HUMIDITY'][0]]))
# -------------------------------------------------------------------------
# match files by date
# -------------------------------------------------------------------------
# log progress
wmsg = 'Matching dark files by observation time (+/- {0} hrs)'
WLOG(p, '', wmsg.format(p['DARK_MASTER_MATCH_TIME']))
# get the time threshold
time_thres = p['DARK_MASTER_MATCH_TIME']
# get items grouped by time
matched_id = spirouImage.GroupFilesByTime(p, dark_time, time_thres)
# -------------------------------------------------------------------------
# get the most recent position
lastpos = np.argmax(dark_time)
# load up the most recent dark
rout = spirouImage.ReadImage(p, filenames[lastpos], log=False)
data_ref, hdr_ref, nx, ny = rout
# set the night name and update the reduced directory
p['ARG_NIGHT_NAME'] = nightnames[lastpos]
p.set_source('ARG_NIGHT_NAME', __NAME__ + '.main()')
p['REDUCED_DIR'] = spirouConfig.Constants.REDUCED_DIR(p)
p.set_source('REDUCED_DIR', __NAME__ + '.main()')
# -------------------------------------------------------------------------
# Read individual files and sum groups
# -------------------------------------------------------------------------
# log process
WLOG(p, '', 'Reading Dark files and combining groups')
# Find all unique groups
u_groups = np.unique(matched_id)
# currently number of bins == number of groups
num_bins = len(u_groups)
# storage of dark cube
dark_cube = np.zeros([num_bins, ny, nx])
bin_cube = np.zeros(num_bins)
# loop through groups
for g_it, group_num in enumerate(u_groups):
# log progress
WLOG(p, '', '\tGroup {0} of {1}'.format(g_it + 1, len(u_groups)))
# find all files for this group
dark_ids = filenames[matched_id == group_num]
# load this groups files into a cube
cube = []
for filename in dark_ids:
# read data
data_it, _, _, _ = spirouImage.ReadImage(p, filename, log=False)
# add to cube
cube.append(data_it)
# median dark cube
groupdark = np.nanmedian(cube, axis=0)
# sum within each bin
dark_cube[g_it % num_bins] += groupdark
# record the number of cubes that are going into this bin
bin_cube[g_it % num_bins] += 1
# need to normalize if we have more than 1 cube per bin
for bin_it in range(num_bins):
dark_cube[bin_it] /= bin_cube[bin_it]
# -------------------------------------------------------------------------
# we perform a median filter over a +/- "med_size" pixel box
# -------------------------------------------------------------------------
# log process
WLOG(p, '', 'Performing median filter for {0} bins'.format(num_bins))
# get med_size from p
med_size = p['DARK_MASTER_MED_SIZE']
# storage of output dark cube
dark_cube1 = np.zeros([num_bins, ny, nx])
# loop around the bins
for bin_it in range(num_bins):
# get the dark for this bin
bindark = dark_cube[bin_it]
# performing a median filter of the image with [-med_size, med_size]
# box in x and 1 pixel wide in y. Skips the pixel considered,
# so this is equivalent of a 2*med_size boxcar
tmp = []
for jt in range(-med_size, med_size + 1):
if jt != 0:
tmp.append(np.roll(bindark, [0, jt]))
# low frequency image
lf_dark = np.nanmedian(tmp, axis=0)
# high frequency image
dark_cube1[bin_it] = bindark - lf_dark
# -------------------------------------------------------------------------
# median the dark cube to create the master dark
master_dark = np.nanmedian(dark_cube1, axis=0)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# add to qc header lists
qc_values.append('None')
qc_names.append('None')
qc_logic.append('None')
qc_pass.append(1)
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ----------------------------------------------------------------------
# Save master dark to file
# ----------------------------------------------------------------------
# set reference filename
reffile = filenames[lastpos]
# construct folder and filename
darkmasterfits, tag = spirouConfig.Constants.DARK_FILE_MASTER(p, reffile)
darkmasterfitsname = os.path.basename(darkmasterfits)
# log writing of file
WLOG(p, '', 'Saving master dark to {0}'.format(darkmasterfitsname))
# construct big dark table
colnames = ['FILENAME', 'NIGHT', 'MJDATE', 'EXPTIME', 'WEATHER_TEMP',
'CASS_TEMP', 'RELHUMID', 'PVERSION', 'GROUPID']
values = [basenames, nightnames, dark_time, dark_exp, dark_wt_temp,
dark_cass_temp, dark_humidity, dark_pp_version, matched_id]
darktable = spirouImage.MakeTable(p, colnames, values)
# add keys from original header file
hdict = spirouImage.CopyOriginalKeys(hdr_ref)
# define new keys to add
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# write master dark + dark table to file
p = spirouImage.WriteImageTable(p, darkmasterfits, image=master_dark,
table=darktable, hdict=hdict)
# ----------------------------------------------------------------------
# Move to calibDB and update calibDB
# ----------------------------------------------------------------------
if p['QC']:
# set dark master key
keydb = 'DARKM'
# copy dark fits file to the calibDB folder
spirouDB.PutCalibFile(p, darkmasterfits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, darkmasterfitsname, hdr_ref)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, hcfile=None, fpfiles=None):
"""
cal_SLIT_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_SLIT_spirou.py [night_directory] [files]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
if hcfile is None or fpfiles is None:
names, types = ['hcfile', 'fpfiles'], [str, str]
customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1],
types,
names,
last_multi=True)
else:
customargs = dict(hcfile=hcfile, fpfile=fpfiles)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsfile='fpfiles')
# ----------------------------------------------------------------------
# Construct reference filename and get fiber type
# ----------------------------------------------------------------------
p, hcfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['HCFILE'])
p, fpfitsfiles = spirouStartup.MultiFileSetup(p, files=p['FPFILES'])
# set fiber (it doesn't matter with the 2D image but we need this to get
# the lamp type for FPFILES and HCFILES, AB == C
p['FIBER'] = 'AB'
p['FIB_TYP'] = [p['FIBER']]
fsource = __NAME__ + '/main()'
p.set_sources(['FIBER', 'FIB_TYP'], fsource)
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# add a force plot off
p['PLOT_PER_ORDER'] = PLOT_PER_ORDER
p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()')
# ----------------------------------------------------------------------
# Read FP and HC files
# ----------------------------------------------------------------------
# read input fp and hc data
rkwargs = dict(filename=fpfitsfiles[0],
filenames=fpfitsfiles[1:],
framemath='add')
p, fpdata, fphdr = spirouImage.ReadImageAndCombine(p, **rkwargs)
hcdata, hchdr, _, _ = spirouImage.ReadImage(p, hcfitsfilename)
# add data and hdr to loc
loc = ParamDict()
loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr
loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr
# set the source
sources = ['HCDATA', 'HCHDR']
loc.set_sources(sources, 'spirouImage.ReadImage()')
sources = ['FPDATA', 'FPHDR']
loc.set_sources(sources, 'spirouImage.ReadImage()')
# ---------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
hcdata = spirouImage.FixNonPreProcess(p, hcdata)
fpdata = spirouImage.FixNonPreProcess(p, fpdata)
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# add a force plot off
p['PLOT_PER_ORDER'] = PLOT_PER_ORDER
p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()')
# ----------------------------------------------------------------------
# Get basic image properties for reference file
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, fphdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, fphdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, fphdr, name='gain')
# get lamp parameters
p = spirouTHORCA.GetLampParams(p, hchdr)
# get FP_FP DPRTYPE
p = spirouImage.ReadParam(p, fphdr, 'KW_DPRTYPE', 'DPRTYPE', dtype=str)
# ----------------------------------------------------------------------
# Correction of reference FP
# ----------------------------------------------------------------------
# set the number of frames
p['NBFRAMES'] = len(fpfitsfiles)
p.set_source('NBFRAMES', __NAME__ + '.main()')
# Correction of DARK
p, fpdatac = spirouImage.CorrectForDark(p, fpdata, fphdr)
# Resize hc data
# rotate the image and convert from ADU/s to e-
fpdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, fpdatac), p=p)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'],
xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'],
yhigh=p['IC_CCDY_HIGH'],
getshape=False)
fpdata1 = spirouImage.ResizeImage(p, fpdata, **bkwargs)
# log change in data size
WLOG(p, '',
('FPref Image format changed to {0}x{1}').format(*fpdata1.shape))
# Correct for the BADPIX mask (set all bad pixels to zero)
bargs = [p, fpdata1, fphdr]
p, fpdata1 = spirouImage.CorrectForBadPix(*bargs)
p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True)
# log progress
WLOG(p, '', 'Cleaning FPref hot pixels')
# correct hot pixels
fpdata1 = spirouEXTOR.CleanHotpix(fpdata1, badpixmask)
# add to loc
loc['FPDATA1'] = fpdata1
loc.set_source('FPDATA1', __NAME__ + '.main()')
# Log the number of dead pixels
# get the number of bad pixels
with warnings.catch_warnings(record=True) as _:
n_bad_pix = np.nansum(fpdata1 <= 0)
n_bad_pix_frac = n_bad_pix * 100 / np.product(fpdata1.shape)
# Log number
wmsg = 'Nb FPref dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# ----------------------------------------------------------------------
# Correction of HC
# ----------------------------------------------------------------------
# set the number of frames
p['NBFRAMES'] = 1
p.set_source('NBFRAMES', __NAME__ + '.main()')
# Correction of DARK
p, hcdatac = spirouImage.CorrectForDark(p, hcdata, hchdr)
# Resize hc data
# rotate the image and convert from ADU/s to e-
hcdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, hcdatac), p=p)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'],
xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'],
yhigh=p['IC_CCDY_HIGH'],
getshape=False)
hcdata1 = spirouImage.ResizeImage(p, hcdata, **bkwargs)
# log change in data size
WLOG(p, '', ('HC Image format changed to {0}x{1}').format(*hcdata1.shape))
# Correct for the BADPIX mask (set all bad pixels to zero)
bargs = [p, hcdata1, hchdr]
p, hcdata1 = spirouImage.CorrectForBadPix(*bargs)
p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True)
# log progress
WLOG(p, '', 'Cleaning HC hot pixels')
# correct hot pixels
hcdata1 = spirouEXTOR.CleanHotpix(hcdata1, badpixmask)
# add to loc
loc['HCDATA1'] = hcdata1
loc.set_source('HCDATA1', __NAME__ + '.main()')
# Log the number of dead pixels
# get the number of bad pixels
with warnings.catch_warnings(record=True) as _:
n_bad_pix = np.nansum(hcdata1 <= 0)
n_bad_pix_frac = n_bad_pix * 100 / np.product(hcdata1.shape)
# Log number
wmsg = 'Nb HC dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# -------------------------------------------------------------------------
# get all FP_FP files
# -------------------------------------------------------------------------
fpfilenames = spirouImage.FindFiles(p,
filetype=p['DPRTYPE'],
allowedtypes=p['ALLOWED_FP_TYPES'])
# convert filenames to a numpy array
fpfilenames = np.array(fpfilenames)
# julian date to know which file we need to
# process together
fp_time = np.zeros(len(fpfilenames))
basenames, fp_exp, fp_pp_version, nightnames = [], [], [], []
# log progress
WLOG(p, '', 'Reading all fp file headers')
# looping through the file headers
for it in range(len(fpfilenames)):
# log progress
wmsg = '\tReading file {0} / {1}'
WLOG(p, 'info', wmsg.format(it + 1, len(fpfilenames)))
# get fp filename
fpfilename = fpfilenames[it]
# get night name
night_name = os.path.dirname(fpfilenames[it]).split(p['TMP_DIR'])[-1]
# read data
data_it, hdr_it, _, _ = spirouImage.ReadImage(p, fpfilename)
# get header
hdr = spirouImage.ReadHeader(p, filepath=fpfilenames[it])
# add MJDATE to dark times
fp_time[it] = float(hdr[p['KW_ACQTIME'][0]])
# add other keys (for tabular output)
basenames.append(os.path.basename(fpfilenames[it]))
nightnames.append(night_name)
fp_exp.append(float(hdr[p['KW_EXPTIME'][0]]))
fp_pp_version.append(hdr[p['KW_PPVERSION'][0]])
# -------------------------------------------------------------------------
# match files by date
# -------------------------------------------------------------------------
# log progress
wmsg = 'Matching FP files by observation time (+/- {0} hrs)'
WLOG(p, '', wmsg.format(p['DARK_MASTER_MATCH_TIME']))
# get the time threshold
time_thres = p['FP_MASTER_MATCH_TIME']
# get items grouped by time
matched_id = spirouImage.GroupFilesByTime(p, fp_time, time_thres)
# -------------------------------------------------------------------------
# construct the master fp file (+ correct for dark/badpix)
# -------------------------------------------------------------------------
cargs = [fpdata1, fpfilenames, matched_id]
fpcube, transforms = spirouImage.ConstructMasterFP(p, *cargs)
# log process
wmsg1 = 'Master FP construction complete.'
wmsg2 = '\tAdding {0} group images to form FP master image'
WLOG(p, 'info', [wmsg1, wmsg2.format(len(fpcube))])
# sum the cube to make fp data
masterfp = np.sum(fpcube, axis=0)
# add to loc
loc['MASTERFP'] = masterfp
loc.set_source('MASTERFP', __NAME__ + '.main()')
# ------------------------------------------------------------------
# Get localisation coefficients
# ------------------------------------------------------------------
# original there is a loop but it is not used --> removed
p = spirouImage.FiberParams(p, p['FIBER'], merge=True)
# get localisation fit coefficients
p, loc = spirouLOCOR.GetCoeffs(p, fphdr, loc)
# ------------------------------------------------------------------
# Get master wave solution map
# ------------------------------------------------------------------
# get master wave map
masterwavefile = spirouDB.GetDatabaseMasterWave(p)
# log process
wmsg1 = 'Getting master wavelength grid'
wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile))
WLOG(p, '', [wmsg1, wmsg2])
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# read master wave map
wout = spirouImage.GetWaveSolution(p,
filename=masterwavefile,
return_wavemap=True,
quiet=True,
return_header=True,
fiber=wave_fiber)
loc['MASTERWAVEP'], loc['MASTERWAVE'] = wout[:2]
loc['MASTERWAVEHDR'], loc['WSOURCE'] = wout[2:]
# set sources
wsource = ['MASTERWAVEP', 'MASTERWAVE', 'MASTERWAVEHDR']
loc.set_sources(wsource, 'spirouImage.GetWaveSolution()')
# ----------------------------------------------------------------------
# Read UNe solution
# ----------------------------------------------------------------------
wave_u_ne, amp_u_ne = spirouImage.ReadLineList(p)
loc['LL_LINE'], loc['AMPL_LINE'] = wave_u_ne, amp_u_ne
source = __NAME__ + '.main() + spirouImage.ReadLineList()'
loc.set_sources(['LL_LINE', 'AMPL_LINE'], source)
# ----------------------------------------------------------------------
# Read cavity length file
# ----------------------------------------------------------------------
loc['CAVITY_LEN_COEFFS'] = spirouImage.ReadCavityLength(p)
source = __NAME__ + '.main() + spirouImage.ReadCavityLength()'
loc.set_source('CAVITY_LEN_COEFFS', source)
# ----------------------------------------------------------------------
# Calculate shape map
# ----------------------------------------------------------------------
# calculate dx map
loc = spirouImage.GetXShapeMap(p, loc)
# if dx map is None we shouldn't continue
if loc['DXMAP'] is None:
fargs = [
loc['MAXDXMAPINFO'][0], loc['MAXDXMAPINFO'][1], loc['MAXDXMAPSTD'],
p['SHAPE_QC_DXMAP_STD']
]
fmsg = ('The std of the dxmap for order {0} y-pixel {1} is too large.'
' std = {2} (limit = {3})'.format(*fargs))
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(fmsg))
WLOG(p, 'warning', 'Cannot continue. Exiting.')
# End Message
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
# calculate dymap
loc = spirouImage.GetYShapeMap(p, loc, fphdr)
# ------------------------------------------------------------------
# Need to straighten the dxmap
# ------------------------------------------------------------------
# copy it first
loc['DXMAP0'] = np.array(loc['DXMAP'])
# straighten it
loc['DXMAP'] = spirouImage.EATransform(loc['DXMAP'], dymap=loc['DYMAP'])
# ------------------------------------------------------------------
# Need to straighten the hc data and fp data for debug
# ------------------------------------------------------------------
# log progress
WLOG(p, '', 'Shape finding complete. Applying transforms.')
# apply very last update of the debananafication
tkwargs = dict(dxmap=loc['DXMAP'], dymap=loc['DYMAP'])
loc['HCDATA2'] = spirouImage.EATransform(loc['HCDATA1'], **tkwargs)
loc['FPDATA2'] = spirouImage.EATransform(loc['FPDATA1'], **tkwargs)
loc.set_sources(['HCDATA2', 'FPDATA2'], __NAME__ + '.main()')
# ------------------------------------------------------------------
# Plotting
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# plots setup: start interactive plot
sPlt.start_interactive_session(p)
# plot the shape process for one order
sPlt.slit_shape_angle_plot(p, loc)
# end interactive section
sPlt.end_interactive_session(p)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# TODO: Decide on some quality control criteria?
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# add to qc header lists
qc_values.append(loc['MAXDXMAPSTD'])
qc_names.append('DXMAP STD')
qc_logic.append('DXMAP STD < {0}'.format(p['SHAPE_QC_DXMAP_STD']))
qc_pass.append(1)
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Writing FP big table
# ------------------------------------------------------------------
# construct big fp table
colnames = [
'FILENAME', 'NIGHT', 'MJDATE', 'EXPTIME', 'PVERSION', 'GROUPID',
'DXREF', 'DYREF', 'A', 'B', 'C', 'D'
]
values = [
basenames, nightnames, fp_time, fp_exp, fp_pp_version, matched_id,
transforms[:, 0], transforms[:, 1], transforms[:, 2], transforms[:, 3],
transforms[:, 4], transforms[:, 5]
]
fptable = spirouImage.MakeTable(p, colnames, values)
# ------------------------------------------------------------------
# Writing DXMAP to file
# ------------------------------------------------------------------
# get the raw tilt file name
raw_shape_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
shapexfits, tag = spirouConfig.Constants.SLIT_XSHAPE_FILE(p)
shapexfitsname = os.path.basename(shapexfits)
# Log that we are saving tilt file
wmsg = 'Saving shape x information in file: {0}'
WLOG(p, '', wmsg.format(shapexfitsname))
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(fphdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='hcfile',
values=p['HCFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE2'],
dim1name='fpfile',
values=p['FPFILES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# write tilt file to file
p = spirouImage.WriteImageTable(p,
shapexfits,
image=loc['DXMAP'],
table=fptable,
hdict=hdict)
# ------------------------------------------------------------------
# Writing DYMAP to file
# ------------------------------------------------------------------
# get the raw tilt file name
raw_shape_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
shapeyfits, tag = spirouConfig.Constants.SLIT_YSHAPE_FILE(p)
shapeyfitsname = os.path.basename(shapeyfits)
# Log that we are saving tilt file
wmsg = 'Saving shape y information in file: {0}'
WLOG(p, '', wmsg.format(shapeyfitsname))
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(fphdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='hcfile',
values=p['HCFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE2'],
dim1name='fpfile',
values=p['FPFILES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# write tilt file to file
p = spirouImage.WriteImageTable(p,
shapeyfits,
image=loc['DYMAP'],
table=fptable,
hdict=hdict)
# ------------------------------------------------------------------
# Writing Master FP to file
# ------------------------------------------------------------------
# get the raw tilt file name
raw_shape_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
fpmasterfits, tag = spirouConfig.Constants.SLIT_MASTER_FP_FILE(p)
fpmasterfitsname = os.path.basename(fpmasterfits)
# Log that we are saving tilt file
wmsg = 'Saving master FP file: {0}'
WLOG(p, '', wmsg.format(fpmasterfitsname))
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(fphdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='hcfile',
values=p['HCFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE2'],
dim1name='fpfile',
values=p['FPFILES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# write tilt file to file
p = spirouImage.WriteImageTable(p,
fpmasterfits,
image=masterfp,
table=fptable,
hdict=hdict)
# ------------------------------------------------------------------
# Writing sanity check files
# ------------------------------------------------------------------
if p['SHAPE_DEBUG_OUTPUTS']:
# log
WLOG(p, '', 'Saving debug sanity check files')
# construct file names
input_fp_file, tag1 = spirouConfig.Constants.SLIT_SHAPE_IN_FP_FILE(p)
output_fp_file, tag2 = spirouConfig.Constants.SLIT_SHAPE_OUT_FP_FILE(p)
input_hc_file, tag3 = spirouConfig.Constants.SLIT_SHAPE_IN_HC_FILE(p)
output_hc_file, tag4 = spirouConfig.Constants.SLIT_SHAPE_OUT_HC_FILE(p)
bdxmap_file, tag5 = spirouConfig.Constants.SLIT_SHAPE_BDXMAP_FILE(p)
# write input fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
p = spirouImage.WriteImage(p, input_fp_file, loc['FPDATA1'], hdict)
# write output fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
p = spirouImage.WriteImage(p, output_fp_file, loc['FPDATA2'], hdict)
# write input fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
p = spirouImage.WriteImage(p, input_hc_file, loc['HCDATA1'], hdict)
# write output fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
p = spirouImage.WriteImage(p, output_hc_file, loc['HCDATA2'], hdict)
# write overlap file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag5)
p = spirouImage.WriteImage(p, bdxmap_file, loc['DXMAP0'], hdict)
# ----------------------------------------------------------------------
# Move to calibDB and update calibDB
# ----------------------------------------------------------------------
if p['QC']:
# add shape x
keydb = 'SHAPEX'
# copy shape file to the calibDB folder
spirouDB.PutCalibFile(p, shapexfits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, shapexfitsname, fphdr)
# add shape y
keydb = 'SHAPEY'
# copy shape file to the calibDB folder
spirouDB.PutCalibFile(p, shapeyfits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, shapeyfitsname, fphdr)
# add fp master
keydb = 'FPMASTER'
# copy shape file to the calibDB folder
spirouDB.PutCalibFile(p, fpmasterfits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, fpmasterfitsname, fphdr)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, fpfile=None, hcfiles=None):
"""
cal_WAVE_E2DS.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_DARK_spirou.py [night_directory] [fpfile] [hcfiles]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param fpfile: string, or None, the FP file to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:param hcfiles: string, list or None, the list of HC files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# test files TC2
# night_name = 'AT5/AT5-12/2018-05-29_17-41-44/'
# fpfile = '2279844a_fp_fp_pp_e2dsff_AB.fits'
# hcfiles = ['2279845c_hc_pp_e2dsff_AB.fits']
# test files TC3
# night_name = 'TC3/AT5/AT5-12/2018-07-24_16-17-57/'
# fpfile = '2294108a_pp_e2dsff_AB.fits'
# hcfiles = ['2294115c_pp_e2dsff_AB.fits']
# night_name = 'TC3/AT5/AT5-12/2018-07-25_16-49-50/'
# fpfile = '2294223a_pp_e2dsff_AB.fits'
# hcfiles = ['2294230c_pp_e2dsff_AB.fits']
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
if hcfiles is None or fpfile is None:
names, types = ['fpfile', 'hcfiles'], [str, str]
customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1],
types,
names,
last_multi=True)
else:
customargs = dict(hcfiles=hcfiles, fpfile=fpfile)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsdir='reduced',
mainfitsfile='hcfiles')
# ----------------------------------------------------------------------
# Construct reference filename and get fiber type
# ----------------------------------------------------------------------
p, fpfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['FPFILE'])
fiber1 = str(p['FIBER'])
p, hcfilenames = spirouStartup.MultiFileSetup(p, files=p['HCFILES'])
fiber2 = str(p['FIBER'])
# set the hcfilename to the first hcfilenames
hcfitsfilename = hcfilenames[0]
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# ----------------------------------------------------------------------
# Have to check that the fibers match
# ----------------------------------------------------------------------
if fiber1 == fiber2:
p['FIBER'] = fiber1
fsource = __NAME__ + '/main() & spirouStartup.GetFiberType()'
p.set_source('FIBER', fsource)
else:
emsg = 'Fiber not matching for {0} and {1}, should be the same'
eargs = [hcfitsfilename, fpfitsfilename]
WLOG(p, 'error', emsg.format(*eargs))
# set the fiber type
p['FIB_TYP'] = [p['FIBER']]
p.set_source('FIB_TYP', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Read FP and HC files
# ----------------------------------------------------------------------
# read and combine all HC files except the first (fpfitsfilename)
rargs = [p, 'add', hcfitsfilename, hcfilenames[1:]]
p, hcdata, hchdr = spirouImage.ReadImageAndCombine(*rargs)
# read first file (fpfitsfilename)
fpdata, fphdr, _, _ = spirouImage.ReadImage(p, fpfitsfilename)
# TODO: ------------------------------------------------------------
# TODO remove to test NaNs
# TODO: ------------------------------------------------------------
# hcmask = np.isfinite(hcdata)
# fpmask = np.isfinite(fpdata)
# hcdata[~hcmask] = 0.0
# fpdata[~fpmask] = 0.0
# TODO: ------------------------------------------------------------
# add data and hdr to loc
loc = ParamDict()
loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr
loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr
# set the source
sources = ['HCDATA', 'HCHDR']
loc.set_sources(sources, 'spirouImage.ReadImageAndCombine()')
sources = ['FPDATA', 'FPHDR']
loc.set_sources(sources, 'spirouImage.ReadImage()')
# ----------------------------------------------------------------------
# Get basic image properties for reference file
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hchdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hchdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hchdr, name='gain')
# get acquisition time
p = spirouImage.GetAcqTime(p, hchdr, name='acqtime', kind='julian')
bjdref = p['ACQTIME']
# set sigdet and conad keywords (sigdet is changed later)
p['KW_CCD_SIGDET'][1] = p['SIGDET']
p['KW_CCD_CONAD'][1] = p['GAIN']
# get lamp parameters
p = spirouTHORCA.GetLampParams(p, hchdr)
# get number of orders
# we always get fibre A number because AB is doubled in constants file
loc['NBO'] = p['QC_LOC_NBO_FPALL']['A']
loc.set_source('NBO', __NAME__ + '.main()')
# get number of pixels in x from hcdata size
loc['NBPIX'] = loc['HCDATA'].shape[1]
loc.set_source('NBPIX', __NAME__ + '.main()')
# ----------------------------------------------------------------------
# Read blaze
# ----------------------------------------------------------------------
# get tilts
p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hchdr)
loc.set_source('BLAZE', __NAME__ + '/main() + /spirouImage.ReadBlazeFile')
# make copy of blaze (as it's overwritten later)
loc['BLAZE2'] = np.copy(loc['BLAZE'])
# ----------------------------------------------------------------------
# Read wave solution
# ----------------------------------------------------------------------
# wavelength file; we will use the polynomial terms in its header,
# NOT the pixel values that would need to be interpolated
# set source of wave file
wsource = __NAME__ + '/main() + /spirouImage.GetWaveSolution'
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# get wave image
wout = spirouImage.GetWaveSolution(p,
hdr=hchdr,
return_wavemap=True,
return_filename=True,
fiber=wave_fiber)
loc['WAVEPARAMS'], loc['WAVE_INIT'], loc['WAVEFILE'], loc['WSOURCE'] = wout
loc.set_sources(['WAVE_INIT', 'WAVEFILE', 'WAVEPARAMS', 'WSOURCE'],
wsource)
poly_wave_sol = loc['WAVEPARAMS']
# ----------------------------------------------------------------------
# Check that wave parameters are consistent with "ic_ll_degr_fit"
# ----------------------------------------------------------------------
loc = spirouImage.CheckWaveSolConsistency(p, loc)
# ----------------------------------------------------------------------
# Read UNe solution
# ----------------------------------------------------------------------
wave_u_ne, amp_u_ne = spirouImage.ReadLineList(p)
loc['LL_LINE'], loc['AMPL_LINE'] = wave_u_ne, amp_u_ne
source = __NAME__ + '.main() + spirouImage.ReadLineList()'
loc.set_sources(['ll_line', 'ampl_line'], source)
# ----------------------------------------------------------------------
# Generate wave map from wave solution
# ----------------------------------------------------------------------
loc = spirouWAVE.generate_wave_map(p, loc)
# ----------------------------------------------------------------------
# Find Gaussian Peaks in HC spectrum
# ----------------------------------------------------------------------
loc = spirouWAVE.find_hc_gauss_peaks(p, loc)
# ----------------------------------------------------------------------
# Start plotting session
# ----------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# start interactive plot
sPlt.start_interactive_session(p)
# ----------------------------------------------------------------------
# Fit Gaussian peaks (in triplets) to
# ----------------------------------------------------------------------
loc = spirouWAVE.fit_gaussian_triplets(p, loc)
# ----------------------------------------------------------------------
# Generate Resolution map and line profiles
# ----------------------------------------------------------------------
# log progress
wmsg = 'Generating resolution map and '
# generate resolution map
loc = spirouWAVE.generate_resolution_map(p, loc)
# map line profile map
if p['DRS_PLOT'] > 0:
sPlt.wave_ea_plot_line_profiles(p, loc)
# ----------------------------------------------------------------------
# End plotting session
# ----------------------------------------------------------------------
# end interactive session
if p['DRS_PLOT'] > 0:
sPlt.end_interactive_session(p)
# ----------------------------------------------------------------------
# Set up all_lines storage
# ----------------------------------------------------------------------
# initialise up all_lines storage
all_lines_1 = []
# get parameters from p
n_ord_start = p['IC_HC_N_ORD_START_2']
n_ord_final = p['IC_HC_N_ORD_FINAL_2']
pixel_shift_inter = p['PIXEL_SHIFT_INTER']
pixel_shift_slope = p['PIXEL_SHIFT_SLOPE']
# get values from loc
xgau = np.array(loc['XGAU_T'])
dv = np.array(loc['DV_T'])
fit_per_order = np.array(loc['POLY_WAVE_SOL'])
ew = np.array(loc['EW_T'])
peak = np.array(loc['PEAK_T'])
amp_catalog = np.array(loc['AMP_CATALOG'])
wave_catalog = np.array(loc['WAVE_CATALOG'])
ord_t = np.array(loc['ORD_T'])
# loop through orders
for iord in range(n_ord_start, n_ord_final):
# keep relevant lines
# -> right order
# -> finite dv
gg = (ord_t == iord) & (np.isfinite(dv))
nlines = np.nansum(gg)
# put lines into ALL_LINES structure
# reminder:
# gparams[0] = output wavelengths
# gparams[1] = output sigma(gauss fit width)
# gparams[2] = output amplitude(gauss fit)
# gparams[3] = difference in input / output wavelength
# gparams[4] = input amplitudes
# gparams[5] = output pixel positions
# gparams[6] = output pixel sigma width (gauss fit width in pixels)
# gparams[7] = output weights for the pixel position
chebval = np.polynomial.chebyshev.chebval
# dummy array for weights
test = np.ones(np.shape(xgau[gg]), 'd') * 1e4
# get the final wavelength value for each peak in order
output_wave_1 = np.polyval(fit_per_order[iord][::-1], xgau[gg])
# output_wave_1 = chebval(xgau[gg], fit_per_order[iord])
# convert the pixel equivalent width to wavelength units
xgau_ew_ini = xgau[gg] - ew[gg] / 2
xgau_ew_fin = xgau[gg] + ew[gg] / 2
ew_ll_ini = np.polyval(fit_per_order[iord, :], xgau_ew_ini)
ew_ll_fin = np.polyval(fit_per_order[iord, :], xgau_ew_fin)
# ew_ll_ini = chebval(xgau_ew_ini, fit_per_order[iord])
# ew_ll_fin = chebval(xgau_ew_fin, fit_per_order[iord])
ew_ll = ew_ll_fin - ew_ll_ini
# put all lines in the order into array
gau_params = np.column_stack(
(output_wave_1, ew_ll, peak[gg], wave_catalog[gg] - output_wave_1,
amp_catalog[gg], xgau[gg], ew[gg], test))
# append the array for the order into a list
all_lines_1.append(gau_params)
# save dv in km/s and auxiliary order number
# res_1 = np.concatenate((res_1,2.997e5*(input_wave - output_wave_1)/
# output_wave_1))
# ord_save = np.concatenate((ord_save, test*iord))
# add to loc
loc['ALL_LINES_1'] = all_lines_1
loc['LL_PARAM_1'] = np.array(fit_per_order)
loc['LL_OUT_1'] = np.array(loc['WAVE_MAP2'])
loc.set_sources(['ALL_LINES_1', 'LL_PARAM_1'], __NAME__ + '/main()')
# For compatibility w/already defined functions, I need to save
# here all_lines_2
all_lines_2 = list(all_lines_1)
loc['ALL_LINES_2'] = all_lines_2
# loc['LL_PARAM_2'] = np.fliplr(fit_per_order)
# loc['LL_OUT_2'] = np.array(loc['WAVE_MAP2'])
# loc.set_sources(['ALL_LINES_2', 'LL_PARAM_2'], __NAME__ + '/main()')
# ------------------------------------------------------------------
# Littrow test
# ------------------------------------------------------------------
start = p['IC_LITTROW_ORDER_INIT_1']
end = p['IC_LITTROW_ORDER_FINAL_1']
# calculate echelle orders
o_orders = np.arange(start, end)
echelle_order = p['IC_HC_T_ORDER_START'] - o_orders
loc['ECHELLE_ORDERS'] = echelle_order
loc.set_source('ECHELLE_ORDERS', __NAME__ + '/main()')
# reset Littrow fit degree
p['IC_LITTROW_FIT_DEG_1'] = 7
# Do Littrow check
ckwargs = dict(ll=loc['LL_OUT_1'][start:end, :], iteration=1, log=True)
loc = spirouTHORCA.CalcLittrowSolution(p, loc, **ckwargs)
# Plot wave solution littrow check
if p['DRS_PLOT'] > 0:
# plot littrow x pixels against fitted wavelength solution
sPlt.wave_littrow_check_plot(p, loc, iteration=1)
# ------------------------------------------------------------------
# extrapolate Littrow solution
# ------------------------------------------------------------------
ekwargs = dict(ll=loc['LL_OUT_1'], iteration=1)
loc = spirouTHORCA.ExtrapolateLittrowSolution(p, loc, **ekwargs)
# ------------------------------------------------------------------
# Plot littrow solution
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# plot littrow x pixels against fitted wavelength solution
sPlt.wave_littrow_extrap_plot(p, loc, iteration=1)
# ------------------------------------------------------------------
# Incorporate FP into solution
# ------------------------------------------------------------------
# Copy LL_OUT_1 and LL_PARAM_1 into new constants (for FP integration)
loc['LITTROW_EXTRAP_SOL_1'] = np.array(loc['LL_OUT_1'])
loc['LITTROW_EXTRAP_PARAM_1'] = np.array(loc['LL_PARAM_1'])
# only use FP if switched on in constants file
if p['IC_WAVE_USE_FP']:
# ------------------------------------------------------------------
# Find FP lines
# ------------------------------------------------------------------
# print message to screen
wmsg = 'Identification of lines in reference file: {0}'
WLOG(p, '', wmsg.format(fpfile))
# ------------------------------------------------------------------
# Get the FP solution
# ------------------------------------------------------------------
loc = spirouTHORCA.FPWaveSolutionNew(p, loc)
# ------------------------------------------------------------------
# FP solution plots
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# Plot the FP extracted spectrum against wavelength solution
sPlt.wave_plot_final_fp_order(p, loc, iteration=1)
# Plot the measured FP cavity width offset against line number
sPlt.wave_local_width_offset_plot(p, loc)
# Plot the FP line wavelength residuals
sPlt.wave_fp_wavelength_residuals(p, loc)
# ------------------------------------------------------------------
# Create new wavelength solution
# ------------------------------------------------------------------
# TODO: Melissa fault - fix later
p['IC_HC_N_ORD_START_2'] = min(p['IC_HC_N_ORD_START_2'],
p['IC_FP_N_ORD_START'])
p['IC_HC_N_ORD_FINAL_2'] = max(p['IC_HC_N_ORD_FINAL_2'],
p['IC_FP_N_ORD_FINAL'])
start = p['IC_HC_N_ORD_START_2']
end = p['IC_HC_N_ORD_FINAL_2']
# recalculate echelle orders for Fit1DSolution
o_orders = np.arange(start, end)
echelle_order = p['IC_HC_T_ORDER_START'] - o_orders
loc['ECHELLE_ORDERS'] = echelle_order
loc.set_source('ECHELLE_ORDERS', __NAME__ + '/main()')
# select the orders to fit
lls = loc['LITTROW_EXTRAP_SOL_1'][start:end]
loc = spirouTHORCA.Fit1DSolution(p, loc, lls, iteration=2)
# from here, LL_OUT_2 wil be 0-47
# ------------------------------------------------------------------
# Repeat Littrow test
# ------------------------------------------------------------------
start = p['IC_LITTROW_ORDER_INIT_2']
end = p['IC_LITTROW_ORDER_FINAL_2']
# recalculate echelle orders for Littrow check
o_orders = np.arange(start, end)
echelle_order = p['IC_HC_T_ORDER_START'] - o_orders
loc['ECHELLE_ORDERS'] = echelle_order
loc.set_source('ECHELLE_ORDERS', __NAME__ + '/main()')
# Do Littrow check
ckwargs = dict(ll=loc['LL_OUT_2'][start:end, :], iteration=2, log=True)
loc = spirouTHORCA.CalcLittrowSolution(p, loc, **ckwargs)
# Plot wave solution littrow check
if p['DRS_PLOT'] > 0:
# plot littrow x pixels against fitted wavelength solution
sPlt.wave_littrow_check_plot(p, loc, iteration=2)
# ------------------------------------------------------------------
# extrapolate Littrow solution
# ------------------------------------------------------------------
ekwargs = dict(ll=loc['LL_OUT_2'], iteration=2)
loc = spirouTHORCA.ExtrapolateLittrowSolution(p, loc, **ekwargs)
# ------------------------------------------------------------------
# Plot littrow solution
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# plot littrow x pixels against fitted wavelength solution
sPlt.wave_littrow_extrap_plot(p, loc, iteration=2)
# ------------------------------------------------------------------
# Join 0-47 and 47-49 solutions
# ------------------------------------------------------------------
loc = spirouTHORCA.JoinOrders(p, loc)
# ------------------------------------------------------------------
# Plot single order, wavelength-calibrated, with found lines
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
sPlt.wave_ea_plot_single_order(p, loc)
# ----------------------------------------------------------------------
# Do correlation on FP spectra
# ----------------------------------------------------------------------
# ------------------------------------------------------------------
# Compute photon noise uncertainty for FP
# ------------------------------------------------------------------
# set up the arguments for DeltaVrms2D
dargs = [loc['FPDATA'], loc['LL_FINAL']]
dkwargs = dict(sigdet=p['IC_DRIFT_NOISE'],
size=p['IC_DRIFT_BOXSIZE'],
threshold=p['IC_DRIFT_MAXFLUX'])
# run DeltaVrms2D
dvrmsref, wmeanref = spirouRV.DeltaVrms2D(*dargs, **dkwargs)
# save to loc
loc['DVRMSREF'], loc['WMEANREF'] = dvrmsref, wmeanref
loc.set_sources(['dvrmsref', 'wmeanref'], __NAME__ + '/main()()')
# log the estimated RV uncertainty
wmsg = 'On fiber {0} estimated RV uncertainty on spectrum is {1:.3f} m/s'
WLOG(p, 'info', wmsg.format(p['FIBER'], wmeanref))
# Use CCF Mask function with drift constants
p['CCF_MASK'] = p['DRIFT_CCF_MASK']
p['TARGET_RV'] = p['DRIFT_TARGET_RV']
p['CCF_WIDTH'] = p['DRIFT_CCF_WIDTH']
p['CCF_STEP'] = p['DRIFT_CCF_STEP']
p['RVMIN'] = p['TARGET_RV'] - p['CCF_WIDTH']
p['RVMAX'] = p['TARGET_RV'] + p['CCF_WIDTH'] + p['CCF_STEP']
# get the CCF mask from file (check location of mask)
loc = spirouRV.GetCCFMask(p, loc)
# TODO Check why Blaze makes bugs in correlbin
loc['BLAZE'] = np.ones((loc['NBO'], loc['NBPIX']))
# set sources
# loc.set_sources(['flat', 'blaze'], __NAME__ + '/main()')
loc.set_source('blaze', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Do correlation on FP
# ----------------------------------------------------------------------
# calculate and fit the CCF
loc['E2DSFF'] = np.array(loc['FPDATA'])
loc.set_source('E2DSFF', __NAME__ + '/main()')
p['CCF_FIT_TYPE'] = 1
loc['BERV'] = 0.0
loc['BERV_MAX'] = 0.0
loc['BJD'] = 0.0
# run the RV coravelation function with these parameters
loc['WAVE_LL'] = np.array(loc['LL_FINAL'])
loc['PARAM_LL'] = np.array(loc['LL_PARAM_FINAL'])
loc = spirouRV.Coravelation(p, loc)
# ----------------------------------------------------------------------
# Update the Correlation stats with values using fiber C (FP) drift
# ----------------------------------------------------------------------
# get the maximum number of orders to use
nbmax = p['CCF_NUM_ORDERS_MAX']
# get the average ccf
loc['AVERAGE_CCF'] = np.nansum(loc['CCF'][:nbmax], axis=0)
# normalize the average ccf
normalized_ccf = loc['AVERAGE_CCF'] / np.nanmax(loc['AVERAGE_CCF'])
# get the fit for the normalized average ccf
ccf_res, ccf_fit = spirouRV.FitCCF(p,
loc['RV_CCF'],
normalized_ccf,
fit_type=1)
loc['CCF_RES'] = ccf_res
loc['CCF_FIT'] = ccf_fit
# get the max cpp
loc['MAXCPP'] = np.nansum(loc['CCF_MAX']) / np.nansum(
loc['PIX_PASSED_ALL'])
# get the RV value from the normalised average ccf fit center location
loc['RV'] = float(ccf_res[1])
# get the contrast (ccf fit amplitude)
loc['CONTRAST'] = np.abs(100 * ccf_res[0])
# get the FWHM value
loc['FWHM'] = ccf_res[2] * spirouCore.spirouMath.fwhm()
# set the source
keys = [
'AVERAGE_CCF', 'MAXCPP', 'RV', 'CONTRAST', 'FWHM', 'CCF_RES', 'CCF_FIT'
]
loc.set_sources(keys, __NAME__ + '/main()')
# ----------------------------------------------------------------------
# log the stats
wmsg = ('FP Correlation: C={0:.1f}[%] DRIFT={1:.5f}[km/s] '
'FWHM={2:.4f}[km/s] maxcpp={3:.1f}')
wargs = [loc['CONTRAST'], float(ccf_res[1]), loc['FWHM'], loc['MAXCPP']]
WLOG(p, 'info', wmsg.format(*wargs))
# ----------------------------------------------------------------------
# rv ccf plot
# ----------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# Plot rv vs ccf (and rv vs ccf_fit)
p['OBJNAME'] = 'FP'
sPlt.ccf_rv_ccf_plot(p, loc['RV_CCF'], normalized_ccf, ccf_fit)
# TODO : Add QC of the FP CCF
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# get parameters ffrom p
p['QC_RMS_LITTROW_MAX'] = p['QC_HC_RMS_LITTROW_MAX']
p['QC_DEV_LITTROW_MAX'] = p['QC_HC_DEV_LITTROW_MAX']
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# quality control on sigma clip (sig1 > qc_hc_wave_sigma_max
if loc['SIG1'] > p['QC_HC_WAVE_SIGMA_MAX']:
fmsg = 'Sigma too high ({0:.5f} > {1:.5f})'
fail_msg.append(fmsg.format(loc['SIG1'], p['QC_HC_WAVE_SIGMA_MAX']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(loc['SIG1'])
qc_names.append('SIG1')
qc_logic.append('SIG1 > {0:.2f}'.format(p['QC_HC_WAVE_SIGMA_MAX']))
# ----------------------------------------------------------------------
# check the difference between consecutive orders is always positive
# get the differences
wave_diff = loc['LL_FINAL'][1:] - loc['LL_FINAL'][:-1]
if np.min(wave_diff) < 0:
fmsg = 'Negative wavelength difference between orders'
fail_msg.append(fmsg)
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(np.min(wave_diff))
qc_names.append('MIN WAVE DIFF')
qc_logic.append('MIN WAVE DIFF < 0')
# ----------------------------------------------------------------------
# check for infinites and NaNs in mean residuals from fit
if ~np.isfinite(loc['X_MEAN_2']):
# add failed message to the fail message list
fmsg = 'NaN or Inf in X_MEAN_2'
fail_msg.append(fmsg)
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(loc['X_MEAN_2'])
qc_names.append('X_MEAN_2')
qc_logic.append('X_MEAN_2 not finite')
# ----------------------------------------------------------------------
# iterate through Littrow test cut values
lit_it = 2
# checks every other value
# TODO: This QC check (or set of QC checks needs re-writing it is
# TODO: nearly impossible to understand
for x_it in range(1, len(loc['X_CUT_POINTS_' + str(lit_it)]), 2):
# get x cut point
x_cut_point = loc['X_CUT_POINTS_' + str(lit_it)][x_it]
# get the sigma for this cut point
sig_littrow = loc['LITTROW_SIG_' + str(lit_it)][x_it]
# get the abs min and max dev littrow values
min_littrow = abs(loc['LITTROW_MINDEV_' + str(lit_it)][x_it])
max_littrow = abs(loc['LITTROW_MAXDEV_' + str(lit_it)][x_it])
# get the corresponding order
min_littrow_ord = loc['LITTROW_MINDEVORD_' + str(lit_it)][x_it]
max_littrow_ord = loc['LITTROW_MAXDEVORD_' + str(lit_it)][x_it]
# check if sig littrow is above maximum
rms_littrow_max = p['QC_RMS_LITTROW_MAX']
dev_littrow_max = p['QC_DEV_LITTROW_MAX']
if sig_littrow > rms_littrow_max:
fmsg = ('Littrow test (x={0}) failed (sig littrow = '
'{1:.2f} > {2:.2f})')
fargs = [x_cut_point, sig_littrow, rms_littrow_max]
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(sig_littrow)
qc_names.append('sig_littrow')
qc_logic.append('sig_littrow > {0:.2f}'.format(rms_littrow_max))
# ----------------------------------------------------------------------
# check if min/max littrow is out of bounds
if np.max([max_littrow, min_littrow]) > dev_littrow_max:
fmsg = ('Littrow test (x={0}) failed (min|max dev = '
'{1:.2f}|{2:.2f} > {3:.2f} for order {4}|{5})')
fargs = [
x_cut_point, min_littrow, max_littrow, dev_littrow_max,
min_littrow_ord, max_littrow_ord
]
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
# TODO: Should this be the QC header values?
# TODO: it does not change the outcome of QC (i.e. passed=False)
# TODO: So what is the point?
# if sig was out of bounds, recalculate
if sig_littrow > rms_littrow_max:
# conditions
check1 = min_littrow > dev_littrow_max
check2 = max_littrow > dev_littrow_max
# get the residuals
respix = loc['LITTROW_YY_' + str(lit_it)][x_it]
# check if both are out of bounds
if check1 and check2:
# remove respective orders
worst_order = (min_littrow_ord, max_littrow_ord)
respix_2 = np.delete(respix, worst_order)
redo_sigma = True
# check if min is out of bounds
elif check1:
# remove respective order
worst_order = min_littrow_ord
respix_2 = np.delete(respix, worst_order)
redo_sigma = True
# check if max is out of bounds
elif check2:
# remove respective order
worst_order = max_littrow_ord
respix_2 = np.delete(respix, max_littrow_ord)
redo_sigma = True
# else do not recalculate sigma
else:
redo_sigma, respix_2, worst_order = False, None, None
wmsg = 'No outlying orders, sig littrow not recalculated'
fail_msg.append(wmsg.format())
# if outlying order, recalculate stats
if redo_sigma:
mean = np.nansum(respix_2) / len(respix_2)
mean2 = np.nansum(respix_2**2) / len(respix_2)
rms = np.sqrt(mean2 - mean**2)
if rms > rms_littrow_max:
fmsg = ('Littrow test (x={0}) failed (sig littrow = '
'{1:.2f} > {2:.2f} removing order {3})')
fargs = [
x_cut_point, rms, rms_littrow_max, worst_order
]
fail_msg.append(fmsg.format(*fargs))
else:
wargs = [
x_cut_point, rms, rms_littrow_max, worst_order
]
wmsg = ('Littrow test (x={0}) passed (sig littrow = '
'{1:.2f} > {2:.2f} removing order {3})')
fail_msg.append(wmsg.format(*wargs))
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(np.max([max_littrow, min_littrow]))
qc_names.append('max or min littrow')
qc_logic.append('max or min littrow > {0:.2f}'
''.format(dev_littrow_max))
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# archive result in e2ds spectra
# ------------------------------------------------------------------
# get raw input file name(s)
raw_infiles1 = []
for hcfile in p['HCFILES']:
raw_infiles1.append(os.path.basename(hcfile))
raw_infile2 = os.path.basename(p['FPFILE'])
# get wave filename
wavefits, tag1 = spirouConfig.Constants.WAVE_FILE_EA_2(p)
wavefitsname = os.path.split(wavefits)[-1]
# log progress
wargs = [p['FIBER'], wavefits]
wmsg = 'Write wavelength solution for Fiber {0} in {1}'
WLOG(p, '', wmsg.format(*wargs))
# write solution to fitsfilename header
# copy original keys
hdict = spirouImage.CopyOriginalKeys(loc['HCHDR'])
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
# set the input files
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=loc['WAVEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=loc['WSOURCE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='fpfile',
values=p['FPFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE2'],
dim1name='hcfile',
values=p['HCFILES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# add wave solution date
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_TIME1'],
value=p['MAX_TIME_HUMAN'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_TIME2'],
value=p['MAX_TIME_UNIX'])
hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_CODE'], value=__NAME__)
# add number of orders
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_ORD_N'],
value=loc['LL_PARAM_FINAL'].shape[0])
# add degree of fit
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_LL_DEG'],
value=loc['LL_PARAM_FINAL'].shape[1] - 1)
# add wave solution
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_WAVE_PARAM'],
values=loc['LL_PARAM_FINAL'])
# add FP CCF drift
# target RV and width
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_TARG_RV'],
value=p['TARGET_RV'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_WIDTH'],
value=p['CCF_WIDTH'])
# the rv step
# rvstep = np.abs(loc['RV_CCF'][0] - loc['RV_CCF'][1])
# hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CDELT'], value=rvstep)
hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_STEP'], value=p['CCF_STEP'])
# add ccf stats
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_DRIFT'],
value=loc['CCF_RES'][1])
hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_FWHM'], value=loc['FWHM'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_CONTRAST'],
value=loc['CONTRAST'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_MAXCPP'],
value=loc['MAXCPP'])
hdict = spirouImage.AddKey(p, hdict, p['KW_WFP_MASK'], value=p['CCF_MASK'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_LINES'],
value=np.nansum(loc['TOT_LINE']))
# write the wave "spectrum"
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
p = spirouImage.WriteImage(p, wavefits, loc['LL_FINAL'], hdict)
# get filename for E2DS calibDB copy of FITSFILENAME
e2dscopy_filename = spirouConfig.Constants.WAVE_E2DS_COPY(p)[0]
wargs = [p['FIBER'], os.path.split(e2dscopy_filename)[-1]]
wmsg = 'Write reference E2DS spectra for Fiber {0} in {1}'
WLOG(p, '', wmsg.format(*wargs))
# make a copy of the E2DS file for the calibBD
p = spirouImage.WriteImage(p, e2dscopy_filename, loc['HCDATA'], hdict)
# only copy over if QC passed
if p['QC']:
# loop around hc files and update header with
for hcfile in p['HCFILES']:
raw_infilepath1 = os.path.join(p['ARG_FILE_DIR'], hcfile)
p = spirouImage.UpdateWaveSolution(p, loc, raw_infilepath1)
# update fp file
raw_infilepath2 = os.path.join(p['ARG_FILE_DIR'], raw_infile2)
p = spirouImage.UpdateWaveSolution(p, loc, raw_infilepath2)
# ------------------------------------------------------------------
# Save to result table
# ------------------------------------------------------------------
# calculate stats for table
final_mean = 1000 * loc['X_MEAN_2']
final_var = 1000 * loc['X_VAR_2']
num_lines = int(np.nansum(loc['X_ITER_2'][:, 2])) # loc['X_ITER_2']
err = 1000 * np.sqrt(loc['X_VAR_2'] / num_lines)
sig_littrow = 1000 * np.array(loc['LITTROW_SIG_' + str(lit_it)])
# construct filename
wavetbl = spirouConfig.Constants.WAVE_TBL_FILE_EA(p)
wavetblname = os.path.basename(wavetbl)
# construct and write table
columnnames = [
'night_name', 'file_name', 'fiber', 'mean', 'rms', 'N_lines', 'err',
'rms_L500', 'rms_L1000', 'rms_L1500', 'rms_L2000', 'rms_L2500',
'rms_L3000', 'rms_L3500'
]
columnformats = [
'{:20s}', '{:30s}', '{:3s}', '{:7.4f}', '{:6.2f}', '{:3d}', '{:6.3f}',
'{:6.2f}', '{:6.2f}', '{:6.2f}', '{:6.2f}', '{:6.2f}', '{:6.2f}',
'{:6.2f}'
]
columnvalues = [[p['ARG_NIGHT_NAME']], [p['ARG_FILE_NAMES'][0]],
[p['FIBER']], [final_mean], [final_var],
[num_lines], [err], [sig_littrow[0]], [sig_littrow[1]],
[sig_littrow[2]], [sig_littrow[3]], [sig_littrow[4]],
[sig_littrow[5]], [sig_littrow[6]]]
# make table
table = spirouImage.MakeTable(p,
columns=columnnames,
values=columnvalues,
formats=columnformats)
# merge table
wmsg = 'Global result summary saved in {0}'
WLOG(p, '', wmsg.format(wavetblname))
spirouImage.MergeTable(p, table, wavetbl, fmt='ascii.rst')
# ----------------------------------------------------------------------
# Save resolution and line profiles to file
# ----------------------------------------------------------------------
raw_infile = os.path.basename(p['FITSFILENAME'])
# get wave filename
resfits, tag3 = spirouConfig.Constants.WAVE_RES_FILE_EA(p)
resfitsname = os.path.basename(resfits)
WLOG(p, '', 'Saving wave resmap to {0}'.format(resfitsname))
# make a copy of the E2DS file for the calibBD
# set the version
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
# get res data in correct format
resdata, hdicts = spirouTHORCA.GenerateResFiles(p, loc, hdict)
# save to file
p = spirouImage.WriteImageMulti(p, resfits, resdata, hdicts=hdicts)
# ------------------------------------------------------------------
# Save line list table file
# ------------------------------------------------------------------
# construct filename
# TODO proper column values
wavelltbl = spirouConfig.Constants.WAVE_LINE_FILE_EA(p)
wavelltblname = os.path.split(wavelltbl)[-1]
# construct and write table
columnnames = ['order', 'll', 'dv', 'w', 'xi', 'xo', 'dvdx']
columnformats = [
'{:.0f}', '{:12.4f}', '{:13.5f}', '{:12.4f}', '{:12.4f}', '{:12.4f}',
'{:8.4f}'
]
columnvalues = []
# construct column values (flatten over orders)
for it in range(len(loc['X_DETAILS_2'])):
for jt in range(len(loc['X_DETAILS_2'][it][0])):
row = [
float(it), loc['X_DETAILS_2'][it][0][jt],
loc['LL_DETAILS_2'][it][0][jt], loc['X_DETAILS_2'][it][3][jt],
loc['X_DETAILS_2'][it][1][jt], loc['X_DETAILS_2'][it][2][jt],
loc['SCALE_2'][it][jt]
]
columnvalues.append(row)
# log saving
wmsg = 'List of lines used saved in {0}'
WLOG(p, '', wmsg.format(wavelltblname))
# make table
columnvalues = np.array(columnvalues).T
table = spirouImage.MakeTable(p,
columns=columnnames,
values=columnvalues,
formats=columnformats)
# write table
spirouImage.WriteTable(p, table, wavelltbl, fmt='ascii.rst')
# ------------------------------------------------------------------
# Move to calibDB and update calibDB
# ------------------------------------------------------------------
if p['QC']:
# set the wave key
keydb = 'WAVE_{0}'.format(p['FIBER'])
# copy wave file to calibDB folder
spirouDB.PutCalibFile(p, wavefits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, wavefitsname, loc['HCHDR'])
# set the hcref key
keydb = 'HCREF_{0}'.format(p['FIBER'])
# copy wave file to calibDB folder
spirouDB.PutCalibFile(p, e2dscopy_filename)
# update the master calib DB file with new key
e2dscopyfits = os.path.split(e2dscopy_filename)[-1]
spirouDB.UpdateCalibMaster(p, keydb, e2dscopyfits, loc['HCHDR'])
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return p and loc
return dict(locals())
def main(night_name=None, files=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p,
night_name,
files,
mainfitsdir='reduced')
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# set up function name
main_name = __NAME__ + '.main()'
# ----------------------------------------------------------------------
# Load first file
# ----------------------------------------------------------------------
loc = ParamDict()
rd = spirouImage.ReadImage(p, p['FITSFILENAME'])
loc['DATA'], loc['DATAHDR'], loc['XDIM'], loc['YDIM'] = rd
loc.set_sources(['DATA', 'DATAHDR', 'XDIM', 'YDIM'], main_name)
# ----------------------------------------------------------------------
# Get object name, airmass and berv
# ----------------------------------------------------------------------
# Get object name
loc['OBJNAME'] = spirouImage.GetObjName(p, loc['DATAHDR'])
# Get the airmass
loc['AIRMASS'] = spirouImage.GetAirmass(p, loc['DATAHDR'])
# Get the Barycentric correction from header
p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, loc['DATAHDR'])
# set sources
source = main_name + '+ spirouImage.ReadParams()'
loc.set_sources(['OBJNAME', 'AIRMASS'], source)
loc.set_sources(['OBJNAME', 'AIRMASS'], source)
# ----------------------------------------------------------------------
# Read wavelength solution
# ----------------------------------------------------------------------
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# used for plotting only
wout = spirouImage.GetWaveSolution(p,
image=loc['DATA'],
hdr=loc['DATAHDR'],
return_wavemap=True,
fiber=wave_fiber)
_, loc['WAVE'], _ = wout
loc.set_source('WAVE', main_name)
# ----------------------------------------------------------------------
# Get and Normalise the blaze
# ----------------------------------------------------------------------
p, loc = spirouTelluric.GetNormalizedBlaze(p, loc, loc['DATAHDR'])
# ----------------------------------------------------------------------
# Load transmission files
# ----------------------------------------------------------------------
transdata = spirouDB.GetDatabaseTellMap(p)
trans_files = transdata[0]
# make sure we have unique filenames for trans_files
trans_files = np.unique(trans_files)
# ----------------------------------------------------------------------
# Start plotting
# ----------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# start interactive plot
sPlt.start_interactive_session(p)
# ----------------------------------------------------------------------
# Load template (if available)
# ----------------------------------------------------------------------
# read filename from telluDB
template_file = spirouDB.GetDatabaseObjTemp(p,
loc['OBJNAME'],
required=False)
# if we don't have a template flag it
if template_file is None:
loc['FLAG_TEMPLATE'] = False
loc['TEMPLATE'] = None
else:
loc['FLAG_TEMPLATE'] = True
# load template
template, _, _, _ = spirouImage.ReadImage(p, template_file)
# add to loc
loc['TEMPLATE'] = template
# set the source for flag and template
loc.set_sources(['FLAG_TEMPLATE', 'TEMPLATE'], main_name)
# ----------------------------------------------------------------------
# load the expected atmospheric transmission
# ----------------------------------------------------------------------
# read filename from telluDB
tapas_file_names = spirouDB.GetDatabaseTellConv(p)
tapas_file_name = tapas_file_names[-1]
# load atmospheric transmission
loc['TAPAS_ALL_SPECIES'] = np.load(tapas_file_name)
loc.set_source('TAPAS_ALL_SPECIES', main_name)
# ----------------------------------------------------------------------
# Generate the absorption map
# ----------------------------------------------------------------------
# get number of files
nfiles = len(trans_files)
npc = p['TELLU_NUMBER_OF_PRINCIPLE_COMP']
# check that we have enough files (greater than number of principle
# components)
if nfiles <= npc:
emsg1 = 'Not enough "TELL_MAP" files in telluDB to run PCA analysis'
emsg2 = '\tNumber of files = {0}, number of PCA components = {1}'
emsg3 = '\tNumber of files > number of PCA components'
emsg4 = '\tAdd more files or reduce number of PCA components'
WLOG(p, 'error', [emsg1, emsg2.format(nfiles, npc), emsg3, emsg4])
# check whether we can used pre-saved abso
filetime = spirouImage.GetMostRecent(trans_files)
tout = spirouConfig.Constants.TELLU_ABSO_SAVE(p, filetime)
abso_save_file, absoprefix = tout
use_saved = os.path.exists(abso_save_file)
# noinspection PyBroadException
try:
# try loading from file
abso = np.load(abso_save_file)
# log progress
wmsg = 'Loaded abso from file {0}'.format(abso_save_file)
WLOG(p, '', wmsg)
except:
# set up storage for the absorption
abso = np.zeros([nfiles, np.product(loc['DATA'].shape)])
# loop around outputfiles and add them to abso
for it, filename in enumerate(trans_files):
# load data
data_it, _, _, _ = spirouImage.ReadImage(p, filename=filename)
# push data into array
abso[it, :] = data_it.reshape(np.product(loc['DATA'].shape))
# log progres
wmsg = 'Saving abso to file {0}'.format(abso_save_file)
WLOG(p, '', wmsg)
# remove all abso save files (only need most recent one)
afolder = os.path.dirname(abso_save_file)
afilelist = os.listdir(afolder)
for afile in afilelist:
if afile.startswith(absoprefix):
os.remove(os.path.join(afolder, afile))
# save to file for later use
np.save(abso_save_file, abso)
# filter out rows of all NaNs
# TODO: Why are we getting all NaN e2ds files?
abso_filtered = []
for row in range(len(abso)):
if np.sum(np.isnan(abso[row])) != len(abso[row]):
abso_filtered.append(abso[row])
else:
wargs = [trans_files[row]]
WLOG(p, '', 'Removing trans file {0} (all NaN)'.format(*wargs))
abso = np.array(abso_filtered)
# log the absorption cube
with warnings.catch_warnings(record=True) as w:
log_abso = np.log(abso)
# ----------------------------------------------------------------------
# Locate valid pixels for PCA
# ----------------------------------------------------------------------
# determining the pixels relevant for PCA construction
keep = np.isfinite(np.sum(abso, axis=0))
# log fraction of valid (non NaN) pixels
fraction = np.nansum(keep) / len(keep)
wmsg = 'Fraction of valid pixels (not NaNs) for PCA construction = {0:.3f}'
WLOG(p, '', wmsg.format(fraction))
# log fraction of valid pixels > 1 - (1/e)
with warnings.catch_warnings(record=True) as w:
keep &= np.min(log_abso, axis=0) > -1
fraction = np.nansum(keep) / len(keep)
wmsg = 'Fraction of valid pixels with transmission > 1 - (1/e) = {0:.3f}'
WLOG(p, '', wmsg.format(fraction))
# ----------------------------------------------------------------------
# Perform PCA analysis on the log of the telluric absorption map
# ----------------------------------------------------------------------
# Requires p:
# TELLU_NUMBER_OF_PRINCIPLE_COMP
# ADD_DERIV_PC
# FIT_DERIV_PC
# Requires loc:
# DATA
# Returns loc:
# PC
# NPC
# FIT_PC
loc = spirouTelluric.CalculateAbsorptionPCA(p, loc, log_abso, keep)
# Plot PCA components
# debug plot
if p['DRS_PLOT'] and (p['DRS_DEBUG'] > 1):
# plot the transmission map plot
sPlt.tellu_pca_comp_plot(p, loc)
# ----------------------------------------------------------------------
# Get master wavelength grid for shifting
# ----------------------------------------------------------------------
# get master wave map
loc['MASTERWAVEFILE'] = spirouDB.GetDatabaseMasterWave(p)
loc.set_source('MASTERWAVEFILE', main_name)
# log progress
wmsg1 = 'Getting master wavelength grid'
wmsg2 = '\tFile = {0}'.format(os.path.basename(loc['MASTERWAVEFILE']))
WLOG(p, '', [wmsg1, wmsg2])
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# read master wave map
mout = spirouImage.GetWaveSolution(p,
filename=loc['MASTERWAVEFILE'],
return_wavemap=True,
quiet=True,
fiber=wave_fiber)
_, loc['MASTERWAVE'], _ = mout
loc.set_source('MASTERWAVE', main_name)
# ----------------------------------------------------------------------
# Loop around telluric files
# ----------------------------------------------------------------------
for basefilename in p['ARG_FILE_NAMES']:
# ------------------------------------------------------------------
# Construct absolute file path
# ------------------------------------------------------------------
filename = os.path.join(p['ARG_FILE_DIR'], basefilename)
# ------------------------------------------------------------------
# Construct output file names
# ------------------------------------------------------------------
outfile1, tag1 = CONSTANTS.TELLU_FIT_OUT_FILE(p, filename)
outfilename1 = os.path.basename(outfile1)
outfile2, tag2 = CONSTANTS.TELLU_FIT_RECON_FILE(p, filename)
outfilename2 = os.path.basename(outfile2)
# ------------------------------------------------------------------
# Read filename
# ------------------------------------------------------------------
# read image
tdata, thdr, _, _ = spirouImage.ReadImage(p, filename)
# normalise with blaze function
loc['SP'] = tdata / loc['NBLAZE']
loc.set_source('SP', main_name)
# ------------------------------------------------------------------
# check that file has valid DPRTYPE
# ------------------------------------------------------------------
# get FP_FP DPRTYPE
p = spirouImage.ReadParam(p, thdr, 'KW_DPRTYPE', 'DPRTYPE', dtype=str)
# if dprtype is incorrect skip
if p['DPRTYPE'] not in p['ALLOWED_TELLURIC_DPRTYPES']:
wmsg1 = 'Skipping file (DPRTYPE incorrect)'
wmsg2 = '\t DPRTYPE = {0}'.format(p['DPRTYPE'])
WLOG(p, 'warning', [wmsg1, wmsg2])
continue
# ------------------------------------------------------------------
# Set storage
# ------------------------------------------------------------------
loc['RECON_ABSO'] = np.ones(np.product(loc['DATA'].shape))
loc['AMPS_ABSOL_TOTAL'] = np.zeros(loc['NPC'])
loc.set_sources(['RECON_ABSO', 'AMPS_ABSOL_TOTAL'], main_name)
# ------------------------------------------------------------------
# Read wavelength solution
# ------------------------------------------------------------------
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# get wavelength solution
wout = spirouImage.GetWaveSolution(p,
image=tdata,
hdr=thdr,
return_wavemap=True,
return_filename=True,
fiber=wave_fiber)
_, loc['WAVE_IT'], loc['WAVEFILE'], loc['WSOURCE'] = wout
loc.set_sources(['WAVE_IT', 'WAVEFILE', 'WSOURCE'], main_name)
# load wave keys
loc = spirouImage.GetWaveKeys(p, loc, thdr)
# ------------------------------------------------------------------
# Interpolate at shifted wavelengths (if we have a template)
# ------------------------------------------------------------------
if loc['FLAG_TEMPLATE']:
# Requires p:
# TELLU_FIT_KEEP_FRAC
# Requires loc:
# DATA
# TEMPLATE
# WAVE_IT
# Returns:
# TEMPLATE2
loc = spirouTelluric.BervCorrectTemplate(p, loc, thdr)
# debug plot
if p['DRS_PLOT'] and (p['DRS_DEBUG'] > 1):
# plot the transmission map plot
sPlt.tellu_fit_tellu_spline_plot(p, loc)
# store PC and TAPAS_ALL_SPECIES before shift
loc['PC_PRESHIFT'] = np.array(loc['PC'])
loc['TAPAS_ALL_PRESHIFT'] = np.array(loc['TAPAS_ALL_SPECIES'])
loc.set_sources(['PC_PRESHIFT', 'TAPAS_ALL_PRESHIFT'], main_name)
# ------------------------------------------------------------------
# Shift the pca components to correct frame
# ------------------------------------------------------------------
# log process
wmsg1 = 'Shifting PCA components from master wavelength grid'
wmsg2 = '\tFile = {0}'.format(os.path.basename(loc['MASTERWAVEFILE']))
WLOG(p, '', [wmsg1, wmsg2])
# shift pca components (one by one)
for comp in range(loc['NPC']):
wargs = [p, loc['PC'][:, comp], loc['MASTERWAVE'], loc['WAVE_IT']]
shift_pc = spirouTelluric.Wave2Wave(*wargs, reshape=True)
loc['PC'][:, comp] = shift_pc.reshape(wargs[1].shape)
wargs = [
p, loc['FIT_PC'][:, comp], loc['MASTERWAVE'], loc['WAVE_IT']
]
shift_fpc = spirouTelluric.Wave2Wave(*wargs, reshape=True)
loc['FIT_PC'][:, comp] = shift_fpc.reshape(wargs[1].shape)
# ------------------------------------------------------------------
# Shift the tapas spectrum to correct frame
# ------------------------------------------------------------------
# log process
wmsg1 = 'Shifting TAPAS spectrum from master wavelength grid'
wmsg2 = '\tFile = {0}'.format(os.path.basename(loc['MASTERWAVEFILE']))
WLOG(p, '', [wmsg1, wmsg2])
# shift tapas
for comp in range(len(loc['TAPAS_ALL_SPECIES'])):
wargs = [
p, loc['TAPAS_ALL_SPECIES'][comp], loc['MASTERWAVE'],
loc['WAVE_IT']
]
stapas = spirouTelluric.Wave2Wave(*wargs, reshape=True)
loc['TAPAS_ALL_SPECIES'][comp] = stapas.reshape(wargs[1].shape)
# Debug plot to test shifting
if p['DRS_PLOT'] and p['DRS_DEBUG'] > 1:
sPlt.tellu_fit_debug_shift_plot(p, loc)
# ------------------------------------------------------------------
# Calculate reconstructed absorption
# ------------------------------------------------------------------
# Requires p:
# TELLU_FIT_MIN_TRANSMISSION
# TELLU_FIT_NITER
# TELLU_LAMBDA_MIN
# TELLU_LAMBDA_MAX
# TELLU_FIT_VSINI
# TRANSMISSION_CUT
# FIT_DERIV_PC
# LOG_OPT
# Requires loc:
# FLAG_TEMPLATE
# TAPAS_ALL_SPECIES
# AMPS_ABSOL_TOTAL
# WAVE_IT
# TEMPLATE2
# FIT_PC
# NPC
# PC
# Returns loc:
# SP2
# TEMPLATE2
# RECON_ABSO
# AMPS_ABSOL_TOTAL
loc = spirouTelluric.CalcReconAbso(p, loc)
# debug plot
if p['DRS_PLOT'] > 0:
# plot the recon abso plot
sPlt.tellu_fit_recon_abso_plot(p, loc)
# ------------------------------------------------------------------
# Get molecular absorption
# ------------------------------------------------------------------
# Requires p:
# TELLU_FIT_LOG_LIMIT
# Requeres loc:
# RECON_ABSO
# TAPAS_ALL_SPECIES
# Returns loc:
# TAPAS_{molecule}
loc = spirouTelluric.CalcMolecularAbsorption(p, loc)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# get SNR for each order from header
nbo = loc['DATA'].shape[0]
snr_order = p['QC_FIT_TELLU_SNR_ORDER']
snr = spirouImage.Read1Dkey(p, thdr, p['kw_E2DS_SNR'][0], nbo)
# check that SNR is high enough
if snr[snr_order] < p['QC_FIT_TELLU_SNR_MIN']:
fmsg = 'low SNR in order {0}: ({1:.2f} < {2:.2f})'
fargs = [snr_order, snr[snr_order], p['QC_FIT_TELLU_SNR_MIN']]
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(snr[snr_order])
qc_name_str = 'SNR[{0}]'.format(snr_order)
qc_names.append(qc_name_str)
qc_logic.append('{0} < {1:.2f}'.format(qc_name_str,
p['QC_FIT_TELLU_SNR_ORDER']))
# ----------------------------------------------------------------------
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
continue
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Get components amplitudes for header
# ------------------------------------------------------------------
# get raw file name
raw_in_file = os.path.basename(p['FITSFILENAME'])
# copy original keys
hdict = spirouImage.CopyOriginalKeys(thdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
# set the input files
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBBLAZE'],
value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=loc['WAVEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=loc['WSOURCE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='file',
values=p['ARG_FILE_NAMES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# set tellu keys
npc = loc['NPC']
hdict = spirouImage.AddKey(p, hdict, p['KW_TELLU_NPC'], value=npc)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TELLU_FIT_DPC'],
value=p['FIT_DERIV_PC'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TELLU_ADD_DPC'],
value=p['ADD_DERIV_PC'])
if p['ADD_DERIV_PC']:
values = loc['AMPS_ABSOL_TOTAL'][:npc - 2]
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_TELLU_AMP_PC'],
values=values,
dim1name='amp')
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TELLU_DV_TELL1'],
value=loc['AMPS_ABSOL_TOTAL'][npc - 2])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TELLU_DV_TELL2'],
value=loc['AMPS_ABSOL_TOTAL'][npc - 1])
else:
values = loc['AMPS_ABSOL_TOTAL'][:npc]
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_TELLU_AMP_PC'],
values=values,
dim1name='PC')
# ------------------------------------------------------------------
# Write corrected spectrum to E2DS
# ------------------------------------------------------------------
# reform the E2DS
sp_out = loc['SP2'] / loc['RECON_ABSO']
sp_out = sp_out.reshape(loc['DATA'].shape)
# multiply by blaze
sp_out = sp_out * loc['NBLAZE']
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
# log progress
wmsg = 'Saving {0} to file'.format(outfilename1)
WLOG(p, '', wmsg)
# write sp_out to file
p = spirouImage.WriteImage(p, outfile1, sp_out, hdict)
# ------------------------------------------------------------------
# 1-dimension spectral S1D (uniform in wavelength)
# ------------------------------------------------------------------
# get arguments for E2DS to S1D
e2dsargs = [loc['WAVE'], sp_out, loc['BLAZE']]
# get 1D spectrum
xs1d1, ys1d1 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='wave')
# Plot the 1D spectrum
if p['DRS_PLOT'] > 0:
sPlt.ext_1d_spectrum_plot(p, xs1d1, ys1d1)
# construct file name
targs = [p, raw_in_file]
s1dfile1, tag3 = spirouConfig.Constants.TELLU_FIT_S1D_FILE1(*targs)
s1dfilename1 = os.path.basename(s1dfile1)
# add header keys
# set the version
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_EXT_TYPE'],
value=p['DPRTYPE'])
# log writing to file
wmsg = 'Saving 1D spectrum (uniform in wavelength) in {0}'
WLOG(p, '', wmsg.format(s1dfilename1))
# Write to file
columns = ['wavelength', 'flux', 'eflux']
values = [xs1d1, ys1d1, np.zeros_like(ys1d1)]
units = ['nm', None, None]
s1d1 = spirouImage.MakeTable(p, columns, values, units=units)
spirouImage.WriteTable(p, s1d1, s1dfile1, header=hdict)
# ------------------------------------------------------------------
# 1-dimension spectral S1D (uniform in velocity)
# ------------------------------------------------------------------
# get arguments for E2DS to S1D
e2dsargs = [loc['WAVE'], sp_out, loc['BLAZE']]
# get 1D spectrum
xs1d2, ys1d2 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='velocity')
# Plot the 1D spectrum
if p['DRS_PLOT'] > 0:
sPlt.ext_1d_spectrum_plot(p, xs1d2, ys1d2)
# construct file name
targs = [p, raw_in_file]
s1dfile2, tag4 = spirouConfig.Constants.TELLU_FIT_S1D_FILE2(*targs)
s1dfilename2 = os.path.basename(s1dfile2)
# add header keys
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_EXT_TYPE'],
value=p['DPRTYPE'])
# log writing to file
wmsg = 'Saving 1D spectrum (uniform in velocity) in {0}'
WLOG(p, '', wmsg.format(s1dfilename2))
# Write to file
columns = ['wavelength', 'flux', 'eflux']
values = [xs1d2, ys1d2, np.zeros_like(ys1d2)]
units = ['nm', None, None]
s1d2 = spirouImage.MakeTable(p, columns, values, units=units)
spirouImage.WriteTable(p, s1d2, s1dfile2, header=hdict)
# ------------------------------------------------------------------
# Write reconstructed absorption to E2DS
# ------------------------------------------------------------------
# set up empty storage
recon_abso2 = np.zeros_like(loc['DATA'])
# get dimensions of data
ydim, xdim = loc['DATA'].shape
# loop around orders
for order_num in range(ydim):
# get start and end points
start, end = xdim * order_num, xdim * order_num + xdim
# save to storage
recon_abso2[order_num, :] = loc['RECON_ABSO'][start:end]
# add molecular absorption to file
for it, molecule in enumerate(p['TELLU_ABSORBERS'][1:]):
# get molecule keyword store and key
molkey = '{0}_{1}'.format(p['KW_TELLU_ABSO'][0], molecule.upper())
molkws = [molkey, 0, 'Absorption in {0}'.format(molecule.upper())]
# load into hdict
hdict = spirouImage.AddKey(p, hdict, molkws, value=loc[molkey])
# add water col
if molecule == 'h2o':
loc['WATERCOL'] = loc[molkey]
# set source
loc.set_source('WATERCOL', main_name)
# add the tau keys
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TAU_H2O'],
value=loc['TAU_H2O'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TAU_REST'],
value=loc['TAU_REST'])
# log progress
wmsg = 'Saving {0} to file'.format(outfilename2)
WLOG(p, '', wmsg)
# write recon_abso to file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
p = spirouImage.WriteImage(p, outfile2, recon_abso2, hdict)
# ------------------------------------------------------------------
# Update the Telluric database
# ------------------------------------------------------------------
if p['QC']:
# add TELLU_OBJ to telluric database
oparams = dict(objname=loc['OBJNAME'],
berv=loc['BERV'],
airmass=loc['AIRMASS'],
watercol=loc['WATERCOL'])
spirouDB.UpdateDatabaseTellObj(p, outfilename1, **oparams)
# copy file to database
spirouDB.PutTelluFile(p, outfile1)
# add TELLU_RECON to telluric database
# add TELLU_OBJ to telluric database
oparams = dict(objname=loc['OBJNAME'],
berv=loc['BERV'],
airmass=loc['AIRMASS'],
watercol=loc['WATERCOL'])
spirouDB.UpdateDatabaseTellRecon(p, outfilename2, **oparams)
# copy file to database
spirouDB.PutTelluFile(p, outfile2)
# ----------------------------------------------------------------------
# End plotting
# ----------------------------------------------------------------------
# debug plot
if p['DRS_PLOT'] > 0:
# end interactive session
sPlt.end_interactive_session(p)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p,
night_name,
files,
mainfitsdir='reduced')
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# set up function name
main_name = __NAME__ + '.main()'
# ----------------------------------------------------------------------
# load up first file
# ----------------------------------------------------------------------
# construct loc parameter dictionary for storing data
loc = ParamDict()
# read first file and assign values
rdata = spirouImage.ReadImage(p, p['FITSFILENAME'])
loc['DATA'], loc['DATAHDR'] = rdata[:2]
# Get output key
loc['OUTPUT'] = loc['DATAHDR'][p['KW_OUTPUT'][0]]
# set source
source = main_name + '+ spirouImage.ReadParams()'
loc.set_source('OUTPUT', source)
# ----------------------------------------------------------------------
# Get database files
# ----------------------------------------------------------------------
# get current telluric maps from telluDB
files, output_codes = [], []
# filter by object name (only keep OBJNAME objects) and only keep
# unique filenames
use_files = []
for it in range(len(files)):
# check that OUTPUT is correct
cond1 = p['KW_OUTPUT'] in output_codes[it]
# check that filename is not already used
cond2 = files[it] not in use_files
# append to file list if criteria correct
if cond1 and cond2:
use_files.append(files[it])
# log if we have no files
if len(use_files) == 0:
wmsg = 'No files found for OUTPUT ="{0}" skipping'
WLOG(p, 'warning', wmsg.format(loc['KW_OUTPUT']))
# End Message
wmsg = 'Recipe {0} has been successfully completed'
WLOG(p, 'info', wmsg.format(p['PROGRAM']))
# return a copy of locally defined variables in the memory
return dict(locals())
else:
# log how many found
wmsg = 'N={0} files found for OUTPUT="{1}"'
WLOG(p, '', wmsg.format(len(use_files), loc['OUTPUT']))
# ----------------------------------------------------------------------
# Set up storage for cubes (NaN arrays)
# ----------------------------------------------------------------------
# set up flat size
dims = [loc['DATA'].shape[0], loc['DATA'].shape[1], len(use_files)]
flatsize = np.product(dims)
# create NaN filled storage
big_cube = np.repeat([np.nan], flatsize).reshape(*dims)
big_cube0 = np.repeat([np.nan], flatsize).reshape(*dims)
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# get master wave map
loc['MASTERWAVEFILE'] = spirouDB.GetDatabaseMasterWave(p)
# read master wave map
mout = spirouImage.GetWaveSolution(p,
filename=loc['MASTERWAVEFILE'],
return_wavemap=True,
quiet=True,
fiber=wave_fiber)
loc['MASTERWAVEPARAMS'], loc['MASTERWAVE'], loc['WSOURCE'] = mout
keys = ['MASTERWAVEPARAMS', 'MASTERWAVE', 'MASTERWAVEFILE', 'WSOURCE']
loc.set_sources(keys, main_name)
# ----------------------------------------------------------------------
# Loop through input files
# ----------------------------------------------------------------------
base_filelist, berv_list = [], []
# loop through files
for it, filename in enumerate(use_files):
# get base filenmae
basefilename = os.path.basename(filename)
# append basename to file list
base_filelist.append(basefilename)
# ------------------------------------------------------------------
# Load the data for this file
tdata, thdr, _, _ = spirouImage.ReadImage(p, filename)
# ------------------------------------------------------------------
# log stats
wmsg = 'Processing file {0} of {1} file={2}'
wargs = [it + 1, len(use_files), basefilename]
WLOG(p, '', wmsg.format(*wargs))
# ------------------------------------------------------------------
# add to cube storage
big_cube0[:, :, it] = tdata
# ----------------------------------------------------------------------
# Save cubes to file
# ----------------------------------------------------------------------
# get raw file name
raw_in_file = os.path.basename(p['FITSFILENAME'])
# construct file names
outfile = raw_in_file.replace('.fits', '_stack.fits')
tag = loc['OUTPUT'] + '_STACK'
# log big cube 1
wmsg1 = 'Saving bigcube to file {0}'.format(os.path.basename(outfile))
# hdict is first file keys
hdict = spirouImage.CopyOriginalKeys(loc['DATAHDR'])
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
# set the input files
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=loc['MASTERWAVEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=loc['WSOURCE'])
# add file list to header
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_OBJFILELIST'],
values=base_filelist)
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_OBJBERVLIST'],
values=berv_list)
# add wave solution coefficients
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_WAVE_PARAM'],
values=loc['MASTERWAVEPARAMS'])
# log big cube 0
wmsg = 'Saving bigcube0 to file {0}'.format(os.path.basename(outfile))
# save big cube 0
big_cube_s0 = np.swapaxes(big_cube0, 1, 2)
p = spirouImage.WriteImageMulti(p, outfile, big_cube_s0, hdict)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p,
night_name,
files,
mainfitsdir='reduced')
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# set up function name
main_name = __NAME__ + '.main()'
# ------------------------------------------------------------------
# Load first file
# ------------------------------------------------------------------
loc = ParamDict()
rd = spirouImage.ReadImage(p, p['FITSFILENAME'])
loc['DATA'], loc['DATAHDR'], loc['YDIM'], loc['XDIM'] = rd
loc.set_sources(['DATA', 'DATAHDR', 'XDIM', 'YDIM'], main_name)
# ------------------------------------------------------------------
# Get the wave solution
# ------------------------------------------------------------------
masterwavefile = spirouDB.GetDatabaseMasterWave(p)
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# read master wave map
wout = spirouImage.GetWaveSolution(p,
filename=masterwavefile,
return_wavemap=True,
quiet=True,
return_header=True,
fiber=wave_fiber)
_, loc['WAVE'], loc['WAVEFILE'], _ = wout
loc.set_sources(['WAVE', 'WAVEFILE'], main_name)
# get the wave keys
loc = spirouImage.GetWaveKeys(p, loc, loc['DATAHDR'])
# ------------------------------------------------------------------
# Construct convolution kernels (used in GetMolecularTellLines)
# ------------------------------------------------------------------
loc = spirouTelluric.ConstructConvKernel1(p, loc)
# ------------------------------------------------------------------
# Get molecular telluric lines
# ------------------------------------------------------------------
loc = spirouTelluric.GetMolecularTellLines(p, loc)
# if TAPAS FNAME is not None we generated a new file so should add to tellDB
if loc['TAPAS_FNAME'] is not None:
# add to the telluric database
spirouDB.UpdateDatabaseTellConv(p, loc['TAPAS_FNAME'], loc['DATAHDR'])
# put file in telluDB
spirouDB.PutTelluFile(p, loc['TAPAS_ABSNAME'])
# ----------------------------------------------------------------------
# load the expected atmospheric transmission
# ----------------------------------------------------------------------
# read filename from telluDB
tapas_file_names = spirouDB.GetDatabaseTellConv(p)
tapas_file_name = tapas_file_names[-1]
# load atmospheric transmission
sp_tapas = np.load(tapas_file_name)
loc['TAPAS_ALL_SPECIES'] = sp_tapas
# extract the water and other line-of-sight optical depths
loc['TAPAS_WATER'] = sp_tapas[1, :]
loc['TAPAS_OTHERS'] = np.prod(sp_tapas[2:, :], axis=0)
loc.set_sources(['TAPAS_ALL_SPECIES', 'TAPAS_WATER', 'TAPAS_OTHERS'],
main_name)
# ------------------------------------------------------------------
# Get master wave solution map
# ------------------------------------------------------------------
# get master wave map
masterwavefile = spirouDB.GetDatabaseMasterWave(p)
# log process
wmsg1 = 'Shifting transmission map on to master wavelength grid'
wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile))
WLOG(p, '', [wmsg1, wmsg2])
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# read master wave map
mout = spirouImage.GetWaveSolution(p,
filename=masterwavefile,
return_wavemap=True,
quiet=True,
return_header=True,
fiber=wave_fiber)
masterwavep, masterwave, masterwaveheader, mwsource = mout
# get wave acqtimes
master_acqtimes = spirouDB.GetTimes(p, masterwaveheader)
# ------------------------------------------------------------------
# Loop around the files
# ------------------------------------------------------------------
# construct extension
tellu_ext = '{0}_{1}.fits'
# get current telluric maps from telluDB
# tellu_db_data = spirouDB.GetDatabaseTellMap(p, required=False)
# tellu_db_files = tellu_db_data[0]
# storage for valid output files
loc['OUTPUTFILES'] = []
# loop around the files
for basefilename in p['ARG_FILE_NAMES']:
# ------------------------------------------------------------------
# Get absolute path of filename
# ------------------------------------------------------------------
filename = os.path.join(p['ARG_FILE_DIR'], basefilename)
# ------------------------------------------------------------------
# Read obj telluric file and correct blaze (per order)
# ------------------------------------------------------------------
# get image
sp, shdr, _, _ = spirouImage.ReadImage(p, filename)
# ------------------------------------------------------------------
# check that file has valid DPRTYPE
# ------------------------------------------------------------------
# get FP_FP DPRTYPE
p = spirouImage.ReadParam(p, shdr, 'KW_DPRTYPE', 'DPRTYPE', dtype=str)
# if dprtype is incorrect skip
if p['DPRTYPE'] not in p['ALLOWED_TELLURIC_DPRTYPES']:
wmsg1 = 'Skipping file (DPRTYPE incorrect)'
wmsg2 = '\t DPRTYPE = {0}'.format(p['DPRTYPE'])
WLOG(p, 'warning', [wmsg1, wmsg2])
continue
# get blaze
p, blaze = spirouImage.ReadBlazeFile(p, shdr)
# get the blaze percentile
blaze_p = p['MKTELLU_BLAZE_PERCENTILE']
# loop through blaze orders, normalize blaze by its peak amplitude
for order_num in range(sp.shape[0]):
# normalize the spectrum
spo, bzo = sp[order_num], blaze[order_num]
sp[order_num] = spo / np.nanpercentile(spo, blaze_p)
# normalize the blaze
blaze[order_num] = bzo / np.nanpercentile(bzo, blaze_p)
# find where the blaze is bad
with warnings.catch_warnings(record=True) as _:
badblaze = blaze < p['MKTELLU_CUT_BLAZE_NORM']
# set bad blaze to NaN
blaze[badblaze] = np.nan
# set to NaN values where spectrum is zero
zeromask = sp == 0
sp[zeromask] = np.nan
# divide spectrum by blaze
with warnings.catch_warnings(record=True) as _:
sp = sp / blaze
# add sp to loc
loc['SP'] = sp
loc.set_source('SP', main_name)
# ----------------------------------------------------------------------
# Get object name, airmass and berv
# ----------------------------------------------------------------------
# Get object name
loc['OBJNAME'] = spirouImage.GetObjName(p, shdr)
# Get the airmass
loc['AIRMASS'] = spirouImage.GetAirmass(p, shdr)
# Get the Barycentric correction from header
p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, shdr)
# set sources
source = main_name + '+ spirouImage.ReadParams()'
loc.set_sources(['OBJNAME', 'AIRMASS'], source)
# ------------------------------------------------------------------
# get output transmission filename
outfile, tag1 = spirouConfig.Constants.TELLU_TRANS_MAP_FILE(
p, filename)
outfilename = os.path.basename(outfile)
loc['OUTPUTFILES'].append(outfile)
# ----------------------------------------------------------------------
# Load template (if available)
# ----------------------------------------------------------------------
# read filename from telluDB
template_file = spirouDB.GetDatabaseObjTemp(p,
loc['OBJNAME'],
required=False)
# if we don't have a template flag it
if template_file is None:
loc['FLAG_TEMPLATE'] = False
loc['TEMPLATE'] = None
# construct progres string
pstring = 'No template found.'
else:
loc['FLAG_TEMPLATE'] = True
# load template
template, _, _, _ = spirouImage.ReadImage(p, template_file)
# add to loc
loc['TEMPLATE'] = template
# construct progres string
template_bfile = os.path.basename(template_file)
pstring = 'Using template {0}'.format(template_bfile)
# set the source for flag and template
loc.set_sources(['FLAG_TEMPLATE', 'TEMPLATE'], main_name)
# ------------------------------------------------------------------
# log processing file
wmsg = 'Processing file {0}. {1}'
WLOG(p, '', [wmsg.format(outfilename, pstring)])
# ------------------------------------------------------------------
# Check that basefile is not in blacklist
# ------------------------------------------------------------------
blacklist_check = spirouTelluric.CheckBlackList(loc['OBJNAME'])
if blacklist_check:
# log black list file found
wmsg = 'File {0} is blacklisted (OBJNAME={1}). Skipping'
wargs = [basefilename, loc['OBJNAME']]
WLOG(p, 'warning', wmsg.format(*wargs))
# skip this file
continue
# ------------------------------------------------------------------
# deal with applying template to spectrum
# ------------------------------------------------------------------
# Requires from loc:
# TEMPLATE (None or template loaded from file)
# FLAG_TEMPLATE
# WAVE
# SP
# BERV
#
# Returns:
# SP (modified if template was used)
# TEMPLATE
# WCONV
loc = spirouTelluric.ApplyTemplate(p, loc)
# ------------------------------------------------------------------
# calcullate telluric absorption (with a sigma clip loop)
# ------------------------------------------------------------------
# Requires from loc:
# AIRMASS
# WAVE
# SP
# WCONV
# Returns:
# PASSED [Bool] True or False
# SP_OUT
# SED_OUT
# RECOV_AIRMASS
# RECOV_WATER
loc = spirouTelluric.CalcTelluAbsorption(p, loc)
# calculate tranmission map from sp and sed
transmission_map = loc['SP_OUT'] / loc['SED_OUT']
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# if array is completely NaNs it shouldn't pass
if np.sum(np.isfinite(transmission_map)) == 0:
fail_msg.append('transmission map is all NaNs')
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append('NaN')
qc_names.append('image')
qc_logic.append('image is all NaN')
# ----------------------------------------------------------------------
# get SNR for each order from header
nbo = loc['DATA'].shape[0]
snr_order = p['QC_MK_TELLU_SNR_ORDER']
snr = spirouImage.Read1Dkey(p, shdr, p['kw_E2DS_SNR'][0], nbo)
# check that SNR is high enough
if snr[snr_order] < p['QC_MK_TELLU_SNR_MIN']:
fmsg = 'low SNR in order {0}: ({1:.2f} < {2:.2f})'
fargs = [snr_order, snr[snr_order], p['QC_MK_TELLU_SNR_MIN']]
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(snr[snr_order])
qc_name_str = 'SNR[{0}]'.format(snr_order)
qc_names.append(qc_name_str)
qc_logic.append('{0} < {1:.2f}'.format(qc_name_str,
p['QC_MK_TELLU_SNR_ORDER']))
# ----------------------------------------------------------------------
# check that the file passed the CalcTelluAbsorption sigma clip loop
if not loc['PASSED']:
fmsg = 'File {0} did not converge on a solution in function: {1}'
fargs = [basefilename, 'spirouTelluric.CalcTelluAbsorption()']
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(basefilename)
qc_names.append('FILE')
qc_logic.append('FILE did not converge')
# ----------------------------------------------------------------------
# check that the airmass is not too different from input airmass
airmass_diff = np.abs(loc['RECOV_AIRMASS'] - loc['AIRMASS'])
fargs = [
loc['RECOV_AIRMASS'], loc['AIRMASS'], p['QC_MKTELLU_AIRMASS_DIFF']
]
if airmass_diff > p['QC_MKTELLU_AIRMASS_DIFF']:
fmsg = ('Recovered airmass to de-similar than input airmass.'
'Recovered: {0:.3f}. Input: {1:.3f}. QC limit = {2}')
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(airmass_diff)
qc_names.append('airmass_diff')
qc_logic.append('airmass_diff > {0:.2f}'
''.format(p['QC_MKTELLU_AIRMASS_DIFF']))
# ----------------------------------------------------------------------
# check that the water vapor is within limits
water_cond1 = loc['RECOV_WATER'] < p['MKTELLU_TRANS_MIN_WATERCOL']
water_cond2 = loc['RECOV_WATER'] > p['MKTELLU_TRANS_MAX_WATERCOL']
fargs = [
p['MKTELLU_TRANS_MIN_WATERCOL'], p['MKTELLU_TRANS_MAX_WATERCOL']
]
if water_cond1 or water_cond2:
fmsg = ('Recovered water vapor optical depth not between {0:.3f} '
'and {1:.3f}')
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(loc['RECOV_WATER'])
qc_names.append('RECOV_WATER')
qc_logic.append('RECOV_WATER not between {0:.3f} and {1:.3f}'
''.format(*fargs))
# ----------------------------------------------------------------------
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
continue
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Save transmission map to file
# ------------------------------------------------------------------
# get raw file name
raw_in_file = os.path.basename(p['FITSFILENAME'])
# copy original keys
hdict = spirouImage.CopyOriginalKeys(loc['DATAHDR'])
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
# set the input files
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBBLAZE'],
value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=os.path.basename(masterwavefile))
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=mwsource)
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='file',
values=p['ARG_FILE_NAMES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# add wave solution date
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_TIME1'],
value=master_acqtimes[0])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_TIME2'],
value=master_acqtimes[1])
# add wave solution number of orders
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_ORD_N'],
value=masterwavep.shape[0])
# add wave solution degree of fit
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_LL_DEG'],
value=masterwavep.shape[1] - 1)
# add wave solution coefficients
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_WAVE_PARAM'],
values=masterwavep)
# add telluric keys
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TELLU_AIRMASS'],
value=loc['RECOV_AIRMASS'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TELLU_WATER'],
value=loc['RECOV_WATER'])
# write to file
p = spirouImage.WriteImage(p, outfile, transmission_map, hdict)
# ------------------------------------------------------------------
# Add transmission map to telluDB
# ------------------------------------------------------------------
if p['QC']:
# copy tellu file to the telluDB folder
spirouDB.PutTelluFile(p, outfile)
# update the master tellu DB file with transmission map
targs = [
p, outfilename, loc['OBJNAME'], loc['RECOV_AIRMASS'],
loc['RECOV_WATER']
]
spirouDB.UpdateDatabaseTellMap(*targs)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p,
night_name,
files,
mainfitsdir='reduced')
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# set up function name
main_name = __NAME__ + '.main()'
# ----------------------------------------------------------------------
# load up first file
# ----------------------------------------------------------------------
# construct loc parameter dictionary for storing data
loc = ParamDict()
# read first file and assign values
rdata = spirouImage.ReadImage(p, p['FITSFILENAME'])
loc['DATA'], loc['DATAHDR'] = rdata[:2]
# Get object name and airmass
loc['OBJNAME'] = spirouImage.GetObjName(p, loc['DATAHDR'])
loc.set_source('OBJNAME', main_name)
loc['AIRMASS'] = spirouImage.GetAirmass(p, loc['DATAHDR'])
# set source
source = main_name + '+ spirouImage.ReadParams()'
loc.set_sources(['OBJNAME', 'AIRMASS'], source)
# ----------------------------------------------------------------------
# Get and Normalise the blaze
# ----------------------------------------------------------------------
p, loc = spirouTelluric.GetNormalizedBlaze(p, loc, loc['DATAHDR'])
# ----------------------------------------------------------------------
# Get database files
# ----------------------------------------------------------------------
# get current telluric maps from telluDB
tellu_db_data = spirouDB.GetDatabaseTellObj(p, required=False)
tellu_db_files, tellu_db_names = tellu_db_data[0], tellu_db_data[1]
# sort files by name
# sortmask = spirouImage.SortByName(tellu_db_files)
# tellu_db_files = np.array(tellu_db_files)[sortmask]
# tellu_db_names = np.array(tellu_db_names)[sortmask]
# filter by object name (only keep OBJNAME objects) and only keep
# unique filenames
tell_files, tell_names = [], []
for it in range(len(tellu_db_files)):
# check that objname is correct
cond1 = loc['OBJNAME'] in tellu_db_names[it]
# check that filename is not already used
cond2 = tellu_db_files[it] not in tell_files
# append to file list if criteria correct
if cond1 and cond2:
tell_files.append(tellu_db_files[it])
tell_names.append(tellu_db_names[it])
# log if we have no files
if len(tell_files) == 0:
wmsg = 'No "TELL_OBJ" files found for object ="{0}" skipping'
WLOG(p, 'warning', wmsg.format(loc['OBJNAME']))
# End Message
wmsg = 'Recipe {0} has been successfully completed'
WLOG(p, 'info', wmsg.format(p['PROGRAM']))
# return a copy of locally defined variables in the memory
return dict(locals())
else:
# log how many found
wmsg = 'N={0} "TELL_OBJ" files found for object ="{1}"'
WLOG(p, 'info', wmsg.format(len(tell_files), loc['OBJNAME']))
# ----------------------------------------------------------------------
# Set up storage for cubes (NaN arrays)
# ----------------------------------------------------------------------
# set up flat size
dims = [loc['DATA'].shape[0], loc['DATA'].shape[1], len(tell_files)]
flatsize = np.product(dims)
# create NaN filled storage
big_cube = np.repeat([np.nan], flatsize).reshape(*dims)
big_cube0 = np.repeat([np.nan], flatsize).reshape(*dims)
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# get master wave map
loc['MASTERWAVEFILE'] = spirouDB.GetDatabaseMasterWave(p)
# read master wave map
mout = spirouImage.GetWaveSolution(p,
filename=loc['MASTERWAVEFILE'],
return_wavemap=True,
quiet=True,
fiber=wave_fiber)
loc['MASTERWAVEPARAMS'], loc['MASTERWAVE'], loc['WSOURCE'] = mout
keys = ['MASTERWAVEPARAMS', 'MASTERWAVE', 'MASTERWAVEFILE', 'WSOURCE']
loc.set_sources(keys, main_name)
# ----------------------------------------------------------------------
# Compile a median SNR for rejection of bad files
# ----------------------------------------------------------------------
snr_all = []
# choose snr to check
snr_order = p['QC_FIT_TELLU_SNR_ORDER']
# loop through files
for it, filename in enumerate(tell_files):
header = spirouImage.ReadHeader(p, filename)
# get the SNR from header
nbo = spirouImage.ReadParam(p,
header,
'KW_WAVE_ORD_N',
dtype=int,
return_value=True)
snr = spirouImage.Read1Dkey(p, header, p['kw_E2DS_SNR'][0], nbo)
# append snr_all
snr_all.append(snr[snr_order])
# work our bad snr (less than half the median SNR)
snr_thres = np.nanmedian(snr_all) / 2.0
bad_snr_objects = np.where(snr_all < snr_thres)[0]
# ----------------------------------------------------------------------
# Loop through input files
# ----------------------------------------------------------------------
# empty lists for storage
loc['BASE_ROWNUM'], loc['BASE_FILELIST'] = [], []
loc['BASE_OBJNAME'], loc['BASE_OBJECT'] = [], []
loc['BASE_BERVLIST'], loc['BASE_WAVELIST'] = [], []
loc['BASE_SNRLIST_{0}'.format(snr_order)] = []
loc['BASE_DATELIST'], loc['BASE_VERSION'] = [], []
loc['BASE_DARKFILE'], loc['BASE_BADFILE'] = [], []
loc['BASE_LOCOFILE'], loc['BASE_BLAZFILE'] = [], []
loc['BASE_FLATFILE'], loc['BASE_SHAPEFILE'] = [], []
# loop through files
for it, filename in enumerate(tell_files):
# get base filenmae
basefilename = os.path.basename(filename)
# ------------------------------------------------------------------
# skip if in bad snr objects
if it in bad_snr_objects:
wargs = [it + 1, len(tell_files), snr_all[it], snr_thres]
wmsg1 = ('Skipping file {0} of {1} due to bad SNR ({2:.3f} < '
'{3:.3f})'.format(*wargs))
wmsg2 = '\tFile = {0}'.format(basefilename)
WLOG(p, 'warning', [wmsg1, wmsg2])
continue
# ------------------------------------------------------------------
# append basename to file list
loc['BASE_FILELIST'].append(basefilename)
# ------------------------------------------------------------------
# create image for storage
image = np.repeat([np.nan], np.product(loc['DATA'].shape))
image = image.reshape(loc['DATA'].shape)
# ------------------------------------------------------------------
# Load the data for this file
tdata0, thdr, _, _ = spirouImage.ReadImage(p, filename)
nbo, npix = tdata0.shape
# Correct for the blaze
tdata = tdata0 / loc['NBLAZE']
# get berv and add to list
if p['KW_BERV'][0] in thdr:
loc['BASE_BERVLIST'].append('{0}'.format(thdr[p['KW_BERV'][0]]))
else:
loc['BASE_BERVLIST'].append('UNKNOWN')
# ------------------------------------------------------------------
# Get parameters from header
snr = snr_all[it]
dateobs = spirouImage.ReadParam(p,
thdr,
'KW_DATE_OBS',
dtype=str,
return_value=True)
utcobs = spirouImage.ReadParam(p,
thdr,
'KW_UTC_OBS',
dtype=str,
return_value=True)
tobjname = spirouImage.ReadParam(p,
thdr,
'KW_OBJNAME',
dtype=str,
return_value=True)
tobject = spirouImage.ReadParam(p,
thdr,
'KW_OBJECT',
dtype=str,
return_value=True)
tversion = spirouImage.ReadParam(p,
thdr,
'KW_version',
dtype=str,
return_value=True)
tdarkfile = spirouImage.ReadParam(p,
thdr,
'KW_CDBDARK',
dtype=str,
return_value=True)
tbadfile = spirouImage.ReadParam(p,
thdr,
'KW_CDBBAD',
dtype=str,
return_value=True)
tlocofile = spirouImage.ReadParam(p,
thdr,
'KW_CDBLOCO',
dtype=str,
return_value=True)
tblazfile = spirouImage.ReadParam(p,
thdr,
'KW_CDBBLAZE',
dtype=str,
return_value=True)
tflatfile = spirouImage.ReadParam(p,
thdr,
'KW_CDBFLAT',
dtype=str,
return_value=True)
tshapfile = spirouImage.ReadParam(p,
thdr,
'KW_CDBSHAPE',
dtype=str,
return_value=True)
# append to lists
loc['BASE_ROWNUM'].append(it)
loc['BASE_SNRLIST_{0}'.format(snr_order)].append(snr_all[it])
loc['BASE_DATELIST'].append('{0}_{1}'.format(dateobs, utcobs))
loc['BASE_OBJNAME'].append(tobjname)
loc['BASE_OBJECT'].append(tobject)
loc['BASE_VERSION'].append(tversion)
loc['BASE_DARKFILE'].append(tdarkfile)
loc['BASE_BADFILE'].append(tbadfile)
loc['BASE_LOCOFILE'].append(tlocofile)
loc['BASE_BLAZFILE'].append(tblazfile)
loc['BASE_FLATFILE'].append(tflatfile)
loc['BASE_SHAPEFILE'].append(tshapfile)
# ------------------------------------------------------------------
# Get the wave solution for this file
# ------------------------------------------------------------------
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# need to reload calibDB
# as we have custom arguments need to load the calibration database
calib_db = dict(p['CALIBDB'])
del p['CALIBDB']
p = spirouStartup.LoadCalibDB(p, header=thdr)
# get wave solution
wout = spirouImage.GetWaveSolution(p,
image=tdata,
hdr=thdr,
return_wavemap=True,
fiber=wave_fiber,
return_filename=True)
_, loc['WAVE'], loc['WAVEFILE'], _ = wout
loc.set_sources(['WAVE', 'WAVEFILE'], main_name)
# add wave to wave list
loc['BASE_WAVELIST'].append(loc['WAVEFILE'])
# readd original calibDB to p
p['CALIBDB'] = dict(calib_db)
# ------------------------------------------------------------------
# Get the Barycentric correction from header
dv, _, _ = spirouImage.GetBERV(p, thdr)
# ------------------------------------------------------------------
# log stats
wmsg = 'Processing file {0} of {1} file={2} dv={3}'
wargs = [it + 1, len(tell_files), basefilename, dv]
WLOG(p, 'info', wmsg.format(*wargs))
# ------------------------------------------------------------------
# shift to correct berv
dvshift = spirouMath.relativistic_waveshift(dv, units='km/s')
image = spirouTelluric.Wave2Wave(p, tdata, loc['WAVE'] * dvshift,
loc['MASTERWAVE'])
# ------------------------------------------------------------------
# loop around orders
for order_num in range(loc['DATA'].shape[0]):
# normalise the tdata
tdata[order_num, :] /= np.nanmedian(tdata[order_num, :])
image[order_num, :] /= np.nanmedian(image[order_num, :])
# ------------------------------------------------------------------
# add to cube storage
big_cube[:, :, it] = image
big_cube0[:, :, it] = tdata
# ----------------------------------------------------------------------
# log if we have no files
if len(loc['BASE_FILELIST']) == 0:
wmsg = 'No good files found for object ="{0}" skipping'
WLOG(p, 'warning', wmsg.format(loc['OBJNAME']))
# End Message
wmsg = 'Recipe {0} has been successfully completed'
WLOG(p, 'info', wmsg.format(p['PROGRAM']))
# return a copy of locally defined variables in the memory
return dict(locals())
# ----------------------------------------------------------------------
# make median image
with warnings.catch_warnings(record=True) as _:
big_cube_med = np.nanmedian(big_cube, axis=2)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# TODO: Needs doing
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# add to qc header lists
qc_values.append('None')
qc_names.append('None')
qc_logic.append('None')
qc_pass.append(1)
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ----------------------------------------------------------------------
# Write Cube median (the template) to file
# ----------------------------------------------------------------------
# get raw file name
raw_in_file = os.path.basename(p['FITSFILENAME'])
# construct filename
outfile, tag = spirouConfig.Constants.OBJTELLU_TEMPLATE_FILE(p, loc)
outfilename = os.path.basename(outfile)
# hdict is first file keys
hdict = spirouImage.CopyOriginalKeys(loc['DATAHDR'])
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
# set the input files
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=loc['MASTERWAVEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=loc['WSOURCE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='file',
values=p['ARG_FILE_NAMES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# add wave solution coefficients
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_WAVE_PARAM'],
values=loc['MASTERWAVEPARAMS'])
# write to file
p = spirouImage.WriteImage(p, outfile, big_cube_med, hdict)
# ----------------------------------------------------------------------
# Update the telluric database with the template
# ----------------------------------------------------------------------
spirouDB.UpdateDatabaseObjTemp(p, outfilename, loc['OBJNAME'],
loc['DATAHDR'])
# put file in telluDB
spirouDB.PutTelluFile(p, outfile)
# ----------------------------------------------------------------------
# Save cubes to file
# ----------------------------------------------------------------------
# make big cube table
big_table = spirouTelluric.ConstructBigTable(p, loc)
# construct file names
outfile1, tag1 = spirouConfig.Constants.OBJTELLU_TEMPLATE_CUBE_FILE1(
p, loc)
outfile2, tag2 = spirouConfig.Constants.OBJTELLU_TEMPLATE_CUBE_FILE2(
p, loc)
# log big cube 1
wmsg1 = 'Saving bigcube to file {0}'.format(os.path.basename(outfile1))
# save big cube 1
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
big_cube_s = np.swapaxes(big_cube, 1, 2)
p = spirouImage.WriteImageTable(p,
outfile1,
image=big_cube_s,
table=big_table,
hdict=hdict)
# log big cube 0
wmsg = 'Saving bigcube0 to file {0}'.format(os.path.basename(outfile2))
# save big cube 0
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
big_cube_s0 = np.swapaxes(big_cube0, 1, 2)
p = spirouImage.WriteImageTable(p,
outfile2,
image=big_cube_s0,
table=big_table,
hdict=hdict)
# # mega plot
# nfiles = big_cube_s0.shape[1]
# ncols = int(np.ceil(np.sqrt(nfiles)))
# nrows = int(np.ceil(nfiles/ncols))
# fig, frames = plt.subplots(ncols=ncols, nrows=nrows)
# for it in range(big_cube_s0.shape[1]):
# jt, kt = it // ncols, it % ncols
# frame = frames[jt][kt]
# frame.imshow(big_cube_s0[:, it, :], origin='lower')
# frame.set(xlim=(2030, 2060))
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files,
mainfitsdir='reduced')
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# set up function name
main_name = __NAME__ + '.main()'
# ------------------------------------------------------------------
# Load first file
# ------------------------------------------------------------------
loc = ParamDict()
rd = spirouImage.ReadImage(p, p['FITSFILENAME'])
loc['DATA'], loc['DATAHDR'], loc['YDIM'], loc['XDIM'] = rd
loc.set_sources(['DATA', 'DATAHDR', 'XDIM', 'YDIM'], main_name)
# ------------------------------------------------------------------
# Get the wave solution
# ------------------------------------------------------------------
wout = spirouImage.GetWaveSolution(p, image=loc['DATA'], hdr=loc['DATAHDR'],
return_wavemap=True,
return_filename=True)
_, loc['WAVE'], loc['WAVEFILE'], _ = wout
loc.set_sources(['WAVE', 'WAVEFILE'], main_name)
# get the wave keys
loc = spirouImage.GetWaveKeys(p, loc, loc['DATAHDR'])
# ------------------------------------------------------------------
# Get and Normalise the blaze
# ------------------------------------------------------------------
p, loc = spirouTelluric.GetNormalizedBlaze(p, loc, loc['DATAHDR'])
# ------------------------------------------------------------------
# Construct convolution kernels
# ------------------------------------------------------------------
loc = spirouTelluric.ConstructConvKernel1(p, loc)
loc = spirouTelluric.ConstructConvKernel2(p, loc, vsini=p['TELLU_VSINI'])
# ------------------------------------------------------------------
# Get molecular telluric lines
# ------------------------------------------------------------------
loc = spirouTelluric.GetMolecularTellLines(p, loc)
# if TAPAS FNAME is not None we generated a new file so should add to tellDB
if loc['TAPAS_FNAME'] is not None:
# add to the telluric database
spirouDB.UpdateDatabaseTellConv(p, loc['TAPAS_FNAME'], loc['DATAHDR'])
# put file in telluDB
spirouDB.PutTelluFile(p, loc['TAPAS_ABSNAME'])
# ------------------------------------------------------------------
# Get master wave solution map
# ------------------------------------------------------------------
# get master wave map
masterwavefile = spirouDB.GetDatabaseMasterWave(p)
# log process
wmsg1 = 'Shifting transmission map on to master wavelength grid'
wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile))
WLOG(p, '', [wmsg1, wmsg2])
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# read master wave map
mout = spirouImage.GetWaveSolution(p, filename=masterwavefile,
return_wavemap=True, quiet=True,
return_header=True, fiber=wave_fiber)
masterwavep, masterwave, masterwaveheader, mwsource = mout
# get wave acqtimes
master_acqtimes = spirouDB.GetTimes(p, masterwaveheader)
# ------------------------------------------------------------------
# Loop around the files
# ------------------------------------------------------------------
# construct extension
tellu_ext = '{0}_{1}.fits'
# get current telluric maps from telluDB
tellu_db_data = spirouDB.GetDatabaseTellMap(p, required=False)
tellu_db_files = tellu_db_data[0]
# storage for valid output files
loc['OUTPUTFILES'] = []
# loop around the files
for basefilename in p['ARG_FILE_NAMES']:
# ------------------------------------------------------------------
# Get absolute path of filename
# ------------------------------------------------------------------
filename = os.path.join(p['ARG_FILE_DIR'], basefilename)
# ------------------------------------------------------------------
# Read obj telluric file and correct for blaze
# ------------------------------------------------------------------
# get image
sp, shdr, _, _ = spirouImage.ReadImage(p, filename)
# divide my blaze
sp = sp / loc['BLAZE']
# ------------------------------------------------------------------
# Get the wave solution
# ------------------------------------------------------------------
wout = spirouImage.GetWaveSolution(p, image=sp, hdr=shdr,
return_wavemap=True,
return_filename=True)
_, loc['WAVE_IT'], loc['WAVEFILE_IT'], _ = wout
loc.set_sources(['WAVE_IT', 'WAVEFILE_IT'], main_name)
# ------------------------------------------------------------------
# Shift data to master wave file
# ------------------------------------------------------------------
# shift map
wargs = [p, sp, loc['WAVE_IT'], masterwave]
sp = spirouTelluric.Wave2Wave(*wargs)
loc['SP'] = np.array(sp)
loc.set_source('SP', main_name)
# ------------------------------------------------------------------
# get output transmission filename
outfile, tag1 = spirouConfig.Constants.TELLU_TRANS_MAP_FILE(p, filename)
outfilename = os.path.basename(outfile)
loc['OUTPUTFILES'].append(outfile)
# if we already have the file skip it
if outfile in tellu_db_files:
wmsg = 'File {0} exists in telluDB, skipping'
WLOG(p, '', wmsg.format(outfilename))
continue
else:
# log processing file
wmsg = 'Processing file {0}'
WLOG(p, '', wmsg.format(outfilename))
# Get object name and airmass
loc['OBJNAME'] = spirouImage.GetObjName(p, shdr)
loc['AIRMASS'] = spirouImage.GetAirmass(p, shdr)
# set source
source = main_name + '+ spirouImage.ReadParams()'
loc.set_sources(['OBJNAME', 'AIRMASS'], source)
# ------------------------------------------------------------------
# Check that basefile is not in blacklist
# ------------------------------------------------------------------
blacklist_check = spirouTelluric.CheckBlackList(loc['OBJNAME'])
if blacklist_check:
# log black list file found
wmsg = 'File {0} is blacklisted (OBJNAME={1}). Skipping'
wargs = [basefilename, loc['OBJNAME']]
WLOG(p, 'warning', wmsg.format(*wargs))
# skip this file
continue
# ------------------------------------------------------------------
# loop around the orders
# ------------------------------------------------------------------
# define storage for the transmission map
transmission_map = np.zeros_like(loc['DATA'])
# define storage for measured rms within expected clean domains
exp_clean_rms = np.zeros(loc['DATA'].shape[0])
# loop around the orders
for order_num in range(loc['DATA'].shape[0]):
# start and end
start = order_num * loc['XDIM']
end = (order_num * loc['XDIM']) + loc['XDIM']
# get this orders combined tapas transmission
trans = loc['TAPAS_ALL_SPECIES'][0, start:end]
# keep track of the pixels that are considered valid for the SED
# determination
mask1 = trans > p['TRANSMISSION_CUT']
mask1 &= np.isfinite(loc['NBLAZE'][order_num, :])
# normalise the spectrum
sp[order_num, :] /= np.nanmedian(sp[order_num, :])
# create a float mask
fmask = np.array(mask1, dtype=float)
# set up an SED to fill
sed = np.ones(loc['XDIM'])
# sigma clip until limit
ww = None
for it in range(p['N_ITER_SED_HOTSTAR']):
# copy the spectrum
sp2 = np.array(sp[order_num, :])
# flag Nans
nanmask = ~np.isfinite(sp2)
# set all NaNs to zero so that it does not propagate when
# we convlve by KER2 - must set sp2[bad] to zero as
# NaN * 0.0 = NaN and we want 0.0!
sp2[nanmask] = 0.0
# trace the invalid points
fmask[nanmask] = 0.0
# multiple by the float mask
sp2 *= fmask
# convolve with the second kernel
sp2b = np.convolve(sp2 / sed, loc['KER2'], mode='same')
# convolve with mask to get weights
ww = np.convolve(fmask, loc['KER2'], mode='same')
# normalise the spectrum by the weights
with warnings.catch_warnings(record=True) as w:
sp2bw = sp2b / ww
# set zero pixels to 1
sp2bw[sp2b == 0] = 1
# recalculate the mask using the deviation from original
with warnings.catch_warnings(record=True) as _:
dev = (sp2bw - sp[order_num, :] / sed)
dev /= np.nanmedian(np.abs(dev))
mask = mask1 * (np.abs(dev) < p['TELLU_SIGMA_DEV'])
# update the SED with the corrected spectrum
sed *= sp2bw
# identify bad pixels
with warnings.catch_warnings(record=True) as _:
bad = (sp[order_num, :] / sed[:] > 1.2)
sed[bad] = np.nan
# debug plot
if p['DRS_PLOT'] and (p['DRS_DEBUG'] > 1) and FORCE_PLOT_ON:
# start non-interactive plot
sPlt.plt.ioff()
# plot the transmission map plot
pargs = [order_num, mask1, sed, trans, sp, ww, outfilename]
sPlt.tellu_trans_map_plot(p, loc, *pargs)
# show and close
sPlt.plt.show()
sPlt.plt.close()
# set all values below a threshold to NaN
sed[ww < p['TELLU_NAN_THRESHOLD']] = np.nan
# save the spectrum (normalised by the SED) to the tranmission map
transmission_map[order_num, :] = sp[order_num, :] / sed
# get expected clean rms
fmaskb = np.array(fmask).astype(bool)
with warnings.catch_warnings(record=True):
zerotrans = np.abs(transmission_map[order_num, fmaskb]-1)
ec_rms = np.nanmedian(zerotrans)
exp_clean_rms[order_num] = ec_rms
# log the rms
wmsg = 'Order {0}: Fractional RMS in telluric free domain = {1:.3f}'
wargs = [order_num, ec_rms]
WLOG(p, '', wmsg.format(*wargs))
# ---------------------------------------------------------------------
# Quality control
# ---------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# get SNR for each order from header
nbo = loc['DATA'].shape[0]
snr_order = p['QC_MK_TELLU_SNR_ORDER']
snr = spirouImage.Read1Dkey(p, shdr, p['kw_E2DS_SNR'][0], nbo)
# check that SNR is high enough
if snr[snr_order] < p['QC_MK_TELLU_SNR_MIN']:
fmsg = 'low SNR in order {0}: ({1:.2f} < {2:.2f})'
fargs = [snr_order, snr[snr_order], p['QC_MK_TELLU_SNR_MIN']]
fail_msg.append(fmsg.format(*fargs))
passed = False
# add to qc header lists
qc_values.append(snr[snr_order])
qc_name_str = 'SNR[{0}]'.format(snr_order)
qc_names.append(qc_name_str)
qc_logic.append('{0} < {1:.2f}'.format(qc_name_str,
p['QC_MK_TELLU_SNR_ORDER']))
qc_pass.append(0)
else:
qc_pass.append(1)
# ----------------------------------------------------------------------
# check that the RMS is not too low
if exp_clean_rms[snr_order] > p['QC_TELLU_CLEAN_RMS_MAX']:
fmsg = ('Expected clean RMS is too high in order {0} '
'({1:.3f} > {2:.3f})')
fargs = [snr_order, exp_clean_rms[snr_order],
p['QC_TELLU_CLEAN_RMS_MAX']]
fail_msg.append(fmsg.format(*fargs))
passed = False
# add to qc header lists
qc_values.append(exp_clean_rms[snr_order])
qc_name_str = 'exp_clean_rms[{0}]'.format(snr_order)
qc_names.append(qc_name_str)
qc_logic.append('{0} > {1:.2f}'.format(qc_name_str,
p['QC_TELLU_CLEAN_RMS_MAX']))
qc_pass.append(0)
else:
qc_pass.append(1)
# ----------------------------------------------------------------------
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info',
'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Save transmission map to file
# ------------------------------------------------------------------
# get raw file name
raw_in_file = os.path.basename(p['FITSFILENAME'])
# copy original keys
hdict = spirouImage.CopyOriginalKeys(loc['DATAHDR'])
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
# set the input files
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'],
value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'],
value=os.path.basename(masterwavefile))
hdict = spirouImage.AddKey(p, hdict, p['KW_WAVESOURCE'],
value=mwsource)
hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'],
dim1name='file',
values=p['ARG_FILE_NAMES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# add wave solution date
hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME1'],
value=master_acqtimes[0])
hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_TIME2'],
value=master_acqtimes[1])
# add wave solution number of orders
hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_ORD_N'],
value=masterwavep.shape[0])
# add wave solution degree of fit
hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_LL_DEG'],
value=masterwavep.shape[1] - 1)
# add wave solution coefficients
hdict = spirouImage.AddKey2DList(p, hdict, p['KW_WAVE_PARAM'],
values=masterwavep)
# write to file
p = spirouImage.WriteImage(p, outfile, transmission_map, hdict)
# ------------------------------------------------------------------
# Generate the absorption map
# ------------------------------------------------------------------
# set up storage for the absorption
abso = np.array(transmission_map)
# set values less than low threshold to low threshold
# set values higher than high threshold to 1
low, high = p['TELLU_ABSO_LOW_THRES'], p['TELLU_ABSO_HIGH_THRES']
with warnings.catch_warnings(record=True) as w:
abso[abso < low] = low
abso[abso > high] = 1.0
# write to loc
loc['RECON_ABSO'] = abso.reshape(np.product(loc['DATA'].shape))
loc.set_source('RECON_ABSO', main_name)
# ------------------------------------------------------------------
# Get molecular absorption
# ------------------------------------------------------------------
loc = spirouTelluric.CalcMolecularAbsorption(p, loc)
# add molecular absorption to file
for it, molecule in enumerate(p['TELLU_ABSORBERS'][1:]):
# get molecule keyword store and key
molkey = '{0}_{1}'.format(p['KW_TELLU_ABSO'][0], molecule.upper())
# add water col
if molecule == 'h2o':
loc['WATERCOL'] = loc[molkey]
# set source
loc.set_source('WATERCOL', main_name)
# ------------------------------------------------------------------
# Add transmission map to telluDB
# ------------------------------------------------------------------
if p['QC']:
# copy tellu file to the telluDB folder
spirouDB.PutTelluFile(p, outfile)
# update the master tellu DB file with transmission map
targs = [p, outfilename, loc['OBJNAME'], loc['AIRMASS'],
loc['WATERCOL']]
spirouDB.UpdateDatabaseTellMap(*targs)
# ----------------------------------------------------------------------
# Optional Absorption maps
# ----------------------------------------------------------------------
if p['TELLU_ABSO_MAPS']:
# ------------------------------------------------------------------
# Generate the absorption map
# ------------------------------------------------------------------
# get number of files
nfiles = len(p['OUTPUTFILES'])
# set up storage for the absorption
abso = np.zeros([nfiles, np.product(loc['DATA'].shape)])
# loop around outputfiles and add them to abso
for it, filename in enumerate(p['OUTPUTFILES']):
# push data into array
data_it, _, _, _ = spirouImage.ReadImage(p, filename)
abso[it, :] = data_it.reshape(np.product(loc['DATA'].shape))
# set values less than low threshold to low threshold
# set values higher than high threshold to 1
low, high = p['TELLU_ABSO_LOW_THRES'], p['TELLU_ABSO_HIGH_THRES']
abso[abso < low] = low
abso[abso > high] = 1.0
# set values less than TELLU_CUT_BLAZE_NORM threshold to NaN
abso[loc['NBLAZE'] < p['TELLU_CUT_BLAZE_NORM']] = np.nan
# reshape data (back to E2DS)
abso_e2ds = abso.reshape(nfiles, loc['YDIM'], loc['XDIM'])
# get file name
abso_map_file, tag2 = spirouConfig.Constants.TELLU_ABSO_MAP_FILE(p)
# get raw file name
raw_in_file = os.path.basename(p['FITSFILENAME'])
# write the map to file
hdict = spirouImage.CopyOriginalKeys(loc['DATAHDR'])
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
# set the input files
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'],
value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBWAVE'],
value=loc['WAVEFILE'])
# write to file
p = spirouImage.WriteImage(p, abso_map_file, abso_e2ds, hdict)
# ------------------------------------------------------------------
# Generate the median and normalized absorption maps
# ------------------------------------------------------------------
# copy the absorption cube
abso2 = np.array(abso)
# log the absorption cube
log_abso = np.log(abso)
# get the threshold from p
threshold = p['TELLU_ABSO_SIG_THRESH']
# calculate the abso_med
abso_med = np.nanmedian(log_abso, axis=0)
# sigma clip around the median
for it in range(p['TELLU_ABSO_SIG_N_ITER']):
# recalculate the abso_med
abso_med = np.nanmedian(log_abso, axis=0)
# loop around each file
for jt in range(nfiles):
# get this iterations row
rowvalue = log_abso[jt, :]
# get the mask of those values above threshold
goodpix = (rowvalue > threshold) & (abso_med > threshold)
# apply the mask of good pixels to work out ratio
part1 = np.nansum(rowvalue[goodpix] * abso_med[goodpix])
part2 = np.nansum(abso_med[goodpix] ** 2)
ratio = part1 / part2
# store normalised absol back on to log_abso
log_abso[jt, :] = log_abso[jt, :] / ratio
# unlog log_abso
abso_out = np.exp(log_abso)
# calculate the median of the log_abso
abso_med_out = np.exp(np.nanmedian(log_abso, axis=0))
# reshape the median
abso_map_n = abso_med_out.reshape(loc['DATA'].shape)
# save the median absorption map to file
abso_med_file, tag3 = spirouConfig.Constants.TELLU_ABSO_MEDIAN_FILE(p)
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
p = spirouImage.WriteImage(p, abso_med_file, abso_med_out, hdict)
# save the normalized absorption map to file
abso_map_file, tag4 = spirouConfig.Constants.TELLU_ABSO_NORM_MAP_FILE(p)
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
p = spirouImage.WriteImage(p, abso_map_file, abso_map_n, hdict)
# ------------------------------------------------------------------
# calculate dv statistic
# ------------------------------------------------------------------
# get the order for dv calculation
dvselect = p['TELLU_ABSO_DV_ORDER']
size = p['TELLU_ABSO_DV_SIZE']
threshold2 = p['TELLU_ABSO_DV_GOOD_THRES']
fitdeg = p['TELLU_ABSO_FIT_DEG']
ydim, xdim = loc['DATA'].shape
# get the start and end points of this order
start, end = xdim * dvselect + size, xdim * dvselect - size + xdim
# get the median for selected order
abso_med2 = np.exp(abso_med[start:end])
# get the dv pixels to extract
dvpixels = np.arange(-np.floor(size / 2), np.ceil(size / 2), 1)
# loop around files
for it, filename in enumerate(p['OUTPUTFILES']):
# storage for the extracted abso ratios for this file
cc = np.zeros(size)
# loop around a box of size="size"
for jt, dv in dvpixels:
# get the start and end position
start = xdim * dvselect + size + dv
end = xdim * dvselect + xdim - size + dv
# get the log abso for this iteration
rowvalue = np.exp(log_abso[it, start:end])
# find the good pixels
goodpix = (rowvalue > threshold2) & (abso_med2 > threshold2)
# get the ratio
part1 = np.nansum(rowvalue[goodpix] * abso_med2[goodpix])
part2 = np.nansum(abso_med2[goodpix] ** 2)
cc[jt] = part1 / part2
# fit the ratio across the points
cfit = nanpolyfit(dvpixels, cc, fitdeg)
# work out the dv pix
dvpix = -0.5 * (cfit[1] / cfit[0])
# log stats
wmsg = 'File: "{0}", dv={1}'
WLOG(p, '', wmsg.format(filename, dvpix))
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files)
# force plotting to 1
p['DRS_PLOT'] = 1
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
data, hdr, nx, ny = spirouImage.ReadImage(p)
# ----------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
data = spirouImage.FixNonPreProcess(p, data, filename=p['FITSFILENAME'])
# ----------------------------------------------------------------------
# Get basic image properties
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# set sigdet and conad keywords (sigdet is changed later)
p['KW_CCD_SIGDET'][1] = p['SIGDET']
p['KW_CCD_CONAD'][1] = p['GAIN']
# now change the value of sigdet if require
if p['IC_EXT_SIGDET'] > 0:
p['SIGDET'] = float(p['IC_EXT_SIGDET'])
# ----------------------------------------------------------------------
# Resize image
# ----------------------------------------------------------------------
# rotate the image and convert from ADU/s to e-
data = spirouImage.ConvertToADU(spirouImage.FlipImage(p, data), p=p)
# convert NaN to zeros
data2 = np.where(~np.isfinite(data), np.zeros_like(data), data)
# resize image
# bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'],
# ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'],
# getshape=False)
# data2 = spirouImage.ResizeImage(data0, **bkwargs)
# log change in data size
# WLOG(p, '', ('Image format changed to '
# '{0}x{1}').format(*data2.shape[::-1]))
# ----------------------------------------------------------------------
# Log the number of dead pixels
# ----------------------------------------------------------------------
# get the number of bad pixels
n_bad_pix = np.nansum(data2 == 0)
n_bad_pix_frac = n_bad_pix * 100 / np.product(data2.shape)
# Log number
wmsg = 'Nb dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
satseuil = 64536.
col = 2100
seuil = 10000
slice0 = 5
plt.ion()
plt.clf()
plt.imshow(data2, origin='lower', clim=(1., seuil))
plt.colorbar()
plt.axis([0, nx, 0, ny])
plt.plot(np.ones(4096) * col, np.arange(4096), c='red')
plt.figure()
plt.clf()
centpart = data2[:, col - slice0:col + slice0]
# centpart = data2[col - slice:col + slice,:]
# weights = np.where((centpart < satseuil) & (centpart > 0), 1, 0.0001)
# y = np.average(centpart, axis=1, weights=weights) ## weighted average
y = np.nanmedian(centpart, axis=1)
# y=average(centpart,axis=1,weights=where((centpart>0),1,0.0001))
# ## weighted average
plt.plot(np.arange(ny), y)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p, outputs=None)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, hcfile=None, fpfiles=None):
"""
cal_SLIT_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_SLIT_spirou.py [night_directory] [files]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
if hcfile is None or fpfiles is None:
names, types = ['hcfile', 'fpfiles'], [str, str]
customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1],
types,
names,
last_multi=True)
else:
customargs = dict(hcfile=hcfile, fpfiles=fpfiles)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsfile='fpfiles')
# ----------------------------------------------------------------------
# Construct reference filename and get fiber type
# ----------------------------------------------------------------------
p, hcfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['HCFILE'])
p, fpfilenames = spirouStartup.MultiFileSetup(p, files=p['FPFILES'])
# set fiber (it doesn't matter with the 2D image but we need this to get
# the lamp type for FPFILES and HCFILES, AB == C
p['FIBER'] = 'AB'
p['FIB_TYP'] = [p['FIBER']]
fsource = __NAME__ + '/main()'
p.set_sources(['FIBER', 'FIB_TYP'], fsource)
# set the hcfilename to the first hcfilenames
fpfitsfilename = fpfilenames[0]
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# add a force plot off
p['PLOT_PER_ORDER'] = PLOT_PER_ORDER
p.set_source('PLOT_PER_ORDER', __NAME__ + '.main()')
# ----------------------------------------------------------------------
# Read FP and HC files
# ----------------------------------------------------------------------
# read and combine all FP files except the first (fpfitsfilename)
rargs = [p, 'add', fpfitsfilename, fpfilenames[1:]]
p, fpdata, fphdr = spirouImage.ReadImageAndCombine(*rargs)
# read first file (hcfitsfilename)
hcdata, hchdr, _, _ = spirouImage.ReadImage(p, hcfitsfilename)
# add data and hdr to loc
loc = ParamDict()
loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr
loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr
# set the source
sources = ['HCDATA', 'HCHDR']
loc.set_sources(sources, 'spirouImage.ReadImageAndCombine()')
sources = ['FPDATA', 'FPHDR']
loc.set_sources(sources, 'spirouImage.ReadImage()')
# ---------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
hcdata = spirouImage.FixNonPreProcess(p, hcdata)
fpdata = spirouImage.FixNonPreProcess(p, fpdata)
# ----------------------------------------------------------------------
# Get basic image properties for reference file
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, fphdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, fphdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, fphdr, name='gain')
# get lamp parameters
p = spirouTHORCA.GetLampParams(p, hchdr)
# ----------------------------------------------------------------------
# Correction of DARK
# ----------------------------------------------------------------------
# p, hcdatac = spirouImage.CorrectForDark(p, hcdata, hchdr)
hcdatac = hcdata
p['DARKFILE'] = 'None'
# p, fpdatac = spirouImage.CorrectForDark(p, fpdata, fphdr)
fpdatac = fpdata
# ----------------------------------------------------------------------
# Resize hc data
# ----------------------------------------------------------------------
# rotate the image and convert from ADU/s to e-
hcdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, hcdatac), p=p)
# convert NaN to zeros
hcdata0 = np.where(~np.isfinite(hcdata), np.zeros_like(hcdata), hcdata)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'],
xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'],
yhigh=p['IC_CCDY_HIGH'],
getshape=False)
hcdata2 = spirouImage.ResizeImage(p, hcdata0, **bkwargs)
# log change in data size
WLOG(p, '', ('HC Image format changed to '
'{0}x{1}').format(*hcdata2.shape))
# ----------------------------------------------------------------------
# Resize fp data
# ----------------------------------------------------------------------
# rotate the image and convert from ADU/s to e-
fpdata = spirouImage.ConvertToE(spirouImage.FlipImage(p, fpdatac), p=p)
# convert NaN to zeros
fpdata0 = np.where(~np.isfinite(fpdata), np.zeros_like(fpdata), fpdata)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'],
xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'],
yhigh=p['IC_CCDY_HIGH'],
getshape=False)
fpdata2 = spirouImage.ResizeImage(p, fpdata0, **bkwargs)
# log change in data size
WLOG(p, '', ('FP Image format changed to '
'{0}x{1}').format(*fpdata2.shape))
# ----------------------------------------------------------------------
# Correct for the BADPIX mask (set all bad pixels to zero)
# ----------------------------------------------------------------------
# p, hcdata2 = spirouImage.CorrectForBadPix(p, hcdata2, hchdr)
# p, fpdata2 = spirouImage.CorrectForBadPix(p, fpdata2, fphdr)
p['BADPFILE'] = 'None'
# save data to loc
loc['HCDATA'] = hcdata2
loc.set_source('HCDATA', __NAME__ + '/main()')
# save data to loc
loc['FPDATA'] = fpdata2
loc.set_source('FPDATA', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Log the number of dead pixels
# ----------------------------------------------------------------------
# get the number of bad pixels
n_bad_pix = np.nansum(hcdata2 <= 0)
n_bad_pix_frac = n_bad_pix * 100 / np.product(hcdata2.shape)
# Log number
wmsg = 'Nb HC dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# ----------------------------------------------------------------------
# Log the number of dead pixels
# ----------------------------------------------------------------------
# get the number of bad pixels
n_bad_pix = np.nansum(fpdata2 <= 0)
n_bad_pix_frac = n_bad_pix * 100 / np.product(fpdata2.shape)
# Log number
wmsg = 'Nb FP dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# ------------------------------------------------------------------
# Get localisation coefficients
# ------------------------------------------------------------------
# original there is a loop but it is not used --> removed
p = spirouImage.FiberParams(p, p['FIBER'], merge=True)
# get localisation fit coefficients
p, loc = spirouLOCOR.GetCoeffs(p, fphdr, loc)
# ------------------------------------------------------------------
# Get master wave solution map
# ------------------------------------------------------------------
# get master wave map
masterwavefile = spirouDB.GetDatabaseMasterWave(p)
# log process
wmsg1 = 'Getting master wavelength grid'
wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile))
WLOG(p, '', [wmsg1, wmsg2])
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# read master wave map
wout = spirouImage.GetWaveSolution(p,
filename=masterwavefile,
return_wavemap=True,
quiet=True,
return_header=True,
fiber=wave_fiber)
loc['MASTERWAVEP'], loc['MASTERWAVE'] = wout[:2]
loc['MASTERWAVEHDR'], loc['WSOURCE'] = wout[2:]
# set sources
wsource = ['MASTERWAVEP', 'MASTERWAVE', 'MASTERWAVEHDR']
loc.set_sources(wsource, 'spirouImage.GetWaveSolution()')
# ----------------------------------------------------------------------
# Read UNe solution
# ----------------------------------------------------------------------
wave_u_ne, amp_u_ne = spirouImage.ReadLineList(p)
loc['LL_LINE'], loc['AMPL_LINE'] = wave_u_ne, amp_u_ne
source = __NAME__ + '.main() + spirouImage.ReadLineList()'
loc.set_sources(['LL_LINE', 'AMPL_LINE'], source)
# ----------------------------------------------------------------------
# Read cavity length file
# ----------------------------------------------------------------------
loc['CAVITY_LEN_COEFFS'] = spirouImage.ReadCavityLength(p)
source = __NAME__ + '.main() + spirouImage.ReadCavityLength()'
loc.set_source('CAVITY_LEN_COEFFS', source)
# ------------------------------------------------------------------
# Calculate shape map
# ------------------------------------------------------------------
loc = spirouImage.GetShapeMap(p, loc)
# ------------------------------------------------------------------
# Plotting
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# plots setup: start interactive plot
sPlt.start_interactive_session(p)
# plot the shape process for one order
sPlt.slit_shape_angle_plot(p, loc)
# end interactive section
sPlt.end_interactive_session(p)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# TODO: Decide on some quality control criteria?
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# add to qc header lists
qc_values.append('None')
qc_names.append('None')
qc_logic.append('None')
qc_pass.append(1)
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Writing DXMAP to file
# ------------------------------------------------------------------
# get the raw tilt file name
raw_shape_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
shapefits, tag = spirouConfig.Constants.SLIT_XSHAPE_FILE(p)
shapefitsname = os.path.basename(shapefits)
# Log that we are saving tilt file
wmsg = 'Saving shape information in file: {0}'
WLOG(p, '', wmsg.format(shapefitsname))
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(fphdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='hcfile',
values=p['HCFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE2'],
dim1name='fpfile',
values=p['FPFILES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# write tilt file to file
p = spirouImage.WriteImage(p, shapefits, loc['DXMAP'], hdict)
# ------------------------------------------------------------------
# Writing sanity check files
# ------------------------------------------------------------------
if p['SHAPE_DEBUG_OUTPUTS']:
# log
WLOG(p, '', 'Saving debug sanity check files')
# construct file names
input_fp_file, tag1 = spirouConfig.Constants.SLIT_SHAPE_IN_FP_FILE(p)
output_fp_file, tag2 = spirouConfig.Constants.SLIT_SHAPE_OUT_FP_FILE(p)
input_hc_file, tag3 = spirouConfig.Constants.SLIT_SHAPE_IN_HC_FILE(p)
output_hc_file, tag4 = spirouConfig.Constants.SLIT_SHAPE_OUT_HC_FILE(p)
overlap_file, tag5 = spirouConfig.Constants.SLIT_SHAPE_OVERLAP_FILE(p)
# write input fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
p = spirouImage.WriteImage(p, input_fp_file, loc['FPDATA'], hdict)
# write output fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
p = spirouImage.WriteImage(p, output_fp_file, loc['FPDATA2'], hdict)
# write input fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
p = spirouImage.WriteImage(p, input_hc_file, loc['HCDATA'], hdict)
# write output fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
p = spirouImage.WriteImage(p, output_hc_file, loc['HCDATA2'], hdict)
# write overlap file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag5)
p = spirouImage.WriteImage(p, overlap_file, loc['ORDER_OVERLAP'],
hdict)
# ----------------------------------------------------------------------
# Move to calibDB and update calibDB
# ----------------------------------------------------------------------
if p['QC']:
keydb = 'SHAPE'
# copy shape file to the calibDB folder
spirouDB.PutCalibFile(p, shapefits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, shapefitsname, fphdr)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, ufile=None, xsize=None, ysize=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# need custom args (to accept full path or wild card
if ufile is None or xsize is None or ysize is None:
pos = [0, 1, 2]
names, types = ['ufile', 'xsize', 'ysize'], [str, int, int]
customargs = spirouStartup.GetCustomFromRuntime(p,
pos,
types,
names,
last_multi=True)
else:
customargs = dict(ufile=ufile, xsize=xsize, ysize=ysize)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p, night_name, customargs=customargs)
# add constants not currently in constants file
# ------------------------------------------------------------------
# Read image file
# ------------------------------------------------------------------
# read the image data
rout = spirouImage.ReadImage(p, filename=p['UFILE'])
image, hdr, nx, ny = rout
# ----------------------------------------------------------------------
# un-Resize image
# ----------------------------------------------------------------------
# create an array of given size
size = np.product([p['YSIZE'], p['XSIZE']])
newimage = np.repeat(np.nan, size).reshape(p['YSIZE'], p['XSIZE'])
# insert image at given pixels
xlow, xhigh = p['IC_CCDX_LOW'], p['IC_CCDX_HIGH']
ylow, yhigh = p['IC_CCDY_LOW'], p['IC_CCDY_HIGH']
newimage[ylow:yhigh, xlow:xhigh] = image
# rotate the image
newimage = spirouImage.FlipImage(p, newimage)
# ------------------------------------------------------------------
# Save rotated image
# ------------------------------------------------------------------
# construct rotated file name
outfits = ufile.replace('.fits', '_old.fits')
outfitsname = os.path.split(outfits)[-1]
# log that we are saving rotated image
WLOG(p, '', 'Saving Rotated Image in ' + outfitsname)
# add keys from original header file
hdict = spirouImage.CopyOriginalKeys(hdr)
# write to file
p = spirouImage.WriteImage(p, outfits, newimage, hdict)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
wmsg = 'Recipe {0} has been successfully completed'
WLOG(p, 'info', wmsg.format(p['PROGRAM']))
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, fpfile=None, hcfiles=None):
"""
cal_wave_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_wave_spirou.py [night_directory] [fpfile] [hcfiles]
:param night_name: string or None, the folder within data reduced directory
containing files (also reduced directory) i.e.
/data/reduced/20170710 would be "20170710" but
/data/reduced/AT5/20180409 is "AT5/20180409"
:param fpfile: string, or None, the FP file to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:param hcfiles: string, list or None, the list of HC files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
if hcfiles is None or fpfile is None:
names, types = ['fpfile', 'hcfiles'], [str, str]
customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1],
types,
names,
last_multi=True)
else:
customargs = dict(hcfiles=hcfiles, fpfile=fpfile)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsdir='reduced',
mainfitsfile='hcfiles')
# ----------------------------------------------------------------------
# Construct reference filename and get fiber type
# ----------------------------------------------------------------------
p, fpfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['FPFILE'])
fiber1 = str(p['FIBER'])
p, hcfilenames = spirouStartup.MultiFileSetup(p, files=p['HCFILES'])
fiber2 = str(p['FIBER'])
# set the hcfilename to the first hcfilenames
hcfitsfilename = hcfilenames[0]
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# ----------------------------------------------------------------------
# Have to check that the fibers match
# ----------------------------------------------------------------------
if fiber1 == fiber2:
p['FIBER'] = fiber1
fsource = __NAME__ + '/main() & spirouStartup.GetFiberType()'
p.set_source('FIBER', fsource)
else:
emsg = 'Fiber not matching for {0} and {1}, should be the same'
eargs = [hcfitsfilename, fpfitsfilename]
WLOG(p, 'error', emsg.format(*eargs))
# set the fiber type
p['FIB_TYP'] = [p['FIBER']]
p.set_source('FIB_TYP', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Read FP and HC files
# ----------------------------------------------------------------------
# read and combine all HC files except the first (fpfitsfilename)
rargs = [p, 'add', hcfitsfilename, hcfilenames[1:]]
p, hcdata, hchdr = spirouImage.ReadImageAndCombine(*rargs)
# read first file (fpfitsfilename)
fpdata, fphdr, _, _ = spirouImage.ReadImage(p, fpfitsfilename)
# add data and hdr to loc
loc = ParamDict()
loc['HCDATA'], loc['HCHDR'], loc['HCCDR'] = hcdata, hchdr, hchdr.comments
loc['FPDATA'], loc['FPHDR'], loc['FPCDR'] = fpdata, fphdr, fphdr.comments
# set the source
sources = ['HCDATA', 'HCHDR', 'HCCDR']
loc.set_sources(sources, 'spirouImage.ReadImageAndCombine()')
sources = ['FPDATA', 'FPHDR', 'FPCDR']
loc.set_sources(sources, 'spirouImage.ReadImage()')
# ----------------------------------------------------------------------
# Get basic image properties for reference file
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hchdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hchdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hchdr, name='gain')
# get acquisition time
p = spirouImage.GetAcqTime(p, hchdr, name='acqtime', kind='julian')
bjdref = p['ACQTIME']
# set sigdet and conad keywords (sigdet is changed later)
p['KW_CCD_SIGDET'][1] = p['SIGDET']
p['KW_CCD_CONAD'][1] = p['GAIN']
# get lamp parameters
p = spirouWAVE2.get_lamp_parameters(p, hchdr)
# get number of orders
# we always get fibre A number because AB is doubled in constants file
loc['NBO'] = p['QC_LOC_NBO_FPALL']['A']
loc.set_source('NBO', __NAME__ + '.main()')
# get number of pixels in x from hcdata size
loc['NBPIX'] = loc['HCDATA'].shape[1]
loc.set_source('NBPIX', __NAME__ + '.main()')
# ----------------------------------------------------------------------
# Read blaze
# ----------------------------------------------------------------------
# get tilts
p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hchdr)
loc.set_source('BLAZE', __NAME__ + '/main() + /spirouImage.ReadBlazeFile')
# make copy of blaze (as it's overwritten later in CCF part)
# TODO is this needed? More sensible to make and set copy in CCF?
loc['BLAZE2'] = np.copy(loc['BLAZE'])
# ----------------------------------------------------------------------
# Read wave solution
# ----------------------------------------------------------------------
# wavelength file; we will use the polynomial terms in its header,
# NOT the pixel values that would need to be interpolated
# set source of wave file
wsource = __NAME__ + '/main() + /spirouImage.GetWaveSolution'
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# get wave image
wout = spirouImage.GetWaveSolution(p,
hdr=hchdr,
return_wavemap=True,
return_filename=True,
fiber=wave_fiber)
loc['WAVEPARAMS'], loc['WAVE_INIT'], loc['WAVEFILE'], loc['WSOURCE'] = wout
loc.set_sources(['WAVE_INIT', 'WAVEFILE', 'WAVEPARAMS', 'WSOURCE'],
wsource)
poly_wave_sol = loc['WAVEPARAMS']
# ----------------------------------------------------------------------
# Check that wave parameters are consistent with "ic_ll_degr_fit"
# ----------------------------------------------------------------------
loc = spirouImage.CheckWaveSolConsistency(p, loc)
# ----------------------------------------------------------------------
# HC wavelength solution
# ----------------------------------------------------------------------
# log that we are running the HC part and the mode
wmsg = 'Now running the HC solution, mode = {0}'
WLOG(p, 'info', wmsg.format(p['WAVE_MODE_HC']))
# get the solution
loc = spirouWAVE2.do_hc_wavesol(p, loc)
# ----------------------------------------------------------------------
# Quality control - HC solution
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# quality control on sigma clip (sig1 > qc_hc_wave_sigma_max
if loc['SIG1'] > p['QC_HC_WAVE_SIGMA_MAX']:
fmsg = 'Sigma too high ({0:.5f} > {1:.5f})'
fail_msg.append(fmsg.format(loc['SIG1'], p['QC_HC_WAVE_SIGMA_MAX']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(loc['SIG1'])
qc_names.append('SIG1 HC')
qc_logic.append('SIG1 > {0:.2f}'.format(p['QC_HC_WAVE_SIGMA_MAX']))
# ----------------------------------------------------------------------
# check the difference between consecutive orders is always positive
# get the differences
wave_diff = loc['WAVE_MAP2'][1:] - loc['WAVE_MAP2'][:-1]
if np.min(wave_diff) < 0:
fmsg = 'Negative wavelength difference between orders'
fail_msg.append(fmsg)
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(np.min(wave_diff))
qc_names.append('MIN WAVE DIFF HC')
qc_logic.append('MIN WAVE DIFF < 0')
# ----------------------------------------------------------------------
# check the difference between consecutive pixels along an order is
# always positive
# loop through the orders
ord_check = np.zeros((loc['NBO']), dtype=bool)
for order in range(loc['NBO']):
oc = np.all(loc['WAVE_MAP2'][order, 1:] > loc['WAVE_MAP2'][order, :-1])
ord_check[order] = oc
# TODO: Melissa Why is this here????
# ord_check[5] = False
if np.all(ord_check):
qc_pass.append(1)
qc_values.append('None')
else:
fmsg = 'Negative wavelength difference along an order'
fail_msg.append(fmsg)
passed = False
qc_pass.append(0)
qc_values.append(np.ndarray.tolist(np.where(~ord_check)[0]))
# add to qc header lists
# vale: array of orders where it fails
qc_names.append('WAVE DIFF ALONG ORDER HC')
qc_logic.append('WAVE DIFF ALONG ORDER < 0')
# ----------------------------------------------------------------------
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ----------------------------------------------------------------------
# log the global stats
# ----------------------------------------------------------------------
# calculate catalog-fit residuals in km/s
res_hc = []
sumres_hc = 0.0
sumres2_hc = 0.0
for order in range(loc['NBO']):
# get HC line wavelengths for the order
order_mask = loc['ORD_T'] == order
hc_x_ord = loc['XGAU_T'][order_mask]
hc_ll_ord = np.polyval(loc['POLY_WAVE_SOL'][order][::-1], hc_x_ord)
hc_ll_cat = loc['WAVE_CATALOG'][order_mask]
hc_ll_diff = hc_ll_ord - hc_ll_cat
res_hc.append(hc_ll_diff * speed_of_light / hc_ll_cat)
sumres_hc += np.nansum(res_hc[order])
sumres2_hc += np.nansum(res_hc[order]**2)
total_lines_hc = len(np.concatenate(res_hc))
final_mean_hc = sumres_hc / total_lines_hc
final_var_hc = (sumres2_hc / total_lines_hc) - (final_mean_hc**2)
wmsg1 = 'On fiber {0} HC fit line statistic:'.format(p['FIBER'])
wargs2 = [
final_mean_hc * 1000.0,
np.sqrt(final_var_hc) * 1000.0, total_lines_hc,
1000.0 * np.sqrt(final_var_hc / total_lines_hc)
]
wmsg2 = ('\tmean={0:.3f}[m/s] rms={1:.1f} {2} HC lines (error on mean '
'value:{3:.4f}[m/s])'.format(*wargs2))
WLOG(p, 'info', [wmsg1, wmsg2])
# ----------------------------------------------------------------------
# Save wave map to file
# ----------------------------------------------------------------------
# TODO single file-naming function? Ask Neil
# get base input filenames
bfilenames = []
for raw_file in p['ARG_FILE_NAMES']:
bfilenames.append(os.path.basename(raw_file))
# get wave filename
wavefits, tag1 = spirouConfig.Constants.WAVE_FILE_EA(p)
wavefitsname = os.path.basename(wavefits)
# log progress
WLOG(p, '', 'Saving wave map to {0}'.format(wavefitsname))
# log progress
wargs = [p['FIBER'], wavefitsname]
wmsg = 'Write wavelength solution for Fiber {0} in {1}'
WLOG(p, '', wmsg.format(*wargs))
# write solution to fitsfilename header
# copy original keys
hdict = spirouImage.CopyOriginalKeys(loc['HCHDR'], loc['HCCDR'])
# set the version
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
# TODO add DRS_DATE and DRS_NOW
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
# set the input files
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BLAZFILE'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# add wave solution date
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_TIME1'],
value=p['MAX_TIME_HUMAN'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_TIME2'],
value=p['MAX_TIME_UNIX'])
hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_CODE'], value=__NAME__)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=loc['WAVEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=loc['WSOURCE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='file',
values=p['ARG_FILE_NAMES'])
# add number of orders
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_ORD_N'],
value=loc['POLY_WAVE_SOL'].shape[0])
# add degree of fit
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_LL_DEG'],
value=loc['POLY_WAVE_SOL'].shape[1] - 1)
# add wave solution
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_WAVE_PARAM'],
values=loc['POLY_WAVE_SOL'])
# write the wave "spectrum"
p = spirouImage.WriteImage(p, wavefits, loc['WAVE_MAP2'], hdict)
# get filename for E2DS calibDB copy of FITSFILENAME
e2dscopy_filename, tag2 = spirouConfig.Constants.WAVE_E2DS_COPY(p)
wargs = [p['FIBER'], os.path.split(e2dscopy_filename)[-1]]
wmsg = 'Write reference E2DS spectra for Fiber {0} in {1}'
WLOG(p, '', wmsg.format(*wargs))
# make a copy of the E2DS file for the calibBD
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
p = spirouImage.WriteImage(p, e2dscopy_filename, loc['HCDATA'], hdict)
# ----------------------------------------------------------------------
# Save resolution and line profiles to file
# ----------------------------------------------------------------------
raw_infile = os.path.basename(p['FITSFILENAME'])
# get wave filename
resfits, tag3 = spirouConfig.Constants.WAVE_RES_FILE_EA(p)
resfitsname = os.path.basename(resfits)
WLOG(p, '', 'Saving wave resmap to {0}'.format(resfitsname))
# make a copy of the E2DS file for the calibBD
# set the version
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
# TODO add DRS_DATE and DRS_NOW
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
# get res data in correct format
resdata, hdicts = spirouWAVE2.generate_res_files(p, loc, hdict)
# save to file
p = spirouImage.WriteImageMulti(p, resfits, resdata, hdicts=hdicts)
# ----------------------------------------------------------------------
# Update calibDB
# ----------------------------------------------------------------------
if p['QC']:
# set the wave key
keydb = 'WAVE_{0}'.format(p['FIBER'])
# copy wave file to calibDB folder
spirouDB.PutCalibFile(p, wavefits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, wavefitsname, loc['HCHDR'])
# set the hcref key
keydb = 'HCREF_{0}'.format(p['FIBER'])
# copy wave file to calibDB folder
spirouDB.PutCalibFile(p, e2dscopy_filename)
# update the master calib DB file with new key
e2dscopyfits = os.path.split(e2dscopy_filename)[-1]
spirouDB.UpdateCalibMaster(p, keydb, e2dscopyfits, loc['HCHDR'])
# ----------------------------------------------------------------------
# Update header of current files
# ----------------------------------------------------------------------
# only copy over if QC passed
if p['QC']:
rdir = os.path.dirname(wavefits)
# loop around hc files and update header with
for rawhcfile in p['ARG_FILE_NAMES']:
hcfile = os.path.join(rdir, rawhcfile)
raw_infilepath1 = os.path.join(p['ARG_FILE_DIR'], hcfile)
p = spirouImage.UpdateWaveSolutionHC(p, loc, raw_infilepath1)
# ----------------------------------------------------------------------
# HC+FP wavelength solution
# ----------------------------------------------------------------------
# check if there's a FP input and if HC solution passed QCs
if has_fp and p['QC']:
# log that we are doing the FP solution
wmsg = 'Now running the combined FP-HC solution, mode = {}'
WLOG(p, 'info', wmsg.format(p['WAVE_MODE_FP']))
# do the wavelength solution
loc = spirouWAVE2.do_fp_wavesol(p, loc)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# get parameters ffrom p
p['QC_RMS_LITTROW_MAX'] = p['QC_HC_RMS_LITTROW_MAX']
p['QC_DEV_LITTROW_MAX'] = p['QC_HC_DEV_LITTROW_MAX']
# set passed variable and fail message list
# passed, fail_msg = True, []
# qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# check the difference between consecutive orders is always positive
# get the differences
wave_diff = loc['LL_FINAL'][1:] - loc['LL_FINAL'][:-1]
if np.min(wave_diff) < 0:
fmsg = 'Negative wavelength difference between orders'
fail_msg.append(fmsg)
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(np.min(wave_diff))
qc_names.append('MIN WAVE DIFF FP-HC')
qc_logic.append('MIN WAVE DIFF < 0')
# ----------------------------------------------------------------------
# check for infinites and NaNs in mean residuals from fit
if ~np.isfinite(loc['X_MEAN_2']):
# add failed message to the fail message list
fmsg = 'NaN or Inf in X_MEAN_2'
fail_msg.append(fmsg)
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(loc['X_MEAN_2'])
qc_names.append('X_MEAN_2')
qc_logic.append('X_MEAN_2 not finite')
# ----------------------------------------------------------------------
# iterate through Littrow test cut values
lit_it = 2
# checks every other value
# TODO: This QC check (or set of QC checks needs re-writing it is
# TODO: nearly impossible to understand
for x_it in range(1, len(loc['X_CUT_POINTS_' + str(lit_it)]), 2):
# get x cut point
x_cut_point = loc['X_CUT_POINTS_' + str(lit_it)][x_it]
# get the sigma for this cut point
sig_littrow = loc['LITTROW_SIG_' + str(lit_it)][x_it]
# get the abs min and max dev littrow values
min_littrow = abs(loc['LITTROW_MINDEV_' + str(lit_it)][x_it])
max_littrow = abs(loc['LITTROW_MAXDEV_' + str(lit_it)][x_it])
# get the corresponding order
min_littrow_ord = loc['LITTROW_MINDEVORD_' + str(lit_it)][x_it]
max_littrow_ord = loc['LITTROW_MAXDEVORD_' + str(lit_it)][x_it]
# check if sig littrow is above maximum
rms_littrow_max = p['QC_RMS_LITTROW_MAX']
dev_littrow_max = p['QC_DEV_LITTROW_MAX']
if sig_littrow > rms_littrow_max:
fmsg = ('Littrow test (x={0}) failed (sig littrow = '
'{1:.2f} > {2:.2f})')
fargs = [x_cut_point, sig_littrow, rms_littrow_max]
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(sig_littrow)
qc_names.append('sig_littrow')
qc_logic.append('sig_littrow > {0:.2f}'.format(rms_littrow_max))
# ----------------------------------------------------------------------
# check if min/max littrow is out of bounds
if np.max([max_littrow, min_littrow]) > dev_littrow_max:
fmsg = ('Littrow test (x={0}) failed (min|max dev = '
'{1:.2f}|{2:.2f} > {3:.2f} for order {4}|{5})')
fargs = [
x_cut_point, min_littrow, max_littrow, dev_littrow_max,
min_littrow_ord, max_littrow_ord
]
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
# TODO: Should this be the QC header values?
# TODO: it does not change the outcome of QC (i.e. passed=False)
# TODO: So what is the point?
# if sig was out of bounds, recalculate
if sig_littrow > rms_littrow_max:
# conditions
check1 = min_littrow > dev_littrow_max
check2 = max_littrow > dev_littrow_max
# get the residuals
respix = loc['LITTROW_YY_' + str(lit_it)][x_it]
# check if both are out of bounds
if check1 and check2:
# remove respective orders
worst_order = (min_littrow_ord, max_littrow_ord)
respix_2 = np.delete(respix, worst_order)
redo_sigma = True
# check if min is out of bounds
elif check1:
# remove respective order
worst_order = min_littrow_ord
respix_2 = np.delete(respix, worst_order)
redo_sigma = True
# check if max is out of bounds
elif check2:
# remove respective order
worst_order = max_littrow_ord
respix_2 = np.delete(respix, max_littrow_ord)
redo_sigma = True
# else do not recalculate sigma
else:
redo_sigma, respix_2, worst_order = False, None, None
wmsg = 'No outlying orders, sig littrow not recalculated'
fail_msg.append(wmsg.format())
# if outlying order, recalculate stats
if redo_sigma:
mean = np.nansum(respix_2) / len(respix_2)
mean2 = np.nansum(respix_2**2) / len(respix_2)
rms = np.sqrt(mean2 - mean**2)
if rms > rms_littrow_max:
fmsg = (
'Littrow test (x={0}) failed (sig littrow = '
'{1:.2f} > {2:.2f} removing order {3})')
fargs = [
x_cut_point, rms, rms_littrow_max, worst_order
]
fail_msg.append(fmsg.format(*fargs))
else:
wargs = [
x_cut_point, rms, rms_littrow_max, worst_order
]
wmsg = (
'Littrow test (x={0}) passed (sig littrow = '
'{1:.2f} > {2:.2f} removing order {3})')
fail_msg.append(wmsg.format(*wargs))
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(np.max([max_littrow, min_littrow]))
qc_names.append('max or min littrow')
qc_logic.append('max or min littrow > {0:.2f}'
''.format(dev_littrow_max))
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# archive result in e2ds spectra
# ------------------------------------------------------------------
# get raw input file name(s)
raw_infiles1 = []
for hcfile in p['HCFILES']:
raw_infiles1.append(os.path.basename(hcfile))
raw_infile2 = os.path.basename(p['FPFILE'])
# get wave filename
wavefits, tag1 = spirouConfig.Constants.WAVE_FILE_EA_2(p)
wavefitsname = os.path.split(wavefits)[-1]
# log progress
wargs = [p['FIBER'], wavefits]
wmsg = 'Write wavelength solution for Fiber {0} in {1}'
WLOG(p, '', wmsg.format(*wargs))
# write solution to fitsfilename header
# copy original keys
hdict = spirouImage.CopyOriginalKeys(loc['HCHDR'], loc['HCCDR'])
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
# set the input files
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBBAD'],
value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=loc['WAVEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=loc['WSOURCE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='fpfile',
values=p['FPFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE2'],
dim1name='hcfile',
values=p['HCFILES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# add wave solution date
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_TIME1'],
value=p['MAX_TIME_HUMAN'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_TIME2'],
value=p['MAX_TIME_UNIX'])
hdict = spirouImage.AddKey(p, hdict, p['KW_WAVE_CODE'], value=__NAME__)
# add number of orders
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_ORD_N'],
value=loc['LL_PARAM_FINAL'].shape[0])
# add degree of fit
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_LL_DEG'],
value=loc['LL_PARAM_FINAL'].shape[1] - 1)
# add wave solution
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_WAVE_PARAM'],
values=loc['LL_PARAM_FINAL'])
# add FP CCF drift
# target RV and width
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_TARG_RV'],
value=p['TARGET_RV'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_WIDTH'],
value=p['CCF_WIDTH'])
# the rv step
# rvstep = np.abs(loc['RV_CCF'][0] - loc['RV_CCF'][1])
# hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CDELT'], value=rvstep)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_STEP'],
value=p['CCF_STEP'])
# add ccf stats
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_DRIFT'],
value=loc['CCF_RES'][1])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_FWHM'],
value=loc['FWHM'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_CONTRAST'],
value=loc['CONTRAST'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_MAXCPP'],
value=loc['MAXCPP'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_MASK'],
value=p['CCF_MASK'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_LINES'],
value=np.nansum(loc['TOT_LINE']))
# write the wave "spectrum"
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
p = spirouImage.WriteImage(p, wavefits, loc['LL_FINAL'], hdict)
# get filename for E2DS calibDB copy of FITSFILENAME
e2dscopy_filename = spirouConfig.Constants.WAVE_E2DS_COPY(p)[0]
wargs = [p['FIBER'], os.path.split(e2dscopy_filename)[-1]]
wmsg = 'Write reference E2DS spectra for Fiber {0} in {1}'
WLOG(p, '', wmsg.format(*wargs))
# make a copy of the E2DS file for the calibBD
p = spirouImage.WriteImage(p, e2dscopy_filename, loc['HCDATA'], hdict)
# only copy over if QC passed
if p['QC']:
# loop around hc files and update header with
for hcfile in p['HCFILES']:
raw_infilepath1 = os.path.join(p['ARG_FILE_DIR'], hcfile)
p = spirouImage.UpdateWaveSolution(p, loc, raw_infilepath1)
# update fp file
raw_infilepath2 = os.path.join(p['ARG_FILE_DIR'], raw_infile2)
p = spirouImage.UpdateWaveSolution(p, loc, raw_infilepath2)
# ------------------------------------------------------------------
# Save to result table
# ------------------------------------------------------------------
# calculate stats for table
final_mean = 1000 * loc['X_MEAN_2']
final_var = 1000 * loc['X_VAR_2']
num_lines = loc['TOTAL_LINES_2']
err = 1000 * np.sqrt(loc['X_VAR_2'] / num_lines)
sig_littrow = 1000 * np.array(loc['LITTROW_SIG_' + str(lit_it)])
# construct filename
wavetbl = spirouConfig.Constants.WAVE_TBL_FILE_EA(p)
wavetblname = os.path.basename(wavetbl)
# construct and write table
columnnames = [
'night_name', 'file_name', 'fiber', 'mean', 'rms', 'N_lines',
'err', 'rms_L500', 'rms_L1000', 'rms_L1500', 'rms_L2000',
'rms_L2500', 'rms_L3000', 'rms_L3500'
]
columnformats = [
'{:20s}', '{:30s}', '{:3s}', '{:7.4f}', '{:6.2f}', '{:3d}',
'{:6.3f}', '{:6.2f}', '{:6.2f}', '{:6.2f}', '{:6.2f}', '{:6.2f}',
'{:6.2f}', '{:6.2f}'
]
columnvalues = [[p['ARG_NIGHT_NAME']], [p['ARG_FILE_NAMES'][0]],
[p['FIBER']], [final_mean], [final_var], [num_lines],
[err], [sig_littrow[0]], [sig_littrow[1]],
[sig_littrow[2]], [sig_littrow[3]], [sig_littrow[4]],
[sig_littrow[5]], [sig_littrow[6]]]
# make table
table = spirouImage.MakeTable(p,
columns=columnnames,
values=columnvalues,
formats=columnformats)
# merge table
wmsg = 'Global result summary saved in {0}'
WLOG(p, '', wmsg.format(wavetblname))
spirouImage.MergeTable(p, table, wavetbl, fmt='ascii.rst')
# ------------------------------------------------------------------
# Save line list table file
# ------------------------------------------------------------------
# construct filename
# TODO proper column values
wavelltbl = spirouConfig.Constants.WAVE_LINE_FILE_EA(p)
wavelltblname = os.path.split(wavelltbl)[-1]
# construct and write table
columnnames = ['order', 'll', 'dv', 'w', 'xi', 'xo', 'dvdx']
columnformats = [
'{:.0f}', '{:12.4f}', '{:13.5f}', '{:12.4f}', '{:12.4f}',
'{:12.4f}', '{:8.4f}'
]
columnvalues = []
# construct column values (flatten over orders)
for it in range(len(loc['X_DETAILS_2'])):
for jt in range(len(loc['X_DETAILS_2'][it][0])):
row = [
float(it), loc['X_DETAILS_2'][it][0][jt],
loc['LL_DETAILS_2'][it][0][jt],
loc['X_DETAILS_2'][it][3][jt],
loc['X_DETAILS_2'][it][1][jt],
loc['X_DETAILS_2'][it][2][jt], loc['SCALE_2'][it][jt]
]
columnvalues.append(row)
# log saving
wmsg = 'List of lines used saved in {0}'
WLOG(p, '', wmsg.format(wavelltblname))
# make table
columnvalues = np.array(columnvalues).T
table = spirouImage.MakeTable(p,
columns=columnnames,
values=columnvalues,
formats=columnformats)
# write table
spirouImage.WriteTable(p, table, wavelltbl, fmt='ascii.rst')
# ------------------------------------------------------------------
# Move to calibDB and update calibDB
# ------------------------------------------------------------------
if p['QC']:
# set the wave key
keydb = 'WAVE_{0}'.format(p['FIBER'])
# copy wave file to calibDB folder
spirouDB.PutCalibFile(p, wavefits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, wavefitsname, loc['HCHDR'])
# set the hcref key
keydb = 'HCREF_{0}'.format(p['FIBER'])
# copy wave file to calibDB folder
spirouDB.PutCalibFile(p, e2dscopy_filename)
# update the master calib DB file with new key
e2dscopyfits = os.path.split(e2dscopy_filename)[-1]
spirouDB.UpdateCalibMaster(p, keydb, e2dscopyfits, loc['HCHDR'])
# If the HC solution failed QCs we do not compute FP-HC solution
elif has_fp and not p['QC']:
wmsg = 'HC solution failed quality controls; FP not processed'
WLOG(p, 'warning', wmsg)
# If there is no FP file we log that
elif not has_fp:
wmsg = 'No FP file given; FP-HC combined solution cannot be generated'
WLOG(p, 'warning', wmsg)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return p and loc
return dict(locals())