def main(night_name=None, files=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files)
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# ----------------------------------------------------------------------
# Section 1
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# Section 2
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ...
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
wmsg = 'Recipe {0} has been successfully completed'
WLOG(p, 'info', wmsg.format(p['PROGRAM']))
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, key=None, filename=None):
"""
Manually add a file the the calibDB with "key"
i.e. adds
key night_name filename human-date unix-time
to the calibDB and copies "filename" from .../reduced_dir/night_name/ into
the calibDB
Note filename must be in .../reduced_dir/night_name/
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# deal with arguments being None (i.e. get from sys.argv)
pos = [0, 1]
fmt = [str, str]
names = ['key', 'filename']
call = [key, filename]
# now get custom arguments
customargs = spirouStartup.GetCustomFromRuntime(p,
pos,
fmt,
names,
calls=call)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsfile='filename',
mainfitsdir='reduced')
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
image, hdr, nbo, nx = spirouImage.ReadData(p, p['FITSFILENAME'])
# ----------------------------------------------------------------------
# Move to calibDB and update calibDB
# ----------------------------------------------------------------------
# set dark key
keydb = p['KEY']
# copy dark fits file to the calibDB folder
spirouDB.PutCalibFile(p, p['FITSFILENAME'])
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, p['FILENAME'], hdr)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(return_locals=False, warn=True, log=True, called=False):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__, quiet=log)
# set log_opt
p['LOG_OPT'] = __NAME__.split('.py')[0]
p.set_source('LOG_OPT', __NAME__ + '.main()')
# set DB_MAX_WAIT
p['DB_MAX_WAIT'] = 3600
# ----------------------------------------------------------------------
# Perform resets
# ----------------------------------------------------------------------
reset1, reset2, reset3 = True, True, True
reset4, reset5, reset6 = True, True, True
if warn:
reset1 = reset_confirmation(p, 'Tmp', called=called)
if reset1:
reset_tmp_folders(p, log)
else:
WLOG(p, '', 'Not resetting tmp folders.')
if warn:
reset2 = reset_confirmation(p, 'Reduced', called=called)
if reset2:
reset_reduced_folders(p, log)
else:
WLOG(p, '', 'Not resetting reduced folders.')
if warn:
reset3 = reset_confirmation(p, 'CalibDB', called=called)
if reset3:
reset_calibdb(p, log)
else:
WLOG(p, '', 'Not resetting CalibDB files.')
if warn:
reset4 = reset_confirmation(p, 'TelluDB', called=called)
if reset4:
reset_telludb(p, log)
else:
WLOG(p, '', 'Not resetting TelluDB files.')
if warn:
reset5 = reset_confirmation(p, 'Log', called=called)
if reset5:
reset_log(p)
if warn:
reset6 = reset_confirmation(p, 'Plot', called=called)
if reset6:
reset_plot(p)
else:
WLOG(p, '', 'Not resetting Log files.')
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
if log:
wmsg = 'Recipe {0} has been successfully completed'
WLOG(p, 'info', wmsg.format(__NAME__))
# return a copy of locally defined variables in the memory
if return_locals:
return dict(locals())
def main(night_name=None, oldfile=None, newfile=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# deal with arguments being None (i.e. get from sys.argv)
pos = [0, 1]
fmt = [str, str]
names = ['oldfile', 'newfile']
call = [oldfile, newfile]
# now get custom arguments
customargs = spirouStartup.GetCustomFromRuntime(p,
pos,
fmt,
names,
calls=call)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsfile='newfile')
# ----------------------------------------------------------------------
# Get files
# ----------------------------------------------------------------------
# check that path exists
emsg = '{0} file = {1} does not exist'
if not os.path.exists(p['OLDFILE']):
WLOG(p, 'error', emsg.format('old', p['OLDFILE']))
# check that paths exists
if not os.path.exists(p['NEWFILE']):
WLOG(p, 'error', emsg.format('new', p['NEWFILE']))
# load files
data1, hdr1, _, _ = spirouImage.ReadImage(p, filename=oldfile)
data2, hdr2, _, _ = spirouImage.ReadImage(p, filename=newfile)
# ----------------------------------------------------------------------
# Do difference image
# ----------------------------------------------------------------------
diff_image(p, data1, data2, 'old image', 'new image', scale=(1, 99))
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
wmsg = 'Recipe {0} has been successfully completed'
WLOG(p, 'info', wmsg.format(p['PROGRAM']))
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, reffile=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# deal with reference file being None (i.e. get from sys.argv)
if reffile is None:
customargs = spirouStartup.GetCustomFromRuntime(p, [0], [str],
['reffile'])
else:
customargs = dict(reffile=reffile)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p, night_name, customargs=customargs,
mainfitsfile='reffile',
mainfitsdir='reduced')
# ----------------------------------------------------------------------
# Construct reference filename and get fiber type
# ----------------------------------------------------------------------
p, reffilename = spirouStartup.SingleFileSetup(p, filename=p['REFFILE'],
skipcheck=True)
p['REFFILENAME'] = reffilename
p.set_source('REFFILENAME', __NAME__ + '.main()')
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
data, hdr, nbo, nx = spirouImage.ReadData(p, p['REFFILENAME'])
# ----------------------------------------------------------------------
# Add keys and save file
# ----------------------------------------------------------------------
newfilename = p['REFFILE'].replace('.fits', '_edit.fits')
newpath = os.path.join(p['ARG_FILE_DIR'], newfilename)
# add keys from original header file
hdict = spirouImage.CopyOriginalKeys(hdr)
# set the version
hdict = spirouImage.AddKey(p, hdict, HEADER_KEY, value=HEADER_VALUE)
# log saving
wmsg = 'Writing file {0} to {1}'
WLOG(p, '', wmsg.format(newfilename, p['ARG_FILE_DIR']))
# save drift values
p = spirouImage.WriteImage(p, newpath, data, hdict)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
"""
cal_DARK_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_DARK_spirou.py [night_directory] [fitsfilename]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files)
p = spirouStartup.InitialFileSetup(p)
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='average')
# ----------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
data = spirouImage.FixNonPreProcess(p, data)
# ----------------------------------------------------------------------
# Find the amplitude to use for the local background
# ----------------------------------------------------------------------
spirouBACK.MakeLocalBackgroundMap(p, data)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p, outputs=None)
# return a copy of locally defined variables in the memory
return dict(locals())
def set_up():
"""
Load the parameter dictionary (i.e. the recipe setup)
:return p: ParamDict, the parameter dictionary with all constants from
config files loaded in
"""
# Set up
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# need custom args (to accept full path or wild card
args, kwargs = [p, 'ICDP_NAME'], dict(required=True, logthis=False)
p = spirouStartup.LoadOtherConfig(*args, **kwargs)
p['RECIPE'] = __NAME__
p['LOG_OPT'], p['PROGRAM'] = p['RECIPE'], p['RECIPE']
# return p
return p
def main(return_locals=False):
# ----------------------------------------------------------------------
# get p
p = spirouStartup.Begin('None')
p['RECIPE'] = __NAME__
# title
spirouStartup.DisplayTitle(p, ' * DRS Dependencies')
# list the version of python found
spirouStartup.DisplaySysInfo(p, logonly=False)
# get all python files
WLOG(p, '', 'Getting python files')
python_files = get_python_files(PATH)
# get all import statements
WLOG(p, '', 'Getting import statements')
rimports, stats, info = get_import_statements(p, python_files)
# clean imports
imports = clean_imports(rimports)
# get versions
versions = get_current_versions(imports)
# print total number of lines
WLOG(p, '', '=' * 50)
WLOG(p, '', 'Stats:')
WLOG(p, '', '=' * 50)
for stat in stats:
WLOG(p, '', '\t{0}: {1}'.format(stat, stats[stat]))
# print import statements
WLOG(p, '', '=' * 50)
WLOG(p, '', 'Import statements found are:')
WLOG(p, '', '=' * 50)
for it in range(len(imports)):
args = [imports[it], versions[it]]
if versions[it] is not None:
WLOG(p, '', '\t{0: <16}({1})'.format(*args))
else:
WLOG(p, '', '\t{0}'.format(*args))
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
wmsg = 'Recipe {0} has been successfully completed'
WLOG(p, 'info', wmsg.format(p['RECIPE']))
# return a copy of locally defined variables in the memory
if return_locals:
return dict(locals())
def main():
# ----------------------------------------------------------------------
# Run Reset
# ----------------------------------------------------------------------
# get constants
p = spirouStartup.Begin(__NAME__)
# log run
WLOG(p, 'info', 'Now Running reset script.')
# run DRS reset
spirouTools.DRS_Reset(log=True, called=True)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p, outputs=None)
# return a copy of locally defined variables in the memory
return dict(locals())
def display_calibdb(max_time=None):
# load other config
p = spirouStartup.Begin(recipe=__NAME__, quiet=True)
# add custom args straight to p
p = spirouStartup.LoadMinimum(p, customargs={})
# set path
path = os.path.join(p['DRS_CALIB_DB'], p['IC_CALIBDB_FILENAME'])
# get master calib db file
if max_time is None:
# log master calib db file
wmsg = 'CalibDB located at: {0}'.format(path)
WLOG(p, '', wmsg)
# get and check the lock file
lock, lock_file = spirouCDB.get_check_lock_file(p)
# try to open the master file
lines = spirouCDB.read_master_file(p, lock, lock_file)
# print master calibDB
for line in lines:
print('', line.replace('\n', ''))
# close and remove lock file
lock.close()
os.remove(lock_file)
else:
# log master calib db file
args = [path, max_time, p['CALIB_DB_MATCH']]
wmsgs = [
'CalibDB located at: {0}'.format(*args),
' max_time={1} method={2}'.format(*args)
]
WLOG(p, '', wmsgs)
# get database at max_time
cdb, p = spirouCDB.get_database(p, max_time)
# loop around entries
for entry in cdb:
values = [entry] + cdb[entry]
print('\t{0} {1} {2} {3} {4}'.format(*values))
def main(night_name=None, reffile=None):
"""
cal_DRIFTPEAK_E2DS_spirou.py main function, if arguments are None uses
arguments from run time i.e.:
cal_DRIFTPEAK_E2DS_spirou.py [night_directory] [reffile]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param reffile: string, the reference file to use
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# deal with reference file being None (i.e. get from sys.argv)
if reffile is None:
customargs = spirouStartup.GetCustomFromRuntime(
p, [0], [str], ['reffile'])
else:
customargs = dict(reffile=reffile)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsfile='reffile',
mainfitsdir='reduced')
# ----------------------------------------------------------------------
# Construct reference filename and get fiber type
# ----------------------------------------------------------------------
p, reffilename = spirouStartup.SingleFileSetup(p, filename=p['REFFILE'])
p['REFFILENAME'] = reffilename
p.set_source('REFFILENAME', __NAME__ + '.main()')
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
speref, hdr, nbo, nx = spirouImage.ReadData(p, p['REFFILENAME'])
# add to loc
loc = ParamDict()
loc['SPEREF'] = speref
loc['NUMBER_ORDERS'] = nbo
loc.set_sources(['SPEREF', 'NUMBER_ORDERS'], __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Get lamp type
# ----------------------------------------------------------------------
# get lamp type
if p['KW_EXT_TYPE'][0] in hdr:
ext_type = hdr[p['KW_EXT_TYPE'][0]]
drift_types = p['DRIFT_PEAK_ALLOWED_TYPES'].keys()
found = False
for kind in drift_types:
if ext_type == kind:
loc['LAMP'] = p['DRIFT_PEAK_ALLOWED_TYPES'][kind]
found = True
if not found:
eargs1 = [p['KW_EXT_TYPE'][0], ' or '.join(drift_types)]
emsg1 = (
'Wrong type of image for Drift, header key "{0}" should be'
'{1}'.format(*eargs1))
emsg2 = '\tPlease check DRIFT_PEAK_ALLOWED_TYPES'
WLOG(p, 'error', [emsg1, emsg2])
else:
emsg = 'Header key = "{0}" missing from file {1}'
eargs = [p['KW_EXT_TYPE'][0], p['REFFILENAME']]
WLOG(p, 'error', emsg.format(*eargs))
loc.set_source('LAMP', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Get basic image properties for reference file
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# get acquisition time
p = spirouImage.GetAcqTime(p, hdr, name='acqtime', kind='julian')
bjdref = p['ACQTIME']
# set sigdet and conad keywords (sigdet is changed later)
p['KW_CCD_SIGDET'][1] = p['SIGDET']
p['KW_CCD_CONAD'][1] = p['GAIN']
# ----------------------------------------------------------------------
# Read wavelength solution
# ----------------------------------------------------------------------
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# get wave image
source = __NAME__ + '/main() + /spirouImage.GetWaveSolution'
wout = spirouImage.GetWaveSolution(p,
hdr=hdr,
return_wavemap=True,
return_filename=True,
fiber=wave_fiber)
_, loc['WAVE'], loc['WAVEFILE'], loc['WSOURCE'] = wout
loc.set_sources(['WAVE', 'WAVEFILE', 'WSOURCE'], source)
# ----------------------------------------------------------------------
# Read Flat file
# ----------------------------------------------------------------------
# get flat
p, loc['FLAT'] = spirouImage.ReadFlatFile(p, hdr)
loc.set_source('FLAT', __NAME__ + '/main() + /spirouImage.ReadFlatFile')
# get all values in flat that are zero to 1
loc['FLAT'] = np.where(loc['FLAT'] == 0, 1.0, loc['FLAT'])
# correct for flat file
loc['SPEREF'] = loc['SPEREF'] / loc['FLAT']
# ----------------------------------------------------------------------
# Background correction
# ----------------------------------------------------------------------
# test whether we want to subtract background
if p['IC_DRIFT_BACK_CORR']:
# Loop around the orders
for order_num in range(loc['NUMBER_ORDERS']):
# get the box size from constants
bsize = p['DRIFT_PEAK_MINMAX_BOXSIZE']
# Measurethe min and max flux
miny, maxy = spirouBACK.MeasureMinMax(loc['SPEREF'][order_num],
bsize)
# subtract off the background (miny)
loc['SPEREF'][order_num] = loc['SPEREF'][order_num] - miny
# ----------------------------------------------------------------------
# Identify FP peaks in reference file
# ----------------------------------------------------------------------
# log that we are identifying peaks
wmsg = 'Identification of lines in reference file: {0}'
WLOG(p, '', wmsg.format(p['REFFILE']))
# get the position of FP peaks from reference file
loc = spirouRV.CreateDriftFile(p, loc)
# ----------------------------------------------------------------------
# Removal of suspiciously wide FP lines
# ----------------------------------------------------------------------
loc = spirouRV.RemoveWidePeaks(p, loc)
# ----------------------------------------------------------------------
# Get reference drift
# ----------------------------------------------------------------------
# are we using gaussfit?
gaussfit = p['DRIFT_PEAK_GETDRIFT_GAUSSFIT']
# get drift
loc['XREF'] = spirouRV.GetDrift(p,
loc['SPEREF'],
loc['ORDPEAK'],
loc['XPEAK'],
gaussfit=gaussfit)
loc.set_source('XREF', __NAME__ + '/main()')
# remove any drifts that are zero (i.e. peak not measured
loc = spirouRV.RemoveZeroPeaks(p, loc)
# ------------------------------------------------------------------
# Reference plots
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# start interactive session if needed
sPlt.start_interactive_session(p)
# plot FP spectral order
sPlt.drift_plot_selected_wave_ref(p, loc)
# ------------------------------------------------------------------
# Get all other files that match kw_OUTPUT and kw_EXT_TYPE from
# ref file
# ------------------------------------------------------------------
# get files
listfiles, listtypes = spirouImage.GetSimilarDriftFiles(p, hdr)
# get the number of files
nfiles = len(listfiles)
# Log the number of files found
wmsgs = [
'Number of files found on directory = {0}'.format(nfiles),
'\tExtensions allowed:'
]
for listtype in listtypes:
wmsgs.append('\t\t - {0}'.format(listtype))
WLOG(p, 'info', wmsgs)
# ------------------------------------------------------------------
# Set up Extract storage for all files
# ------------------------------------------------------------------
# decide whether we need to skip (for large number of files)
if len(listfiles) >= p['DRIFT_NLARGE']:
skip = p['DRIFT_PEAK_FILE_SKIP']
nfiles = int(nfiles / skip)
else:
skip = 1
# set up storage
loc['DRIFT'] = np.zeros((nfiles, loc['NUMBER_ORDERS']))
loc['DRIFT_LEFT'] = np.zeros((nfiles, loc['NUMBER_ORDERS']))
loc['DRIFT_RIGHT'] = np.zeros((nfiles, loc['NUMBER_ORDERS']))
loc['ERRDRIFT'] = np.zeros((nfiles, loc['NUMBER_ORDERS']))
loc['DELTATIME'] = np.zeros(nfiles)
loc['MEANRV'] = np.zeros(nfiles)
loc['MEANRV_LEFT'] = np.zeros(nfiles)
loc['MEANRV_RIGHT'] = np.zeros(nfiles)
loc['MERRDRIFT'] = np.zeros(nfiles)
loc['FLUXRATIO'] = np.zeros(nfiles)
# add sources
source = __NAME__ + '/main()'
keys = [
'drift', 'drift_left', 'drift_right', 'errdrift', 'deltatime',
'meanrv', 'meanrv_left', 'meanrv_right', 'merrdrift', 'fluxratio'
]
loc.set_sources(keys, source)
# ------------------------------------------------------------------
# Loop around all files: correct for dark, reshape, extract and
# calculate dvrms and meanpond
# ------------------------------------------------------------------
# get the maximum number of orders to use
nomin = p['IC_DRIFT_PEAK_N_ORDER_MIN']
nomax = p['IC_DRIFT_PEAK_N_ORDER_MAX']
# loop around files
for i_it in range(nfiles):
# get file for this iteration
fpfile = listfiles[::skip][i_it]
# Log the file we are reading
wmsg = 'Reading file {0}'
WLOG(p, '', wmsg.format(os.path.split(fpfile)[-1]))
# ------------------------------------------------------------------
# read e2ds files and get timestamp
# ------------------------------------------------------------------
# read data
rout = spirouImage.ReadData(p, filename=fpfile, log=False)
loc['SPE'], hdri, nxi, nyi = rout
# apply flat
loc['SPE'] = loc['SPE'] / loc['FLAT']
# get acqtime
bjdspe = spirouImage.GetAcqTime(p,
hdri,
name='acqtime',
return_value=1,
kind='julian')
# ----------------------------------------------------------------------
# Background correction
# ----------------------------------------------------------------------
# test whether we want to subtract background
if p['IC_DRIFT_BACK_CORR']:
# Loop around the orders
for order_num in range(loc['NUMBER_ORDERS']):
# get the box size from constants
bsize = p['DRIFT_PEAK_MINMAX_BOXSIZE']
# Measurethe min and max flux
miny, maxy = spirouBACK.MeasureMinMax(loc['SPE'][order_num],
bsize)
# subtract off the background (miny)
loc['SPE'][order_num] = loc['SPE'][order_num] - miny
# ------------------------------------------------------------------
# calculate flux ratio
# ------------------------------------------------------------------
sorder = p['IC_DRIFT_ORDER_PLOT']
fratio = np.nansum(loc['SPE'][sorder]) / np.nansum(
loc['SPEREF'][sorder])
loc['FLUXRATIO'][i_it] = fratio
# ------------------------------------------------------------------
# Calculate delta time
# ------------------------------------------------------------------
# calculate the time from reference (in hours)
loc['DELTATIME'][i_it] = (bjdspe - bjdref) * 24
# ------------------------------------------------------------------
# Calculate PearsonR coefficient
# ------------------------------------------------------------------
pargs = [loc['NUMBER_ORDERS'], loc['SPE'], loc['SPEREF']]
correlation_coeffs = spirouRV.PearsonRtest(*pargs)
# ----------------------------------------------------------------------
# Get drift with comparison to the reference image
# ----------------------------------------------------------------------
# only calculate drift if the correlation between orders and
# reference file is above threshold
prcut = p['DRIFT_PEAK_PEARSONR_CUT']
if np.min(correlation_coeffs[nomin:nomax]) > prcut:
# get drifts for each order
dargs = [p, loc['SPE'], loc['ORDPEAK'], loc['XREF']]
x = spirouRV.GetDrift(*dargs, gaussfit=gaussfit)
# get delta v
loc['DV'] = (x - loc['XREF']) * loc['VRPEAK']
# sigma clip
loc = spirouRV.SigmaClip(loc, sigma=p['DRIFT_PEAK_SIGMACLIP'])
# work out median drifts per order
loc = spirouRV.DriftPerOrder(loc, i_it)
# work out mean drift across all orders
loc = spirouRV.DriftAllOrders(p, loc, i_it, nomin, nomax)
# log the mean drift
wmsg = ('Time from ref= {0:.2f} h - Flux Ratio= {1:.3f} '
'- Drift mean= {2:.2f} +- {3:.2f} m/s')
wargs = [
loc['DELTATIME'][i_it], loc['FLUXRATIO'][i_it],
loc['MEANRV'][i_it], loc['MERRDRIFT'][i_it]
]
WLOG(p, 'info', wmsg.format(*wargs))
# else we can't use this extract
else:
if p['DRS_PLOT'] > 0:
# start interactive session if needed
sPlt.plt.ioff()
# plot comparison between spe and ref
sPlt.drift_plot_correlation_comp(p, loc, correlation_coeffs,
i_it)
sPlt.plt.show()
sPlt.plt.close()
# turn interactive plotting back on
sPlt.plt.ion()
# log that we cannot use this extraction
wmsg1 = 'The correlation of some orders compared to the template is'
wmsg2 = ' < {0}, something went wrong in the extract.'
WLOG(p, 'warning', wmsg1)
WLOG(p, 'warning', wmsg2.format(prcut))
# ------------------------------------------------------------------
# peak to peak drift
driftptp = np.max(loc['MEANRV']) - np.min(loc['MEANRV'])
driftrms = np.std(loc['MEANRV'])
# log th etotal drift peak-to-peak and rms
wmsg = ('Total drift Peak-to-Peak={0:.3f} m/s RMS={1:.3f} m/s in '
'{2:.2f} hour')
wargs = [driftptp, driftrms, np.max(loc['DELTATIME'])]
WLOG(p, '', wmsg.format(*wargs))
# ------------------------------------------------------------------
# Plot of mean drift
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# start interactive session if needed
sPlt.start_interactive_session(p)
# plot delta time against median drift
sPlt.drift_peak_plot_dtime_against_drift(p, loc)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# TODO: Needs doing
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# add to qc header lists
qc_values.append('None')
qc_names.append('None')
qc_logic.append('None')
qc_pass.append(1)
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Save drift values to file
# ------------------------------------------------------------------
# get raw input file name
raw_infile = os.path.basename(p['REFFILE'])
# construct filename
driftfits, tag = spirouConfig.Constants.DRIFTPEAK_E2DS_FITS_FILE(p)
driftfitsname = os.path.split(driftfits)[-1]
# log that we are saving drift values
wmsg = 'Saving drift values of Fiber {0} in {1}'
WLOG(p, '', wmsg.format(p['FIBER'], driftfitsname))
# add keys from original header file
hdict = spirouImage.CopyOriginalKeys(hdr)
# set the version
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
# set the input files
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFLAT'], value=p['FLATFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_REFFILE'], value=raw_infile)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=loc['WAVEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=loc['WSOURCE'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# save drift values
p = spirouImage.WriteImage(p, driftfits, loc['DRIFT'], hdict)
# ------------------------------------------------------------------
# print .tbl result
# ------------------------------------------------------------------
# construct filename
drifttbl = spirouConfig.Constants.DRIFTPEAK_E2DS_TBL_FILE(p)
drifttblname = os.path.split(drifttbl)[-1]
# construct and write table
columnnames = ['time', 'drift', 'drifterr', 'drift_left', 'drift_right']
columnformats = ['7.4f', '6.2f', '6.3f', '6.2f', '6.2f']
columnvalues = [
loc['DELTATIME'], loc['MEANRV'], loc['MERRDRIFT'], loc['MEANRV_LEFT'],
loc['MEANRV_RIGHT']
]
table = spirouImage.MakeTable(p,
columns=columnnames,
values=columnvalues,
formats=columnformats)
# write table
wmsg = 'Average Drift saved in {0} Saved '
WLOG(p, '', wmsg.format(drifttblname))
spirouImage.WriteTable(p, table, drifttbl, fmt='ascii.rst')
# ------------------------------------------------------------------
# Plot amp and llpeak
# ------------------------------------------------------------------
if p['DRS_PLOT'] and p['DRIFT_PEAK_PLOT_LINE_LOG_AMP']:
# start interactive session if needed
sPlt.start_interactive_session(p)
# plot delta time against median drift
sPlt.drift_peak_plot_llpeak_amps(p, loc)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
"""
cal_SLIT_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_SLIT_spirou.py [night_directory] [files]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files)
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# set the fiber type
p['FIBER'] = 'AB'
p.set_source('FIBER', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add')
# ----------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
data = spirouImage.FixNonPreProcess(p, data)
# ----------------------------------------------------------------------
# Get basic image properties
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# ----------------------------------------------------------------------
# Correction of DARK
# ----------------------------------------------------------------------
p, datac = spirouImage.CorrectForDark(p, data, hdr)
datac = data
# ----------------------------------------------------------------------
# Resize image
# ----------------------------------------------------------------------
# rotate the image and convert from ADU/s to e-
data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p)
# convert NaN to zeros
data0 = np.where(~np.isfinite(data), np.zeros_like(data), data)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'],
getshape=False)
data2 = spirouImage.ResizeImage(p, data0, **bkwargs)
# log change in data size
WLOG(p, '', ('Image format changed to '
'{0}x{1}').format(*data2.shape))
# ----------------------------------------------------------------------
# Correct for the BADPIX mask (set all bad pixels to zero)
# ----------------------------------------------------------------------
p, data2 = spirouImage.CorrectForBadPix(p, data2, hdr)
p, badpixmap = spirouImage.CorrectForBadPix(p, data2, hdr, return_map=True)
# ----------------------------------------------------------------------
# Background computation
# ----------------------------------------------------------------------
if p['IC_DO_BKGR_SUBTRACTION']:
# log that we are doing background measurement
WLOG(p, '', 'Doing background measurement on raw frame')
# get the bkgr measurement
bargs = [p, data2, hdr, badpixmap]
# background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs)
p, background = spirouBACK.MeasureBackgroundMap(*bargs)
else:
background = np.zeros_like(data2)
p['BKGRDFILE'] = 'None'
p.set_source('BKGRDFILE', __NAME__ + '.main()')
# apply background correction to data
data2 = data2 - background
# save data to loc
loc = ParamDict()
loc['DATA'] = data2
loc.set_source('DATA', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Log the number of dead pixels
# ----------------------------------------------------------------------
# get the number of bad pixels
n_bad_pix = np.nansum(data2 <= 0)
n_bad_pix_frac = n_bad_pix * 100 / np.product(data2.shape)
# Log number
wmsg = 'Nb dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# ------------------------------------------------------------------
# Get localisation coefficients
# ------------------------------------------------------------------
# original there is a loop but it is not used --> removed
p = spirouImage.FiberParams(p, p['FIBER'], merge=True)
# get localisation fit coefficients
p, loc = spirouLOCOR.GetCoeffs(p, hdr, loc)
# ------------------------------------------------------------------
# Calculate shape map
# ------------------------------------------------------------------
loc = spirouImage.GetShapeMap(p, loc)
# ------------------------------------------------------------------
# Plotting
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# plots setup: start interactive plot
sPlt.start_interactive_session(p)
# plot the shape process for each order
sPlt.slit_shape_angle_plot(p, loc)
# end interactive section
sPlt.end_interactive_session(p)
# ------------------------------------------------------------------
# Writing to file
# ------------------------------------------------------------------
# get the raw tilt file name
raw_shape_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
shapefits, tag = spirouConfig.Constants.SLIT_XSHAPE_FILE(p)
shapefitsname = os.path.basename(shapefits)
# Log that we are saving tilt file
wmsg = 'Saving shape information in file: {0}'
WLOG(p, '', wmsg.format(shapefitsname))
# Copy keys from fits file
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(hdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPE'], value=raw_shape_file)
# write tilt file to file
p = spirouImage.WriteImage(p, shapefits, loc['DXMAP'], hdict)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# TODO: Decide on some quality control criteria?
# set passed variable and fail message list
passed, fail_msg = True, []
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Move to calibDB and update calibDB
# ----------------------------------------------------------------------
if p['QC']:
keydb = 'SHAPE'
# copy shape file to the calibDB folder
spirouDB.PutCalibFile(p, shapefits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, shapefitsname, hdr)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
# =============================================================================
# Define variables
# =============================================================================
__NAME__ = 'drs_tools.py'
# Get version and author
__version__ = spirouConfig.Constants.VERSION()
__author__ = spirouConfig.Constants.AUTHORS()
__date__ = spirouConfig.Constants.LATEST_EDIT()
__release__ = spirouConfig.Constants.RELEASE()
# Get param dict
ParamDict = spirouConfig.ParamDict
# Get Logging function
WLOG = spirouCore.wlog
# get default config parameters
PARAMS = spirouStartup.Begin(recipe=__NAME__, quiet=True)
# -----------------------------------------------------------------------------
# =============================================================================
# Define user functions
# =============================================================================
def list_raw_files(prefix=None,
suffix=None,
substring=None,
nightname=None,
raw=False):
# set path
path = PARAMS['DRS_DATA_RAW']
# list files
def main(night_name=None, files=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files)
# force plotting to 1
p['DRS_PLOT'] = 1
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
data, hdr, nx, ny = spirouImage.ReadImage(p)
# ----------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
data = spirouImage.FixNonPreProcess(p, data, filename=p['FITSFILENAME'])
# ----------------------------------------------------------------------
# Get basic image properties
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# set sigdet and conad keywords (sigdet is changed later)
p['KW_CCD_SIGDET'][1] = p['SIGDET']
p['KW_CCD_CONAD'][1] = p['GAIN']
# now change the value of sigdet if require
if p['IC_EXT_SIGDET'] > 0:
p['SIGDET'] = float(p['IC_EXT_SIGDET'])
# ----------------------------------------------------------------------
# Resize image
# ----------------------------------------------------------------------
# rotate the image and convert from ADU/s to e-
data = spirouImage.ConvertToADU(spirouImage.FlipImage(p, data), p=p)
# convert NaN to zeros
data2 = np.where(~np.isfinite(data), np.zeros_like(data), data)
# resize image
# bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'],
# ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'],
# getshape=False)
# data2 = spirouImage.ResizeImage(data0, **bkwargs)
# log change in data size
# WLOG(p, '', ('Image format changed to '
# '{0}x{1}').format(*data2.shape[::-1]))
# ----------------------------------------------------------------------
# Log the number of dead pixels
# ----------------------------------------------------------------------
# get the number of bad pixels
n_bad_pix = np.nansum(data2 == 0)
n_bad_pix_frac = n_bad_pix * 100 / np.product(data2.shape)
# Log number
wmsg = 'Nb dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
satseuil = 64536.
col = 2100
seuil = 10000
slice0 = 5
plt.ion()
plt.clf()
plt.imshow(data2, origin='lower', clim=(1., seuil))
plt.colorbar()
plt.axis([0, nx, 0, ny])
plt.plot(np.ones(4096) * col, np.arange(4096), c='red')
plt.figure()
plt.clf()
centpart = data2[:, col - slice0:col + slice0]
# centpart = data2[col - slice:col + slice,:]
# weights = np.where((centpart < satseuil) & (centpart > 0), 1, 0.0001)
# y = np.average(centpart, axis=1, weights=weights) ## weighted average
y = np.nanmedian(centpart, axis=1)
# y=average(centpart,axis=1,weights=where((centpart>0),1,0.0001))
# ## weighted average
plt.plot(np.arange(ny), y)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p, outputs=None)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(runname=None, quiet=False):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__, quiet=True)
# now get custom arguments
ckwargs = dict(positions=[0],
types=[str],
names=['RUNNAME'],
calls=[runname],
require_night_name=False,
required=[False])
customargs = spirouStartup.GetCustomFromRuntime(p, **ckwargs)
# add custom args straight to p
p = spirouStartup.LoadMinimum(p, customargs=customargs, quiet=True)
# ----------------------------------------------------------------------
# Read the run file and extract parameters
# ----------------------------------------------------------------------
# construct filename
rfile = os.path.join(UNIT_TEST_PATH, p['RUNNAME'])
# check if RUNNAME is None
if p['RUNNAME'] == 'None':
exists = False
emsgs = ['No unit test run file defined.']
# check that rfile exists
elif not os.path.exists(rfile):
emsgs = ['Unit test run file "{0}" does not exist'.format(rfile)]
exists = False
else:
exists = True
emsgs = []
# deal with file wrong (or no file defined) --> print valid unit tests
if not exists:
emsgs.append('')
emsgs.append('Available units tests are:')
for rfile in os.listdir(UNIT_TEST_PATH):
emsgs.append('\t{0}'.format(rfile))
emsgs.append('')
emsgs.append('Located at {0}'.format(UNIT_TEST_PATH))
WLOG(p, 'error', emsgs)
# get the parameters in the run file
rparams = spirouConfig.GetConfigParams(p, filename=rfile)
# reset the DRS
if not quiet:
spirouTools.DRS_Reset(log=True, called=True)
# ----------------------------------------------------------------------
# Get runs
# ----------------------------------------------------------------------
runs = spirouUnitTests.get_runs(p, rparams, rfile)
# ----------------------------------------------------------------------
# Run runs
# ----------------------------------------------------------------------
# storage for times
times = OrderedDict()
errors = OrderedDict()
# log the start of the unit tests
spirouUnitTests.unit_log_title(p)
# loop around runs and process each
for runn in list(runs.keys()):
if p['DRS_DEBUG'] > 0:
# do run
rargs = [p, runn, runs[runn], times]
times = spirouUnitTests.manage_run(*rargs)
else:
# try to run
try:
# do run
rargs = [p, runn, runs[runn], times]
times = spirouUnitTests.manage_run(*rargs)
# Manage unexpected errors
except Exception as e:
wmsgs = ['Run "{0}" had an unexpected error:'.format(runn)]
for msg in str(e).split('\n'):
wmsgs.append('\t' + msg)
WLOG(p, 'warning', wmsgs)
errors[runn] = str(e)
# Manage expected errors
except SystemExit as e:
wmsgs = ['Run "{0}" had an expected error:'.format(runn)]
for msg in str(e).split('\n'):
wmsgs.append('\t' + msg)
WLOG(p, 'warning', wmsgs)
errors[runn] = str(e)
# ----------------------------------------------------------------------
# Make sure all plots are close
# ----------------------------------------------------------------------
sPlt.closeall()
# ----------------------------------------------------------------------
# Print timings
# ----------------------------------------------------------------------
spirouUnitTests.log_timings(p, times)
# ----------------------------------------------------------------------
# Print errors
# ----------------------------------------------------------------------
spirouUnitTests.log_errors(p, errors)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
wmsg = 'Recipe {0} has been successfully completed'
WLOG(p, 'info', wmsg.format(p['PROGRAM']))
# return a copy of locally defined variables in the memory
return dict(locals())
def main():
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p)
# ----------------------------------------------------------------------
# Get all raw files in DRS_DATA_RAW
# ----------------------------------------------------------------------
# define search path
search_path = os.path.join(p['DRS_DATA_RAW'], '**')
# get all files
rawfiles = glob.glob(search_path, recursive=True)
# ----------------------------------------------------------------------
# Loop around files and identify valid files
# ----------------------------------------------------------------------
valid_files = []
for rawfilename in rawfiles:
# skip directories
if os.path.isdir(rawfilename):
continue
# get base name and dir name
basename = os.path.basename(rawfilename)
directory = os.path.dirname(rawfilename)
# skip pre-processed files
if p['PROCESSED_SUFFIX'] in basename:
continue
# search for pre-processed file
if not FORCE_REPROCESS:
# get odometer name
odo_name = str(basename).split('_')[0].split('.fits')[0]
# get directory files
dir_files = os.listdir(directory)
# set preprocessed file found flag
pp_found = False
# loop around all files in this directory
for dir_file in dir_files:
# check if preprocessed suffix in file
cond1 = p['PROCESSED_SUFFIX'] in dir_file
# check that odometer name in file
cond2 = odo_name in dir_file
# if both of these are found we have found a preprocessed file
# for this rawfilename
if cond1 and cond2:
pp_found = True
# if we have found a preprocessed file then do not add to
# valid files
if pp_found:
continue
else:
valid_files.append(rawfilename)
# finally if nothing else add to valid_files
else:
valid_files.append(rawfilename)
# ----------------------------------------------------------------------
# Run pre-processing
# ----------------------------------------------------------------------
# get number of valid files
number_files = len(valid_files)
if number_files == 0:
WLOG(p, 'warning', 'All files up-to-date.')
# loop around valid files
for v_it, valid_file in enumerate(valid_files):
# get base name and dir name
basename = os.path.basename(valid_file)
directory = os.path.dirname(valid_file)
# get night_name
night_name = directory.split(p['DRS_DATA_RAW'])[-1][len(os.path.sep):]
# print progress
WLOG('', '', '')
WLOG('', '', '=' * 50)
wmsg = 'Processing file {0} of {1}'.format(v_it + 1, number_files)
WLOG(p, '', wmsg)
WLOG('', '', '=' * 50)
WLOG('', '', '')
# run pre-processing
_ = cal_preprocess_spirou.main(night_name, basename)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
wmsg = 'Recipe {0} has been successfully completed'
WLOG(p, 'info', wmsg.format(p['PROGRAM']))
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p,
night_name,
files,
mainfitsdir='reduced')
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# set up function name
main_name = __NAME__ + '.main()'
# ------------------------------------------------------------------
# Load first file
# ------------------------------------------------------------------
loc = ParamDict()
rd = spirouImage.ReadImage(p, p['FITSFILENAME'])
loc['DATA'], loc['DATAHDR'], loc['YDIM'], loc['XDIM'] = rd
loc.set_sources(['DATA', 'DATAHDR', 'XDIM', 'YDIM'], main_name)
# ------------------------------------------------------------------
# Get the wave solution
# ------------------------------------------------------------------
masterwavefile = spirouDB.GetDatabaseMasterWave(p)
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# read master wave map
wout = spirouImage.GetWaveSolution(p,
filename=masterwavefile,
return_wavemap=True,
quiet=True,
return_header=True,
fiber=wave_fiber)
_, loc['WAVE'], loc['WAVEFILE'], _ = wout
loc.set_sources(['WAVE', 'WAVEFILE'], main_name)
# get the wave keys
loc = spirouImage.GetWaveKeys(p, loc, loc['DATAHDR'])
# ------------------------------------------------------------------
# Construct convolution kernels (used in GetMolecularTellLines)
# ------------------------------------------------------------------
loc = spirouTelluric.ConstructConvKernel1(p, loc)
# ------------------------------------------------------------------
# Get molecular telluric lines
# ------------------------------------------------------------------
loc = spirouTelluric.GetMolecularTellLines(p, loc)
# if TAPAS FNAME is not None we generated a new file so should add to tellDB
if loc['TAPAS_FNAME'] is not None:
# add to the telluric database
spirouDB.UpdateDatabaseTellConv(p, loc['TAPAS_FNAME'], loc['DATAHDR'])
# put file in telluDB
spirouDB.PutTelluFile(p, loc['TAPAS_ABSNAME'])
# ----------------------------------------------------------------------
# load the expected atmospheric transmission
# ----------------------------------------------------------------------
# read filename from telluDB
tapas_file_names = spirouDB.GetDatabaseTellConv(p)
tapas_file_name = tapas_file_names[-1]
# load atmospheric transmission
sp_tapas = np.load(tapas_file_name)
loc['TAPAS_ALL_SPECIES'] = sp_tapas
# extract the water and other line-of-sight optical depths
loc['TAPAS_WATER'] = sp_tapas[1, :]
loc['TAPAS_OTHERS'] = np.prod(sp_tapas[2:, :], axis=0)
loc.set_sources(['TAPAS_ALL_SPECIES', 'TAPAS_WATER', 'TAPAS_OTHERS'],
main_name)
# ------------------------------------------------------------------
# Get master wave solution map
# ------------------------------------------------------------------
# get master wave map
masterwavefile = spirouDB.GetDatabaseMasterWave(p)
# log process
wmsg1 = 'Shifting transmission map on to master wavelength grid'
wmsg2 = '\tFile = {0}'.format(os.path.basename(masterwavefile))
WLOG(p, '', [wmsg1, wmsg2])
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# read master wave map
mout = spirouImage.GetWaveSolution(p,
filename=masterwavefile,
return_wavemap=True,
quiet=True,
return_header=True,
fiber=wave_fiber)
masterwavep, masterwave, masterwaveheader, mwsource = mout
# get wave acqtimes
master_acqtimes = spirouDB.GetTimes(p, masterwaveheader)
# ------------------------------------------------------------------
# Loop around the files
# ------------------------------------------------------------------
# construct extension
tellu_ext = '{0}_{1}.fits'
# get current telluric maps from telluDB
# tellu_db_data = spirouDB.GetDatabaseTellMap(p, required=False)
# tellu_db_files = tellu_db_data[0]
# storage for valid output files
loc['OUTPUTFILES'] = []
# loop around the files
for basefilename in p['ARG_FILE_NAMES']:
# ------------------------------------------------------------------
# Get absolute path of filename
# ------------------------------------------------------------------
filename = os.path.join(p['ARG_FILE_DIR'], basefilename)
# ------------------------------------------------------------------
# Read obj telluric file and correct blaze (per order)
# ------------------------------------------------------------------
# get image
sp, shdr, _, _ = spirouImage.ReadImage(p, filename)
# ------------------------------------------------------------------
# check that file has valid DPRTYPE
# ------------------------------------------------------------------
# get FP_FP DPRTYPE
p = spirouImage.ReadParam(p, shdr, 'KW_DPRTYPE', 'DPRTYPE', dtype=str)
# if dprtype is incorrect skip
if p['DPRTYPE'] not in p['ALLOWED_TELLURIC_DPRTYPES']:
wmsg1 = 'Skipping file (DPRTYPE incorrect)'
wmsg2 = '\t DPRTYPE = {0}'.format(p['DPRTYPE'])
WLOG(p, 'warning', [wmsg1, wmsg2])
continue
# get blaze
p, blaze = spirouImage.ReadBlazeFile(p, shdr)
# get the blaze percentile
blaze_p = p['MKTELLU_BLAZE_PERCENTILE']
# loop through blaze orders, normalize blaze by its peak amplitude
for order_num in range(sp.shape[0]):
# normalize the spectrum
spo, bzo = sp[order_num], blaze[order_num]
sp[order_num] = spo / np.nanpercentile(spo, blaze_p)
# normalize the blaze
blaze[order_num] = bzo / np.nanpercentile(bzo, blaze_p)
# find where the blaze is bad
with warnings.catch_warnings(record=True) as _:
badblaze = blaze < p['MKTELLU_CUT_BLAZE_NORM']
# set bad blaze to NaN
blaze[badblaze] = np.nan
# set to NaN values where spectrum is zero
zeromask = sp == 0
sp[zeromask] = np.nan
# divide spectrum by blaze
with warnings.catch_warnings(record=True) as _:
sp = sp / blaze
# add sp to loc
loc['SP'] = sp
loc.set_source('SP', main_name)
# ----------------------------------------------------------------------
# Get object name, airmass and berv
# ----------------------------------------------------------------------
# Get object name
loc['OBJNAME'] = spirouImage.GetObjName(p, shdr)
# Get the airmass
loc['AIRMASS'] = spirouImage.GetAirmass(p, shdr)
# Get the Barycentric correction from header
p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, shdr)
# set sources
source = main_name + '+ spirouImage.ReadParams()'
loc.set_sources(['OBJNAME', 'AIRMASS'], source)
# ------------------------------------------------------------------
# get output transmission filename
outfile, tag1 = spirouConfig.Constants.TELLU_TRANS_MAP_FILE(
p, filename)
outfilename = os.path.basename(outfile)
loc['OUTPUTFILES'].append(outfile)
# ----------------------------------------------------------------------
# Load template (if available)
# ----------------------------------------------------------------------
# read filename from telluDB
template_file = spirouDB.GetDatabaseObjTemp(p,
loc['OBJNAME'],
required=False)
# if we don't have a template flag it
if template_file is None:
loc['FLAG_TEMPLATE'] = False
loc['TEMPLATE'] = None
# construct progres string
pstring = 'No template found.'
else:
loc['FLAG_TEMPLATE'] = True
# load template
template, _, _, _ = spirouImage.ReadImage(p, template_file)
# add to loc
loc['TEMPLATE'] = template
# construct progres string
template_bfile = os.path.basename(template_file)
pstring = 'Using template {0}'.format(template_bfile)
# set the source for flag and template
loc.set_sources(['FLAG_TEMPLATE', 'TEMPLATE'], main_name)
# ------------------------------------------------------------------
# log processing file
wmsg = 'Processing file {0}. {1}'
WLOG(p, '', [wmsg.format(outfilename, pstring)])
# ------------------------------------------------------------------
# Check that basefile is not in blacklist
# ------------------------------------------------------------------
blacklist_check = spirouTelluric.CheckBlackList(loc['OBJNAME'])
if blacklist_check:
# log black list file found
wmsg = 'File {0} is blacklisted (OBJNAME={1}). Skipping'
wargs = [basefilename, loc['OBJNAME']]
WLOG(p, 'warning', wmsg.format(*wargs))
# skip this file
continue
# ------------------------------------------------------------------
# deal with applying template to spectrum
# ------------------------------------------------------------------
# Requires from loc:
# TEMPLATE (None or template loaded from file)
# FLAG_TEMPLATE
# WAVE
# SP
# BERV
#
# Returns:
# SP (modified if template was used)
# TEMPLATE
# WCONV
loc = spirouTelluric.ApplyTemplate(p, loc)
# ------------------------------------------------------------------
# calcullate telluric absorption (with a sigma clip loop)
# ------------------------------------------------------------------
# Requires from loc:
# AIRMASS
# WAVE
# SP
# WCONV
# Returns:
# PASSED [Bool] True or False
# SP_OUT
# SED_OUT
# RECOV_AIRMASS
# RECOV_WATER
loc = spirouTelluric.CalcTelluAbsorption(p, loc)
# calculate tranmission map from sp and sed
transmission_map = loc['SP_OUT'] / loc['SED_OUT']
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# if array is completely NaNs it shouldn't pass
if np.sum(np.isfinite(transmission_map)) == 0:
fail_msg.append('transmission map is all NaNs')
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append('NaN')
qc_names.append('image')
qc_logic.append('image is all NaN')
# ----------------------------------------------------------------------
# get SNR for each order from header
nbo = loc['DATA'].shape[0]
snr_order = p['QC_MK_TELLU_SNR_ORDER']
snr = spirouImage.Read1Dkey(p, shdr, p['kw_E2DS_SNR'][0], nbo)
# check that SNR is high enough
if snr[snr_order] < p['QC_MK_TELLU_SNR_MIN']:
fmsg = 'low SNR in order {0}: ({1:.2f} < {2:.2f})'
fargs = [snr_order, snr[snr_order], p['QC_MK_TELLU_SNR_MIN']]
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(snr[snr_order])
qc_name_str = 'SNR[{0}]'.format(snr_order)
qc_names.append(qc_name_str)
qc_logic.append('{0} < {1:.2f}'.format(qc_name_str,
p['QC_MK_TELLU_SNR_ORDER']))
# ----------------------------------------------------------------------
# check that the file passed the CalcTelluAbsorption sigma clip loop
if not loc['PASSED']:
fmsg = 'File {0} did not converge on a solution in function: {1}'
fargs = [basefilename, 'spirouTelluric.CalcTelluAbsorption()']
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(basefilename)
qc_names.append('FILE')
qc_logic.append('FILE did not converge')
# ----------------------------------------------------------------------
# check that the airmass is not too different from input airmass
airmass_diff = np.abs(loc['RECOV_AIRMASS'] - loc['AIRMASS'])
fargs = [
loc['RECOV_AIRMASS'], loc['AIRMASS'], p['QC_MKTELLU_AIRMASS_DIFF']
]
if airmass_diff > p['QC_MKTELLU_AIRMASS_DIFF']:
fmsg = ('Recovered airmass to de-similar than input airmass.'
'Recovered: {0:.3f}. Input: {1:.3f}. QC limit = {2}')
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(airmass_diff)
qc_names.append('airmass_diff')
qc_logic.append('airmass_diff > {0:.2f}'
''.format(p['QC_MKTELLU_AIRMASS_DIFF']))
# ----------------------------------------------------------------------
# check that the water vapor is within limits
water_cond1 = loc['RECOV_WATER'] < p['MKTELLU_TRANS_MIN_WATERCOL']
water_cond2 = loc['RECOV_WATER'] > p['MKTELLU_TRANS_MAX_WATERCOL']
fargs = [
p['MKTELLU_TRANS_MIN_WATERCOL'], p['MKTELLU_TRANS_MAX_WATERCOL']
]
if water_cond1 or water_cond2:
fmsg = ('Recovered water vapor optical depth not between {0:.3f} '
'and {1:.3f}')
fail_msg.append(fmsg.format(*fargs))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(loc['RECOV_WATER'])
qc_names.append('RECOV_WATER')
qc_logic.append('RECOV_WATER not between {0:.3f} and {1:.3f}'
''.format(*fargs))
# ----------------------------------------------------------------------
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
continue
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Save transmission map to file
# ------------------------------------------------------------------
# get raw file name
raw_in_file = os.path.basename(p['FITSFILENAME'])
# copy original keys
hdict = spirouImage.CopyOriginalKeys(loc['DATAHDR'])
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
# set the input files
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBBLAZE'],
value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=os.path.basename(masterwavefile))
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=mwsource)
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='file',
values=p['ARG_FILE_NAMES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# add wave solution date
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_TIME1'],
value=master_acqtimes[0])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_TIME2'],
value=master_acqtimes[1])
# add wave solution number of orders
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_ORD_N'],
value=masterwavep.shape[0])
# add wave solution degree of fit
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_LL_DEG'],
value=masterwavep.shape[1] - 1)
# add wave solution coefficients
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_WAVE_PARAM'],
values=masterwavep)
# add telluric keys
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TELLU_AIRMASS'],
value=loc['RECOV_AIRMASS'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_TELLU_WATER'],
value=loc['RECOV_WATER'])
# write to file
p = spirouImage.WriteImage(p, outfile, transmission_map, hdict)
# ------------------------------------------------------------------
# Add transmission map to telluDB
# ------------------------------------------------------------------
if p['QC']:
# copy tellu file to the telluDB folder
spirouDB.PutTelluFile(p, outfile)
# update the master tellu DB file with transmission map
targs = [
p, outfilename, loc['OBJNAME'], loc['RECOV_AIRMASS'],
loc['RECOV_WATER']
]
spirouDB.UpdateDatabaseTellMap(*targs)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
"""
cal_shape_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_shape_spirou.py [night_directory] [fitsfilename]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files)
p = spirouStartup.InitialFileSetup(p)
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add')
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# ----------------------------------------------------------------------
# Get basic image properties for FP file
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# ----------------------------------------------------------------------
# Correction of reference FP
# ----------------------------------------------------------------------
# set the number of frames
p['NBFRAMES'] = 1
p.set_source('NBFRAMES', __NAME__ + '.main()')
# Correction of DARK
p, datac = spirouImage.CorrectForDark(p, data, hdr)
# Resize hc data
# rotate the image and convert from ADU/s to e-
data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'],
getshape=False)
data1 = spirouImage.ResizeImage(p, data, **bkwargs)
# log change in data size
WLOG(p, '',
('FPref Image format changed to {0}x{1}').format(*data1.shape))
# Correct for the BADPIX mask (set all bad pixels to zero)
bargs = [p, data1, hdr]
p, data1 = spirouImage.CorrectForBadPix(*bargs)
p, badpixmask = spirouImage.CorrectForBadPix(*bargs, return_map=True)
# log progress
WLOG(p, '', 'Cleaning FPref hot pixels')
# correct hot pixels
data1 = spirouEXTOR.CleanHotpix(data1, badpixmask)
# Log the number of dead pixels
# get the number of bad pixels
with warnings.catch_warnings(record=True) as _:
n_bad_pix = np.nansum(data1 <= 0)
n_bad_pix_frac = n_bad_pix * 100 / np.product(data1.shape)
# Log number
wmsg = 'Nb FPref dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# ----------------------------------------------------------------------
# Get master FP file
# ----------------------------------------------------------------------
# log progress
WLOG(p, '', 'Getting FP Master from calibDB')
# get master fp
p, masterfp = spirouImage.GetFPMaster(p, hdr)
# ----------------------------------------------------------------------
# Get transform parameters
# ----------------------------------------------------------------------
# log progress
wargs = [p['ARG_FILE_NAMES'][0], p['FPMASTERFILE']]
WLOG(p, 'info', 'Calculating transforming for {0} onto {1}'.format(*wargs))
gout = spirouImage.GetLinearTransformParams(p, masterfp, data1)
transform, xres, yres = gout
# ------------------------------------------------------------------
# Need to straighten the fp data for debug
# ------------------------------------------------------------------
p, shapem_x = spirouImage.GetShapeX(p, hdr)
p, shapem_y = spirouImage.GetShapeY(p, hdr)
data2 = spirouImage.EATransform(data1, transform, dxmap=shapem_x,
dymap=shapem_y)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# if transform is None means the fp image quality was too poor
if transform is None:
fail_msg.append('FP Image quality too poor (sigma clip failed)')
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
qc_values.append('None')
qc_names.append('Image Quality')
qc_logic.append('Image too poor')
# ----------------------------------------------------------------------
# get residual qc parameter
qc_res = p['SHAPE_QC_LINEAR_TRANS_RES_THRES']
# assess quality of x residuals
if xres > qc_res:
fail_msg.append('x-resdiuals too high {0} > {1}'.format(xres, qc_res))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
qc_values.append(xres)
qc_names.append('XRES')
qc_logic.append('XRES > {0}'.format(qc_res))
# assess quality of x residuals
if yres > qc_res:
fail_msg.append('y-resdiuals too high {0} > {1}'.format(yres, qc_res))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
qc_values.append(yres)
qc_names.append('YRES')
qc_logic.append('YRES > {0}'.format(qc_res))
# ----------------------------------------------------------------------
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Writing shape to file
# ------------------------------------------------------------------
# get the raw tilt file name
raw_shape_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
shapefits, tag = spirouConfig.Constants.SLIT_SHAPE_LOCAL_FILE(p)
shapefitsname = os.path.basename(shapefits)
# Log that we are saving tilt file
wmsg = 'Saving shape information in file: {0}'
WLOG(p, '', wmsg.format(shapefitsname))
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(hdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEX'],
value=p['SHAPEXFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBSHAPEY'],
value=p['SHAPEYFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBFPMASTER'],
value=p['FPMASTERFILE'])
hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file',
values=p['ARG_FILE_NAMES'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# add the transform parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_DX'], value=transform[0])
hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_DY'], value=transform[1])
hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_A'], value=transform[2])
hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_B'], value=transform[3])
hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_C'], value=transform[4])
hdict = spirouImage.AddKey(p, hdict, p['KW_SHAPE_D'], value=transform[5])
# write tilt file to file
p = spirouImage.WriteImage(p, shapefits, [transform], hdict)
# ----------------------------------------------------------------------
# Move to calibDB and update calibDB
# ----------------------------------------------------------------------
if p['QC']:
# add shape
keydb = 'SHAPE'
# copy shape file to the calibDB folder
spirouDB.PutCalibFile(p, shapefits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, shapefitsname, hdr)
# ------------------------------------------------------------------
# Writing sanity check files
# ------------------------------------------------------------------
if p['SHAPE_DEBUG_OUTPUTS']:
# log
WLOG(p, '', 'Saving debug sanity check files')
# construct file names
dargs = [p, p['ARG_FILE_NAMES'][0]]
out1 = spirouConfig.Constants.SLIT_SHAPE_IN_FP_FILE(*dargs)
input_fp_file, tag1= out1
out2 = spirouConfig.Constants.SLIT_SHAPE_OUT_FP_FILE(*dargs)
output_fp_file, tag2 = out2
# write input fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
p = spirouImage.WriteImage(p, input_fp_file, data1, hdict)
# write output fp file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
p = spirouImage.WriteImage(p, output_fp_file, data2, hdict)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None,
e2dsfile=None,
mask=None,
rv=None,
width=None,
step=None):
"""
cal_CCF_E2DS_spirou.py main function, if arguments are None uses
arguments from run time i.e.:
cal_CCF_E2DS_spirou.py [night_directory] [E2DSfilename] [mask] [RV]
[width] [step]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param e2dsfile: string, the E2DS file to use
:param mask: string, the mask file to use (i.e. "UrNe.mas")
:param rv: float, the target RV to use
:param width: float, the CCF width to use
:param step: float, the CCF step to use
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# deal with arguments being None (i.e. get from sys.argv)
pos = [0, 1, 2, 3, 4]
fmt = [str, str, float, float, float]
name = ['e2dsfile', 'ccf_mask', 'target_rv', 'ccf_width', 'ccf_step']
lname = ['input_file', 'CCF_mask', 'RV', 'CCF_width', 'CCF_step']
req = [True, True, True, False, False]
call = [e2dsfile, mask, rv, width, step]
call_priority = [True, True, True, True, True]
# now get custom arguments
customargs = spirouStartup.GetCustomFromRuntime(p, pos, fmt, name, req,
call, call_priority, lname)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsfile='e2dsfile',
mainfitsdir='reduced')
# ----------------------------------------------------------------------
# Construct reference filename and get fiber type
# ----------------------------------------------------------------------
p, e2dsfilename = spirouStartup.SingleFileSetup(p, filename=p['E2DSFILE'])
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# ----------------------------------------------------------------------
# Deal with optional run time arguments
# ----------------------------------------------------------------------
# define default arguments (if ccf_width and ccf_step are not defined
# in function call or run time arguments
if 'ccf_width' not in p:
p['CCF_WIDTH'] = p['IC_CCF_WIDTH']
if 'ccf_step' not in p:
p['CCF_STEP'] = p['IC_CCF_STEP']
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
e2ds, hdr, nbo, nx = spirouImage.ReadData(p, e2dsfilename)
# add to loc
loc = ParamDict()
loc['E2DS'] = e2ds
loc['NUMBER_ORDERS'] = nbo
loc.set_sources(['E2DS', 'number_orders'], __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Get basic image properties for reference file
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# get acquisition time
p = spirouImage.GetAcqTime(p, hdr, name='acqtime', kind='julian')
# get obj name
p = spirouImage.ReadParam(p, hdr, 'KW_OBJNAME', name='OBJNAME', dtype=str)
bjdref = p['ACQTIME']
# set sigdet and conad keywords (sigdet is changed later)
p['KW_CCD_SIGDET'][1] = p['SIGDET']
p['KW_CCD_CONAD'][1] = p['GAIN']
# ----------------------------------------------------------------------
# Earth Velocity calculation
# ----------------------------------------------------------------------
if p['IC_IMAGE_TYPE'] == 'H4RG':
p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, hdr)
# ----------------------------------------------------------------------
# Read wavelength solution
# ----------------------------------------------------------------------
# log
WLOG(p, '', 'Reading wavelength solution ')
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# get wave image
wout = spirouImage.GetWaveSolution(p,
hdr=hdr,
return_wavemap=True,
return_filename=True,
fiber=wave_fiber)
param_ll, wave_ll, wavefile, wsource = wout
# save to storage
loc['PARAM_LL'], loc['WAVE_LL'], loc['WAVEFILE'], loc['WSOURCE'] = wout
source = __NAME__ + '/main() + spirouTHORCA.GetWaveSolution()'
loc.set_sources(['WAVE_LL', 'PARAM_LL', 'WAVEFILE', 'WSOURCE'], source)
# ----------------------------------------------------------------------
# Read Flat file
# ----------------------------------------------------------------------
# TODO We do not need to correct FLAT
# log
# WLOG(p, '', 'Reading Flat-Field ')
# get flat
# loc['FLAT'] = spirouImage.ReadFlatFile(p, hdr)
# loc.set_source('FLAT', __NAME__ + '/main() + /spirouImage.ReadFlatFile')
# get all values in flat that are zero to 1
# loc['FLAT'] = np.where(loc['FLAT'] == 0, 1.0, loc['FLAT'])
# get blaze
# p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hdr)
p, blaze0 = spirouImage.ReadBlazeFile(p, hdr)
# ----------------------------------------------------------------------
# Preliminary set up = no flat, no blaze
# ----------------------------------------------------------------------
# reset flat to all ones
# loc['FLAT'] = np.ones((nbo, nx))
# set blaze to all ones (if not bug in correlbin !!!
# TODO Check why Blaze makes bugs in correlbin
loc['BLAZE'] = np.ones((nbo, nx))
# set sources
# loc.set_sources(['flat', 'blaze'], __NAME__ + '/main()')
loc.set_sources(['blaze'], __NAME__ + '/main()')
# Modification of E2DS array with N.A.N
if np.isnan(np.sum(e2ds)):
WLOG(p, 'warning', 'NaN values found in e2ds, converting process')
# First basic approach Replacing N.A.N by zeros
# e2ds[np.isnan(e2ds)] = 0
# Second approach replacing N.A.N by the Adjusted Blaze
e2dsb = e2ds / blaze0
for i in np.arange(len(e2ds)):
with warnings.catch_warnings(record=True) as _:
rap = np.mean(e2dsb[i][np.isfinite(e2dsb[i])])
if np.isnan(rap):
rap = 0.0
e2ds[i] = np.where(np.isfinite(e2dsb[i]), e2ds[i], blaze0[i] * rap)
# ----------------------------------------------------------------------
# correct extracted image for flat
# ----------------------------------------------------------------------
# loc['E2DSFF'] = e2ds/loc['FLAT']
# loc['E2DSFF'] = e2ds*1.
loc['E2DSFF'] = e2ds
loc.set_source('E2DSFF', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Compute photon noise uncertainty for reference file
# ----------------------------------------------------------------------
# set up the arguments for DeltaVrms2D
dargs = [loc['E2DS'], loc['WAVE_LL']]
dkwargs = dict(sigdet=p['IC_DRIFT_NOISE'],
size=p['IC_DRIFT_BOXSIZE'],
threshold=p['IC_DRIFT_MAXFLUX'])
# run DeltaVrms2D
dvrmsref, wmeanref = spirouRV.DeltaVrms2D(*dargs, **dkwargs)
# save to loc
loc['DVRMSREF'], loc['WMEANREF'] = dvrmsref, wmeanref
loc.set_sources(['dvrmsref', 'wmeanref'], __NAME__ + '/main()()')
# log the estimated RV uncertainty
# wmsg = 'On fiber {0} estimated RV uncertainty on spectrum is {1:.3f} m/s'
# WLOG(p, 'info', wmsg.format(p['FIBER'], wmeanref))
wmsg = 'On fiber estimated RV uncertainty on spectrum is {0:.3f} m/s'
WLOG(p, 'info', wmsg.format(wmeanref))
# TEST N.A.N
# loc['E2DSFF'][20:22,2000:3000]=np.nan
# e2ds[20:30,1000:3000]=np.nan
# ----------------------------------------------------------------------
# Reference plots
# ----------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# start interactive session if needed
sPlt.start_interactive_session(p)
# plot FP spectral order
sPlt.drift_plot_selected_wave_ref(p,
loc,
x=loc['WAVE_LL'],
y=loc['E2DS'])
# plot photon noise uncertainty
sPlt.drift_plot_photon_uncertainty(p, loc)
# ----------------------------------------------------------------------
# Get template RV (from ccf_mask)
# ----------------------------------------------------------------------
# get the CCF mask from file (check location of mask)
loc = spirouRV.GetCCFMask(p, loc)
# check and deal with mask in microns (should be in nm)
if np.mean(loc['LL_MASK_CTR']) < 2.0:
loc['LL_MASK_CTR'] *= 1000.0
loc['LL_MASK_D'] *= 1000.0
# ----------------------------------------------------------------------
# Do correlation
# ----------------------------------------------------------------------
# calculate and fit the CCF
loc = spirouRV.Coravelation(p, loc)
# ----------------------------------------------------------------------
# Correlation stats
# ----------------------------------------------------------------------
# get the maximum number of orders to use
nbmax = p['CCF_NUM_ORDERS_MAX']
# get the average ccf
loc['AVERAGE_CCF'] = np.nansum(loc['CCF'][:nbmax], axis=0)
# normalize the average ccf
normalized_ccf = loc['AVERAGE_CCF'] / np.max(loc['AVERAGE_CCF'])
# get the fit for the normalized average ccf
ccf_res, ccf_fit = spirouRV.FitCCF(p,
loc['RV_CCF'],
normalized_ccf,
fit_type=0)
loc['CCF_RES'] = ccf_res
loc['CCF_FIT'] = ccf_fit
# get the max cpp
loc['MAXCPP'] = np.nansum(loc['CCF_MAX']) / np.nansum(
loc['PIX_PASSED_ALL'])
# get the RV value from the normalised average ccf fit center location
loc['RV'] = float(ccf_res[1])
# get the contrast (ccf fit amplitude)
loc['CONTRAST'] = np.abs(100 * ccf_res[0])
# get the FWHM value
loc['FWHM'] = ccf_res[2] * spirouCore.spirouMath.fwhm()
# ----------------------------------------------------------------------
# set the source
keys = [
'average_ccf', 'maxcpp', 'rv', 'contrast', 'fwhm', 'ccf_res', 'ccf_fit'
]
loc.set_sources(keys, __NAME__ + '/main()')
# ----------------------------------------------------------------------
# log the stats
wmsg = ('Correlation: C={0:.1f}[%] RV={1:.5f}[km/s] '
'FWHM={2:.4f}[km/s] maxcpp={3:.1f}')
wargs = [loc['CONTRAST'], loc['RV'], loc['FWHM'], loc['MAXCPP']]
WLOG(p, 'info', wmsg.format(*wargs))
# ----------------------------------------------------------------------
# rv ccf plot
# ----------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# Plot rv vs ccf (and rv vs ccf_fit)
sPlt.ccf_rv_ccf_plot(p, loc['RV_CCF'], normalized_ccf, ccf_fit)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# TODO: Needs doing
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# add to qc header lists
qc_values.append('None')
qc_names.append('None')
qc_logic.append('None')
qc_pass.append(1)
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ----------------------------------------------------------------------
# archive ccf to table
# ----------------------------------------------------------------------
# construct filename
res_table_file = spirouConfig.Constants.CCF_TABLE_FILE(p)
# log progress
WLOG(p, '', 'Archiving CCF on file {0}'.format(res_table_file))
# define column names
columns = ['order', 'maxcpp', 'nlines', 'contrast', 'RV', 'sig']
# define values for each column
values = [
loc['ORDERS'], loc['CCF_MAX'] / loc['PIX_PASSED_ALL'], loc['TOT_LINE'],
np.abs(100 * loc['CCF_ALL_RESULTS'][:, 0]),
loc['CCF_ALL_RESULTS'][:, 1], loc['CCF_ALL_RESULTS'][:, 2]
]
# define the format for each column
formats = ['2.0f', '5.0f', '4.0f', '4.1f', '9.4f', '7.4f']
# construct astropy table from column names, values and formats
table = spirouImage.MakeTable(p, columns, values, formats)
# save table to file
spirouImage.WriteTable(p, table, res_table_file, fmt='ascii')
# ----------------------------------------------------------------------
# archive ccf to fits file
# ----------------------------------------------------------------------
raw_infile = os.path.basename(p['E2DSFILE'])
# construct folder and filename
corfile, tag = spirouConfig.Constants.CCF_FITS_FILE(p)
corfilename = os.path.split(corfile)[-1]
# log that we are archiving the CCF on file
WLOG(p, '', 'Archiving CCF on file {0}'.format(corfilename))
# get constants from p
mask = p['CCF_MASK']
# if file exists remove it
if os.path.exists(corfile):
os.remove(corfile)
# add the average ccf to the end of ccf
data = np.vstack([loc['CCF'], loc['AVERAGE_CCF']])
# add drs keys
hdict = spirouImage.CopyOriginalKeys(hdr)
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
# set the input files
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBLAZE'], value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=loc['WAVEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=loc['WSOURCE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='file',
values=p['E2DSFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_INCCFMASK'],
value=p['CCF_MASK'])
hdict = spirouImage.AddKey(p, hdict, p['KW_INRV'], value=p['TARGET_RV'])
hdict = spirouImage.AddKey(p, hdict, p['KW_INWIDTH'], value=p['CCF_WIDTH'])
hdict = spirouImage.AddKey(p, hdict, p['KW_INSTEP'], value=p['CCF_STEP'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# add CCF keys
hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CTYPE'], value='km/s')
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CCF_CRVAL'],
value=loc['RV_CCF'][0])
# the rv step
rvstep = np.abs(loc['RV_CCF'][0] - loc['RV_CCF'][1])
hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_CDELT'], value=rvstep)
# add ccf stats
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CCF_RV'],
value=loc['CCF_RES'][1])
hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_RVC'], value=loc['RV'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_FWHM'], value=loc['FWHM'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CCF_WMREF'],
value=loc['WMEANREF'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CCF_CONTRAST'],
value=loc['CONTRAST'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CCF_MAXCPP'],
value=loc['MAXCPP'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CCF_MASK'], value=p['CCF_MASK'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CCF_LINES'],
value=np.nansum(loc['TOT_LINE']))
# add berv values
hdict = spirouImage.AddKey(p, hdict, p['KW_BERV'], value=loc['BERV'])
hdict = spirouImage.AddKey(p, hdict, p['KW_BJD'], value=loc['BJD'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_BERV_MAX'],
value=loc['BERV_MAX'])
# write image and add header keys (via hdict)
p = spirouImage.WriteImage(p, corfile, data, hdict)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(runname=None, quiet=False):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__, quiet=True)
# now get custom arguments
ckwargs = dict(positions=[0], types=[str], names=['RUNNAME'],
calls=[runname], require_night_name=False,
required=[False])
customargs = spirouStartup.GetCustomFromRuntime(p, **ckwargs)
# add custom args straight to p
p = spirouStartup.LoadMinimum(p, customargs=customargs)
# ----------------------------------------------------------------------
# Read the run file and extract parameters
# ----------------------------------------------------------------------
# construct filename
rfile = os.path.join(UNIT_TEST_PATH, p['RUNNAME'])
# check if RUNNAME is None
if p['RUNNAME'] == 'None':
exists = False
emsgs = ['No unit test run file defined.']
# check that rfile exists
elif not os.path.exists(rfile):
emsgs = ['Unit test run file "{0}" does not exist'.format(rfile)]
exists = False
else:
exists = True
emsgs = []
# deal with file wrong (or no file defined) --> print valid unit tests
if not exists:
emsgs.append('')
emsgs.append('Available units tests are:')
for rfile in os.listdir(UNIT_TEST_PATH):
emsgs.append('\t{0}'.format(rfile))
emsgs.append('')
emsgs.append('Located at {0}'.format(UNIT_TEST_PATH))
WLOG(p, 'error', emsgs)
# get the parameters in the run file
rparams = spirouConfig.GetConfigParams(p, filename=rfile)
# reset the DRS
if not quiet:
spirouTools.DRS_Reset(log=False, called=True)
# ----------------------------------------------------------------------
# Get runs
# ----------------------------------------------------------------------
runs = spirouUnitTests.get_runs(p, rparams, rfile)
# ----------------------------------------------------------------------
# group runs (for parallelisation)
# ----------------------------------------------------------------------
# get groups that can be run in parallel
groups = group_runs(runs)
# split groups by max number of processes
groups = parallelize(groups, MAX_PROCESSES)
# loop around groups
for group_name in groups:
# process storage
pp = []
# loop around sub groups (to be run at the same time)
for sub_group in groups[group_name]:
# make sub_group a dict
sruns = make_subgroup_dict(sub_group, group_name)
# do parallel run
process = Process(target=unit_wrapper, args=(p, sruns))
process.start()
pp.append(process)
# do not continue until
for process in pp:
while process.is_alive():
pass
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
wmsg = 'Recipe {0} has been successfully completed'
WLOG(p, 'info', wmsg.format(p['PROGRAM']))
# return a copy of locally defined variables in the memory
return dict(locals())
def dark_setup(night_name, files):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files)
p = spirouStartup.InitialFileSetup(p)
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='average')
# ----------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
data = spirouImage.FixNonPreProcess(p, data)
# ----------------------------------------------------------------------
# Get basic image properties
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# ----------------------------------------------------------------------
# Dark exposure time check
# ----------------------------------------------------------------------
# log the Dark exposure time
WLOG(p, 'info', 'Dark Time = {0:.3f} s'.format(p['EXPTIME']))
# Quality control: make sure the exposure time is longer than qc_dark_time
if p['EXPTIME'] < p['QC_DARK_TIME']:
emsg = 'Dark exposure time too short (< {0:.1f} s)'
WLOG(p, 'error', emsg.format(p['QC_DARK_TIME']))
# ----------------------------------------------------------------------
# Resize image
# ----------------------------------------------------------------------
# # rotate the image and conver from ADU/s to e-
# data = data[::-1, ::-1] * p['exptime'] * p['gain']
# convert NaN to zeros
nanmask = ~np.isfinite(data)
data = np.where(nanmask, np.zeros_like(data), data)
# resize blue image
bkwargs = dict(xlow=p['IC_CCDX_BLUE_LOW'],
xhigh=p['IC_CCDX_BLUE_HIGH'],
ylow=p['IC_CCDY_BLUE_LOW'],
yhigh=p['IC_CCDY_BLUE_HIGH'])
datablue, nx2, ny2 = spirouImage.ResizeImage(p, data, **bkwargs)
# Make sure we have data in the blue image
if nx2 == 0 or ny2 == 0:
WLOG(p, 'error', ('IC_CCD(X/Y)_BLUE_(LOW/HIGH) remove '
'all pixels from image.'))
# resize red image
rkwargs = dict(xlow=p['IC_CCDX_RED_LOW'],
xhigh=p['IC_CCDX_RED_HIGH'],
ylow=p['IC_CCDY_RED_LOW'],
yhigh=p['IC_CCDY_RED_HIGH'])
datared, nx3, ny3 = spirouImage.ResizeImage(p, data, **rkwargs)
# Make sure we have data in the red image
if nx3 == 0 or ny3 == 0:
WLOG(p, 'error', ('IC_CCD(X/Y)_RED_(LOW/HIGH) remove '
'all pixels from image.'))
# ----------------------------------------------------------------------
# Dark Measurement
# ----------------------------------------------------------------------
# Log that we are doing dark measurement
WLOG(p, '', 'Doing Dark measurement')
# measure dark for whole frame
p = spirouImage.MeasureDark(p, data, 'Whole det', 'full')
# measure dark for blue part
p = spirouImage.MeasureDark(p, datablue, 'Blue part', 'blue')
# measure dark for rede part
p = spirouImage.MeasureDark(p, datared, 'Red part', 'red')
# get stats
stats1 = [
data.size,
np.nansum(~np.isfinite(data)),
np.nanmedian(data),
np.nansum(~np.isfinite(data)) * 100 / np.product(data.shape),
p['DADEAD_FULL'], datablue.size,
np.nansum(~np.isfinite(datablue)),
np.nanmedian(datablue),
np.nansum(~np.isfinite(datablue)) * 100 / np.product(datablue.shape),
p['DADEAD_BLUE'], datared.size,
np.nansum(~np.isfinite(datared)),
np.nanmedian(datared),
np.nansum(~np.isfinite(datared)) * 100 / np.product(datared.shape),
p['DADEAD_RED']
]
return stats1
def main(night_name=None, files=None):
"""
cal_loc_RAW_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_loc_RAW_spirou.py [night_name] [files]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files)
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add')
# ----------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
data = spirouImage.FixNonPreProcess(p, data)
# ----------------------------------------------------------------------
# Get basic image properties
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# ----------------------------------------------------------------------
# Correction of DARK
# ----------------------------------------------------------------------
p, datac = spirouImage.CorrectForDark(p, data, hdr)
# ----------------------------------------------------------------------
# Interpolation over bad regions (to fill in the holes)
# ----------------------------------------------------------------------
# log process
# wmsg = 'Interpolating over bad regions'
# WLOG(p, '', wmsg)
# run interpolation
# datac = spirouImage.InterpolateBadRegions(p, datac)
# ----------------------------------------------------------------------
# Resize image
# ----------------------------------------------------------------------
# rotate the image and convert from ADU/s to e-
data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p)
# convert NaN to zeros
data0 = np.where(~np.isfinite(data), np.zeros_like(data), data)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'],
xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'],
yhigh=p['IC_CCDY_HIGH'],
getshape=False)
data2 = spirouImage.ResizeImage(p, data0, **bkwargs)
# log change in data size
WLOG(p, '', ('Image format changed to ' '{0}x{1}').format(*data2.shape))
# ----------------------------------------------------------------------
# Correct for the BADPIX mask (set all bad pixels to zero)
# ----------------------------------------------------------------------
p, data2 = spirouImage.CorrectForBadPix(p, data2, hdr)
# ----------------------------------------------------------------------
# Background computation
# ----------------------------------------------------------------------
if p['IC_DO_BKGR_SUBTRACTION']:
# log that we are doing background measurement
WLOG(p, '', 'Doing background measurement on raw frame')
# get the bkgr measurement
bargs = [p, data2, hdr]
# background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs)
p, background = spirouBACK.MeasureBackgroundMap(*bargs)
else:
background = np.zeros_like(data2)
p['BKGRDFILE'] = 'None'
p.set_source('BKGRDFILE', __NAME__ + '.main()')
# apply background correction to data
data2 = data2 - background
# ----------------------------------------------------------------------
# Construct image order_profile
# ----------------------------------------------------------------------
# log that we are doing background measurement
WLOG(p, '', 'Creating Order Profile')
order_profile = spirouLOCOR.BoxSmoothedImage(data2, p['LOC_BOX_SIZE'])
# data 2 is now set to the order profile
data2o = data2.copy()
data2 = order_profile.copy()
# ----------------------------------------------------------------------
# Write image order_profile to file
# ----------------------------------------------------------------------
# Construct folder and filename
rawfits, tag1 = spirouConfig.Constants.LOC_ORDER_PROFILE_FILE(p)
rawfitsname = os.path.split(rawfits)[-1]
# log saving order profile
wmsg = 'Saving processed raw frame in {0}'
WLOG(p, '', wmsg.format(rawfitsname))
# add keys from original header file
hdict = spirouImage.CopyOriginalKeys(hdr)
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
# write to file
p = spirouImage.WriteImage(p, rawfits, order_profile, hdict)
# ----------------------------------------------------------------------
# Move order_profile to calibDB and update calibDB
# ----------------------------------------------------------------------
# set key for calibDB
keydb = 'ORDER_PROFILE_{0}'.format(p['FIBER'])
# copy dark fits file to the calibDB folder
spirouDB.PutCalibFile(p, rawfits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, rawfitsname, hdr)
# ######################################################################
# Localization of orders on central column
# ######################################################################
# storage dictionary for localization parameters
loc = ParamDict()
# Plots setup: start interactive plot
if p['DRS_PLOT'] > 0:
sPlt.start_interactive_session(p)
# ----------------------------------------------------------------------
# Measurement and correction of background on the central column
# ----------------------------------------------------------------------
loc = spirouBACK.MeasureBkgrdGetCentPixs(p, loc, data2)
# ----------------------------------------------------------------------
# Search for order center on the central column - quick estimation
# ----------------------------------------------------------------------
# log progress
WLOG(p, '', 'Searching order center on central column')
# plot the minimum of ycc and ic_locseuil if in debug and plot mode
if p['DRS_DEBUG'] > 0 and p['DRS_PLOT']:
sPlt.debug_locplot_min_ycc_loc_threshold(p, loc['YCC'])
# find the central positions of the orders in the central
posc_all = spirouLOCOR.FindPosCentCol(loc['YCC'], p['IC_LOCSEUIL'])
# depending on the fiber type we may need to skip some pixels and also
# we need to add back on the ic_offset applied
start = p['IC_FIRST_ORDER_JUMP']
posc = posc_all[start:] + p['IC_OFFSET']
# work out the number of orders to use (minimum of ic_locnbmaxo and number
# of orders found in 'LocateCentralOrderPositions')
number_of_orders = np.min([p['IC_LOCNBMAXO'], len(posc)])
# log the number of orders than have been detected
wargs = [p['FIBER'], int(number_of_orders / p['NBFIB']), p['NBFIB']]
WLOG(p, 'info', ('On fiber {0} {1} orders have been detected '
'on {2} fiber(s)').format(*wargs))
# ----------------------------------------------------------------------
# Search for order center and profile on specific columns
# ----------------------------------------------------------------------
# get fit polynomial orders for position and width
fitpos, fitwid = p['IC_LOCDFITC'], p['IC_LOCDFITW']
# Create arrays to store position and width of order for each order
loc['CTRO'] = np.zeros((number_of_orders, data2.shape[1]), dtype=float)
loc['SIGO'] = np.zeros((number_of_orders, data2.shape[1]), dtype=float)
# Create arrays to store coefficients for position and width
loc['ACC'] = np.zeros((number_of_orders, fitpos + 1))
loc['ASS'] = np.zeros((number_of_orders, fitpos + 1))
# Create arrays to store rms values for position and width
loc['RMS_CENTER'] = np.zeros(number_of_orders)
loc['RMS_FWHM'] = np.zeros(number_of_orders)
# Create arrays to store point to point max value for position and width
loc['MAX_PTP_CENTER'] = np.zeros(number_of_orders)
loc['MAX_PTP_FRACCENTER'] = np.zeros(number_of_orders)
loc['MAX_PTP_FWHM'] = np.zeros(number_of_orders)
loc['MAX_PTP_FRACFWHM'] = np.zeros(number_of_orders)
# Create arrays to store rejected points
loc['MAX_RMPTS_POS'] = np.zeros(number_of_orders)
loc['MAX_RMPTS_WID'] = np.zeros(number_of_orders)
# set the central col centers in the cpos_orders array
loc['CTRO'][:, p['IC_CENT_COL']] = posc[0:number_of_orders]
# set source for all locs
loc.set_all_sources(__NAME__ + '/main()')
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# storage for plotting
loc['XPLOT'], loc['YPLOT'] = [], []
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# loop around each order
rorder_num = 0
for order_num in range(number_of_orders):
# find the row centers of the columns
loc = spirouLOCOR.FindOrderCtrs(p, data2, loc, order_num)
# only keep the orders with non-zero width
mask = loc['SIGO'][order_num, :] != 0
loc['X'] = np.arange(data2.shape[1])[mask]
loc.set_source('X', __NAME__ + '/main()')
# check that we have enough data points to fit data
if len(loc['X']) > (fitpos + 1):
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# initial fit params
iofargs = [p, loc, mask, rorder_num]
# initial fit for center positions for this order
loc, cf_data = spirouLOCOR.InitialOrderFit(*iofargs, kind='center')
# initial fit for widths for this order
loc, wf_data = spirouLOCOR.InitialOrderFit(*iofargs, kind='fwhm')
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# Log order number and fit at central pixel and width and rms
wargs = [
rorder_num, cf_data['cfitval'], wf_data['cfitval'],
cf_data['rms']
]
WLOG(p, '', ('ORDER: {0} center at pixel {1:.1f} width '
'{2:.1f} rms {3:.3f}').format(*wargs))
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# sigma fit params
sigfargs = [p, loc, cf_data, mask, order_num, rorder_num]
# sigma clip fit for center positions for this order
cf_data = spirouLOCOR.SigClipOrderFit(*sigfargs, kind='center')
# load results into storage arrags for this order
loc['ACC'][rorder_num] = cf_data['a']
loc['RMS_CENTER'][rorder_num] = cf_data['rms']
loc['MAX_PTP_CENTER'][rorder_num] = cf_data['max_ptp']
loc['MAX_PTP_FRACCENTER'][rorder_num] = cf_data['max_ptp_frac']
loc['MAX_RMPTS_POS'][rorder_num] = cf_data['max_rmpts']
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# sigma fit params
sigfargs = [p, loc, wf_data, mask, order_num, rorder_num]
# sigma clip fit for width positions for this order
wf_data = spirouLOCOR.SigClipOrderFit(*sigfargs, kind='fwhm')
# load results into storage arrags for this order
loc['ASS'][rorder_num] = wf_data['a']
loc['RMS_FWHM'][rorder_num] = wf_data['rms']
loc['MAX_PTP_FWHM'][rorder_num] = wf_data['max_ptp']
loc['MAX_PTP_FRACFWHM'][rorder_num] = wf_data['max_ptp_frac']
loc['MAX_RMPTS_WID'][rorder_num] = wf_data['max_rmpts']
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# increase the roder_num iterator
rorder_num += 1
# else log that the order is unusable
else:
WLOG(p, '', 'Order found too much incomplete, discarded')
# ----------------------------------------------------------------------
# Plot the image (ready for fit points to be overplotted later)
if p['DRS_PLOT'] > 0:
# get saturation threshold
satseuil = p['IC_SATSEUIL'] * p['GAIN'] * p['NBFRAMES']
# plot image above saturation threshold
sPlt.locplot_im_sat_threshold(p, loc, data2, satseuil)
# ----------------------------------------------------------------------
# Log that order geometry has been measured
WLOG(p, 'info', ('On fiber {0} {1} orders geometry have been '
'measured').format(p['FIBER'], rorder_num))
# Get mean rms
mean_rms_center = np.nansum(
loc['RMS_CENTER'][:rorder_num]) * 1000 / rorder_num
mean_rms_fwhm = np.nansum(loc['RMS_FWHM'][:rorder_num]) * 1000 / rorder_num
# Log mean rms values
wmsg = 'Average uncertainty on {0}: {1:.2f} [mpix]'
WLOG(p, 'info', wmsg.format('position', mean_rms_center))
WLOG(p, 'info', wmsg.format('width', mean_rms_fwhm))
# ----------------------------------------------------------------------
# Plot of RMS for positions and widths
# ----------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
sPlt.locplot_order_number_against_rms(p, loc, rorder_num)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# check that max number of points rejected in center fit is below threshold
if np.nansum(loc['MAX_RMPTS_POS']) > p['QC_LOC_MAXLOCFIT_REMOVED_CTR']:
fmsg = 'abnormal points rejection during ctr fit ({0:.2f} > {1:.2f})'
fail_msg.append(
fmsg.format(np.nansum(loc['MAX_RMPTS_POS']),
p['QC_LOC_MAXLOCFIT_REMOVED_CTR']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(np.nansum(loc['MAX_RMPTS_POS']))
qc_names.append('sum(MAX_RMPTS_POS)')
qc_logic.append('sum(MAX_RMPTS_POS) > {0:.2f}'
''.format(p['QC_LOC_MAXLOCFIT_REMOVED_CTR']))
# ----------------------------------------------------------------------
# check that max number of points rejected in width fit is below threshold
if np.nansum(loc['MAX_RMPTS_WID']) > p['QC_LOC_MAXLOCFIT_REMOVED_WID']:
fmsg = 'abnormal points rejection during width fit ({0:.2f} > {1:.2f})'
fail_msg.append(
fmsg.format(np.nansum(loc['MAX_RMPTS_WID']),
p['QC_LOC_MAXLOCFIT_REMOVED_WID']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(np.nansum(loc['MAX_RMPTS_WID']))
qc_names.append('sum(MAX_RMPTS_WID)')
qc_logic.append('sum(MAX_RMPTS_WID) > {0:.2f}'
''.format(p['QC_LOC_MAXLOCFIT_REMOVED_WID']))
# ----------------------------------------------------------------------
# check that the rms in center fit is lower than qc threshold
if mean_rms_center > p['QC_LOC_RMSMAX_CENTER']:
fmsg = 'too high rms on center fitting ({0:.2f} > {1:.2f})'
fail_msg.append(fmsg.format(mean_rms_center,
p['QC_LOC_RMSMAX_CENTER']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(mean_rms_center)
qc_names.append('mean_rms_center')
qc_logic.append('mean_rms_center > {0:.2f}'
''.format(p['QC_LOC_RMSMAX_CENTER']))
# ----------------------------------------------------------------------
# check that the rms in center fit is lower than qc threshold
if mean_rms_fwhm > p['QC_LOC_RMSMAX_FWHM']:
fmsg = 'too high rms on profile fwhm fitting ({0:.2f} > {1:.2f})'
fail_msg.append(fmsg.format(mean_rms_fwhm, p['QC_LOC_RMSMAX_CENTER']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(mean_rms_fwhm)
qc_names.append('mean_rms_fwhm')
qc_logic.append('mean_rms_fwhm > {0:.2f}'
''.format(p['QC_LOC_RMSMAX_CENTER']))
# ----------------------------------------------------------------------
# check for abnormal number of identified orders
if rorder_num != p['QC_LOC_NBO']:
fmsg = ('abnormal number of identified orders (found {0:.2f} '
'expected {1:.2f})')
fail_msg.append(fmsg.format(rorder_num, p['QC_LOC_NBO']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(rorder_num)
qc_names.append('rorder_num')
qc_logic.append('rorder_num != {0:.2f}'.format(p['QC_LOC_NBO']))
# ----------------------------------------------------------------------
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ----------------------------------------------------------------------
# Save and record of image of localization with order center and keywords
# ----------------------------------------------------------------------
raw_loco_file = os.path.basename(p['FITSFILENAME'])
# construct filename
locofits, tag2 = spirouConfig.Constants.LOC_LOCO_FILE(p)
locofitsname = os.path.split(locofits)[-1]
# log that we are saving localization file
WLOG(p, '', ('Saving localization information '
'in file: {0}').format(locofitsname))
# add keys from original header file
hdict = spirouImage.CopyOriginalKeys(hdr)
# define new keys to add
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=raw_loco_file)
hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_SIGDET'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_CONAD'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOCO_BCKGRD'],
value=loc['MEAN_BACKGRD'])
hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_NBO'], value=rorder_num)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOCO_DEG_C'],
value=p['IC_LOCDFITC'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOCO_DEG_W'],
value=p['IC_LOCDFITW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_E'])
hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DELTA'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_MAXFLX'],
value=float(loc['MAX_SIGNAL']))
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_SMAXPTS_CTR'],
value=np.nansum(loc['MAX_RMPTS_POS']))
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_SMAXPTS_WID'],
value=np.nansum(loc['MAX_RMPTS_WID']))
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_RMS_CTR'],
value=mean_rms_center)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_RMS_WID'],
value=mean_rms_fwhm)
# write 2D list of position fit coefficients
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_LOCO_CTR_COEFF'],
values=loc['ACC'][0:rorder_num])
# write 2D list of width fit coefficients
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_LOCO_FWHM_COEFF'],
values=loc['ASS'][0:rorder_num])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# write center fits and add header keys (via hdict)
center_fits = spirouLOCOR.CalcLocoFits(loc['ACC'], data2.shape[1])
p = spirouImage.WriteImage(p, locofits, center_fits, hdict)
# ----------------------------------------------------------------------
# Save and record of image of sigma
# ----------------------------------------------------------------------
# construct filename
locofits2, tag3 = spirouConfig.Constants.LOC_LOCO_FILE2(p)
locofits2name = os.path.split(locofits2)[-1]
# log that we are saving localization file
wmsg = 'Saving FWHM information in file: {0}'
WLOG(p, '', wmsg.format(locofits2name))
# add keys from original header file
hdict = spirouImage.CopyOriginalKeys(hdr)
# define new keys to add
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'], value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'], value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBACK'], value=p['BKGRDFILE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='file',
values=p['ARG_FILE_NAMES'])
# add outputs
hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_SIGDET'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CCD_CONAD'])
hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_NBO'], value=rorder_num)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOCO_DEG_C'],
value=p['IC_LOCDFITC'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOCO_DEG_W'],
value=p['IC_LOCDFITW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_LOCO_DEG_E'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_MAXFLX'],
value=float(loc['MAX_SIGNAL']))
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_SMAXPTS_CTR'],
value=np.nansum(loc['MAX_RMPTS_POS']))
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_SMAXPTS_WID'],
value=np.nansum(loc['MAX_RMPTS_WID']))
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_RMS_CTR'],
value=mean_rms_center)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOC_RMS_WID'],
value=mean_rms_fwhm)
# write 2D list of position fit coefficients
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_LOCO_CTR_COEFF'],
values=loc['ACC'][0:rorder_num])
# write 2D list of width fit coefficients
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_LOCO_FWHM_COEFF'],
values=loc['ASS'][0:rorder_num])
# add quality control
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
# write image and add header keys (via hdict)
width_fits = spirouLOCOR.CalcLocoFits(loc['ASS'], data2.shape[1])
p = spirouImage.WriteImage(p, locofits2, width_fits, hdict)
# ----------------------------------------------------------------------
# Save and Record of image of localization
# ----------------------------------------------------------------------
if p['IC_LOCOPT1']:
# construct filename
locofits3, tag4 = spirouConfig.Constants.LOC_LOCO_FILE3(p)
locofits3name = os.path.split(locofits3)[-1]
# log that we are saving localization file
wmsg1 = 'Saving localization image with superposition of orders in '
wmsg2 = 'file: {0}'.format(locofits3name)
WLOG(p, '', [wmsg1, wmsg2])
# superpose zeros over the fit in the image
data4 = spirouLOCOR.ImageLocSuperimp(data2o, loc['ACC'][0:rorder_num])
# save this image to file
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_FIBER'], value=p['FIBER'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBDARK'],
value=p['DARKFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBBAD'],
value=p['BADPFILE'])
p = spirouImage.WriteImage(p, locofits3, data4, hdict)
# ----------------------------------------------------------------------
# Update the calibration database
# ----------------------------------------------------------------------
if p['QC'] == 1:
keydb = 'LOC_' + p['FIBER']
# copy localisation file to the calibDB folder
spirouDB.PutCalibFile(p, locofits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, locofitsname, hdr)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(cores=1, objects=None, filetype='EXT_E2DS_FF_AB'):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# set up function name
main_name = __NAME__ + '.main()'
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# now get custom arguments
pos, fmt = [0, 1, 2], [int, str, str]
names = ['cores', 'objects', 'filetype']
call = [cores, objects, filetype]
required = [False, False, False]
customargs = spirouStartup.GetCustomFromRuntime(p,
pos,
fmt,
names,
calls=call,
required=required,
require_night_name=False)
p = spirouStartup.LoadArguments(p,
None,
customargs=customargs,
mainfitsdir='reduced',
require_night_name=False)
# keep track of errors
errors = []
# -------------------------------------------------------------------------
# find all objects (matching 'objects' if not None)
target_list = spirouTelluric.FindObjects(p)
# print number found
nfound = len(target_list)
nstar = 0
for target in list(target_list.keys()):
nstar += len(target_list[target])
wmsg = 'Found target "{0}" ({1} observations total)'
WLOG(p, 'info', wmsg.format(target, len(target_list[target])))
# list total found
wmsg = 'Found {0} observations total for {1} object(s)'
WLOG(p, 'info', wmsg.format(nstar, nfound))
# -------------------------------------------------------------------------
# Step 1: Run fit_tellu on all objects
# -------------------------------------------------------------------------
# for t_it, target in enumerate(list(target_list.keys())):
# # loop around object filenames
# for o_it, objfilename in enumerate(target_list[target]):
# # Log progress
# pargs = [p, 'Fit Tellurics', target, t_it, nfound,
# o_it, len(target_list[target])]
# spirouTelluric.UpdateProcessDB(*pargs)
# # get arguments from filename
# args = spirouTelluric.GetDBarguments(p, objfilename)
# # run obj_mk_tellu
# try:
# obj_fit_tellu.main(**args)
# except SystemExit as e:
# errors.append([pargs[1], objfilename, e])
# # force close all plots
# sPlt.closeall()
# -------------------------------------------------------------------------
# Step 2: Run mk_obj_template on each science target
# -------------------------------------------------------------------------
for t_it, target in enumerate(list(target_list.keys())):
# log progress (big)
pmsg = 'Make Telluric Template: Processing object = {0} ({1}/{2}'
wmsgs = [
'', '=' * 60, '',
pmsg.format(target, t_it + 1, nfound), '', '=' * 60, ''
]
WLOG(p, 'info', wmsgs, wrap=False)
# get last object
objfilename = target_list[target][-1]
# get arguments from filename
args = spirouTelluric.GetDBarguments(p, objfilename)
# run obj_mk_obj_template
try:
obj_mk_obj_template.main(**args)
except SystemExit as e:
errors.append(['Telluric Template', target, e])
# force close all plots
sPlt.closeall()
# -------------------------------------------------------------------------
# Step 3: Re-Run fit_tellu on all objects
# -------------------------------------------------------------------------
for t_it, target in enumerate(list(target_list.keys())):
# loop around object filenames
for o_it, objfilename in enumerate(target_list[target]):
# Log progress
pargs = [
p, 'Fit Tellurics II', target, t_it, nfound, o_it,
len(target_list[target])
]
spirouTelluric.UpdateProcessDB(*pargs)
# get arguments from filename
args = spirouTelluric.GetDBarguments(p, objfilename)
# run obj_mk_tellu
try:
obj_fit_tellu.main(**args)
except SystemExit as e:
errors.append([pargs[1], objfilename, e])
# force close all plots
sPlt.closeall()
# -------------------------------------------------------------------------
# Step 4: Re-Run mk_obj_template on each science target
# -------------------------------------------------------------------------
for t_it, target in enumerate(list(target_list.keys())):
# log progress (big)
pmsg = 'Make Telluric Template II: Processing object = {0} ({1}/{2}'
wmsgs = [
'', '=' * 60, '',
pmsg.format(target, t_it + 1, nfound), '', '=' * 60, ''
]
WLOG(p, 'info', wmsgs, wrap=False)
# get last object
objfilename = target_list[target][-1]
# get arguments from filename
args = spirouTelluric.GetDBarguments(p, objfilename)
# run obj_mk_obj_template
try:
obj_mk_obj_template.main(**args)
except SystemExit as e:
errors.append(['Telluric Template', target, e])
# force close all plots
sPlt.closeall()
# ----------------------------------------------------------------------
# Print all errors
# ----------------------------------------------------------------------
if len(errors) > 0:
emsgs = ['', '=' * 50, 'Errors were as follows: ']
# loop around errors
for error in errors:
emsgs.append('')
emsgs.append('{0}: Object = {1}'.format(error[0], error[1]))
emsgs.append('\t{0}'.format(error[2]))
emsgs.append('')
WLOG(p, 'error', emsgs)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# get parameters from configuration files and run time arguments
customargs = spirouStartup.GetCustomFromRuntime(p, [0], [str], ['reffile'])
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsfile='reffile',
mainfitsdir='reduced')
# setup files and get fiber
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# set the fiber type
p['FIB_TYP'] = [p['FIBER']]
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
gfkwargs = dict(path=p['REDUCED_DIR'], filename=p['REFFILE'])
p['REFFILENAME'] = spirouStartup.GetFile(p, **gfkwargs)
p.set_source('REFFILENAME', __NAME__ + '/main()')
# get the fiber type
p['FIBER'] = 'AB'
e2ds, hdr, nx, ny = spirouImage.ReadImage(p)
# Force A and B to AB solution
if p['FIBER'] in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = p['FIBER']
# get wave image
_, wave, _ = spirouImage.GetWaveSolution(p,
hdr=hdr,
return_wavemap=True,
fiber=wave_fiber)
blaze = spirouImage.ReadBlazeFile(p)
# ----------------------------------------------------------------------
# Get lamp params
# ----------------------------------------------------------------------
# get lamp parameters
p = spirouTHORCA.GetLampParams(p, hdr)
# ----------------------------------------------------------------------
# Get catalogue and fitted line list
# ----------------------------------------------------------------------
# load line file (from p['IC_LL_LINE_FILE'])
ll_line_cat, ampl_line_cat = spirouImage.ReadLineList(p)
# construct fitted lines table filename
wavelltbl = spirouConfig.Constants.WAVE_LINE_FILE(p)
WLOG(p, '', wavelltbl)
# read fitted lines
ll_ord, ll_line_fit, ampl_line_fit = np.genfromtxt(wavelltbl,
skip_header=4,
skip_footer=2,
unpack=True,
usecols=(0, 1, 3))
# ----------------------------------------------------------------------
# Plots
# ----------------------------------------------------------------------
# define line colours
col = ['magenta', 'purple']
# get order parity
ll_ord_par = np.mod(ll_ord, 2)
print(ll_ord_par)
col2 = [col[int(x)] for x in ll_ord_par]
# start interactive plot
sPlt.start_interactive_session(p)
plt.figure()
for order_num in np.arange(nx):
plt.plot(wave[order_num], e2ds[order_num])
# get heights
heights = []
for line in range(len(ll_line_cat)):
heights.append(200000 + np.max([np.min(e2ds), ampl_line_cat[line]]))
# plot ll_line_cat
plt.vlines(ll_line_cat,
0,
heights,
colors='darkgreen',
linestyles='dashed')
# get heights
heights = []
for line in range(len(ll_line_fit)):
heights.append(200000 + np.max([np.min(e2ds), ampl_line_fit[line]]))
# plot ll_line_fit
plt.vlines(ll_line_fit, 0, heights, colors=col2, linestyles='dashdot')
plt.xlabel('Wavelength [nm]')
plt.ylabel('Flux e-')
plt.title(p['REFFILENAME'])
# end interactive session
# sPlt.end_interactive_session()
# old code:
# plt.ion()
# plt.figure()
#
# for order_num in np.arange(nx):
# plt.plot(wave[order_num], e2ds[order_num])
#
# for line in range(len(ll_line_cat)):
# plt.vlines(ll_line_cat[line], 0, 200000 +
# max(np.min(e2ds), ampl_line_cat[line]),
# colors='darkgreen', linestyles='dashed')
#
# for line in range(len(ll_line_fit)):
# plt.vlines(ll_line_fit[line], 0, 200000 +
# max(np.min(e2ds), ampl_line_fit[line]),
# colors='magenta', linestyles='dashdot')
#
# plt.xlabel('Wavelength [nm]')
# plt.ylabel('Flux e-')
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p, outputs=None)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None, fiber_type=None, **kwargs):
"""
cal_DRIFT_E2DS_spirou.py main function, if night_name and files are
None uses arguments from run time i.e.:
cal_DRIFT_E2DS_spirou.py [night_directory] [files]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:param fiber_type: string, if None does all fiber types (defined in
constants_SPIROU FIBER_TYPES (default is AB, A, B, C
if defined then only does this fiber type (but must
be in FIBER_TYPES)
:param kwargs: any keyword to overwrite constant in param dict "p"
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files)
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# deal with fiber type
if fiber_type is None:
fiber_type = p['FIBER_TYPES']
if type(fiber_type) == str:
if fiber_type.upper() == 'ALL':
fiber_type = p['FIBER_TYPES']
elif fiber_type in p['FIBER_TYPES']:
fiber_type = [fiber_type]
else:
emsg = 'fiber_type="{0}" not understood'
WLOG(p, 'error', emsg.format(fiber_type))
# set fiber type
p['FIB_TYPE'] = fiber_type
p.set_source('FIB_TYPE', __NAME__ + '__main__()')
# Overwrite keys from source
for kwarg in kwargs:
p[kwarg] = kwargs[kwarg]
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add')
# ----------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
data = spirouImage.FixNonPreProcess(p, data)
# ----------------------------------------------------------------------
# Get basic image properties
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# set sigdet and conad keywords (sigdet is changed later)
p['KW_CCD_SIGDET'][1] = p['SIGDET']
p['KW_CCD_CONAD'][1] = p['GAIN']
# now change the value of sigdet if require
if p['IC_EXT_SIGDET'] > 0:
p['SIGDET'] = float(p['IC_EXT_SIGDET'])
# get DPRTYPE from header (Will have it if valid)
p = spirouImage.ReadParam(p, hdr, 'KW_DPRTYPE', required=False, dtype=str)
# check the DPRTYPE is not None
if (p['DPRTYPE'] == 'None') or (['DPRTYPE'] is None):
emsg1 = 'Error: {0} is not set in header for file {1}'
eargs = [p['KW_DPRTYPE'][0], p['FITSFILENAME']]
emsg2 = '\tPlease run pre-processing on file.'
emsg3 = ('\tIf pre-processing fails or skips file, file is not '
'currrently as valid DRS fits file.')
WLOG(p, 'error', [emsg1.format(*eargs), emsg2, emsg3])
else:
p['DPRTYPE'] = p['DPRTYPE'].strip()
# ----------------------------------------------------------------------
# Correction of DARK
# ----------------------------------------------------------------------
p, datac = spirouImage.CorrectForDark(p, data, hdr)
# ----------------------------------------------------------------------
# Resize image
# ----------------------------------------------------------------------
# rotate the image and convert from ADU/s to ADU
data = spirouImage.ConvertToADU(spirouImage.FlipImage(p, datac), p=p)
# convert NaN to zeros
data0 = np.where(~np.isfinite(data), np.zeros_like(data), data)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'],
xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'],
yhigh=p['IC_CCDY_HIGH'],
getshape=False)
data1 = spirouImage.ResizeImage(p, data0, **bkwargs)
# log change in data size
wmsg = 'Image format changed to {1}x{0}'
WLOG(p, '', wmsg.format(*data1.shape))
# ----------------------------------------------------------------------
# Correct for the BADPIX mask (set all bad pixels to zero)
# ----------------------------------------------------------------------
p, data1 = spirouImage.CorrectForBadPix(p, data1, hdr)
# ----------------------------------------------------------------------
# Log the number of dead pixels
# ----------------------------------------------------------------------
# get the number of bad pixels
n_bad_pix = np.sum(~np.isfinite(data1))
n_bad_pix_frac = n_bad_pix * 100 / np.product(data1.shape)
# Log number
wmsg = 'Nb dead pixels = {0} / {1:.4f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# ----------------------------------------------------------------------
# Get the miny, maxy and max_signal for the central column
# ----------------------------------------------------------------------
# get the central column
y = data1[p['IC_CENT_COL'], :]
# get the min max and max signal using box smoothed approach
miny, maxy, max_signal, diff_maxmin = spirouBACK.MeasureMinMaxSignal(p, y)
# Log max average flux/pixel
wmsg = 'Maximum average flux/pixel in the spectrum: {0:.1f} [ADU]'
WLOG(p, 'info', wmsg.format(max_signal / p['NBFRAMES']))
# ----------------------------------------------------------------------
# Background computation
# ----------------------------------------------------------------------
if p['IC_DO_BKGR_SUBTRACTION']:
# log that we are doing background measurement
WLOG(p, '', 'Doing background measurement on raw frame')
# get the bkgr measurement
bargs = [p, data1, hdr]
# background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs)
p, background = spirouBACK.MeasureBackgroundMap(*bargs)
else:
background = np.zeros_like(data1)
p['BKGRDFILE'] = 'None'
p.set_source('BKGRDFILE', __NAME__ + '.main()')
# apply background correction to data (and set to zero where negative)
data1 = data1 - background
# ----------------------------------------------------------------------
# Read tilt slit angle
# ----------------------------------------------------------------------
# define loc storage parameter dictionary
loc = ParamDict()
# get tilts (if the mode requires it)
if p['IC_EXTRACT_TYPE'] not in EXTRACT_SHAPE_TYPES:
p, loc['TILT'] = spirouImage.ReadTiltFile(p, hdr)
loc.set_source('TILT',
__NAME__ + '/main() + /spirouImage.ReadTiltFile')
else:
loc['TILT'] = None
loc.set_source('TILT', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Earth Velocity calculation
# ----------------------------------------------------------------------
if p['IC_IMAGE_TYPE'] == 'H4RG':
p, loc = spirouImage.GetEarthVelocityCorrection(p, loc, hdr)
# ----------------------------------------------------------------------
# Get all fiber data (for all fibers)
# ----------------------------------------------------------------------
# TODO: This is temp solution for options 5a and 5b
loc_fibers = spirouLOCOR.GetFiberData(p, hdr)
# ------------------------------------------------------------------
# Deal with debananafication
# ------------------------------------------------------------------
# if mode 4a or 4b we need to straighten in x only
if p['IC_EXTRACT_TYPE'] in ['4a', '4b']:
# get the shape parameters
p, shapem_x = spirouImage.GetShapeX(p, hdr)
p, shape_local = spirouImage.GetShapeLocal(p, hdr)
# log progress
WLOG(p, '', 'Debananafying (straightening) image')
# apply shape transforms
targs = dict(lin_transform_vect=shape_local, dxmap=shapem_x)
data2 = spirouImage.EATransform(data1, **targs)
# if mode 5a or 5b we need to straighten in x and y using the
# polynomial fits for location
elif p['IC_EXTRACT_TYPE'] in ['5a', '5b']:
# get the shape parameters
p, shapem_x = spirouImage.GetShapeX(p, hdr)
p, shapem_y = spirouImage.GetShapeY(p, hdr)
p, shape_local = spirouImage.GetShapeLocal(p, hdr)
p, fpmaster = spirouImage.GetFPMaster(p, hdr)
# get the bad pixel map
bkwargs = dict(return_map=True, quiet=True)
p, badpix = spirouImage.CorrectForBadPix(p, data1, hdr, **bkwargs)
# log progress
WLOG(p, '', 'Cleaning image')
# clean the image
data1 = spirouEXTOR.CleanHotpix(data1, badpix)
# log progress
WLOG(p, '', 'Debananafying (straightening) image')
# apply shape transforms
targs = dict(lin_transform_vect=shape_local,
dxmap=shapem_x,
dymap=shapem_y)
data2 = spirouImage.EATransform(data1, **targs)
# in any other mode we do not straighten
else:
data2 = np.array(data1)
# ----------------------------------------------------------------------
# Fiber loop
# ----------------------------------------------------------------------
# loop around fiber types
for fiber in p['FIB_TYPE']:
# set fiber
p['FIBER'] = fiber
p.set_source('FIBER', __NAME__ + '/main()()')
# ------------------------------------------------------------------
# Read wavelength solution
# ------------------------------------------------------------------
# set source of wave file
wsource = __NAME__ + '/main() + /spirouImage.GetWaveSolution'
# Force A and B to AB solution
if fiber in ['A', 'B']:
wave_fiber = 'AB'
else:
wave_fiber = fiber
# get wave image
wkwargs = dict(hdr=hdr,
return_wavemap=True,
return_filename=True,
return_header=True,
fiber=wave_fiber)
wout = spirouImage.GetWaveSolution(p, **wkwargs)
loc['WAVEPARAMS'], loc['WAVE'], loc['WAVEFILE'] = wout[:3]
loc['WAVEHDR'], loc['WSOURCE'] = wout[3:]
source_names = ['WAVE', 'WAVEFILE', 'WAVEPARAMS', 'WAVEHDR']
loc.set_sources(source_names, wsource)
# get dates
loc['WAVE_ACQTIMES'] = spirouDB.GetTimes(p, loc['WAVEHDR'])
loc.set_source('WAVE_ACQTIMES', __NAME__ + '.main()')
# get the recipe that produced the wave solution
if 'WAVECODE' in loc['WAVEHDR']:
loc['WAVE_CODE'] = loc['WAVEHDR']['WAVECODE']
else:
loc['WAVE_CODE'] = 'UNKNOWN'
loc.set_source('WAVE_CODE', __NAME__ + '.main()')
# ----------------------------------------------------------------------
# Get WFP keys
# ----------------------------------------------------------------------
# Read the WFP keys - if they don't exist set to None and deal
# with later
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_DRIFT',
name='WFP_DRIFT',
required=False)
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_FWHM',
name='WFP_FWHM',
required=False)
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_CONTRAST',
name='WFP_CONTRAST',
required=False)
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_MAXCPP',
name='WFP_MAXCPP',
required=False)
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_MASK',
name='WFP_MASK',
required=False)
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_LINES',
name='WFP_LINES',
required=False)
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_TARG_RV',
name='WFP_TARG_RV',
required=False)
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_WIDTH',
name='WFP_WIDTH',
required=False)
p = spirouImage.ReadParam(p,
loc['WAVEHDR'],
'KW_WFP_STEP',
name='WFP_STEP',
required=False)
# ----------------------------------------------------------------------
# Read Flat file
# ----------------------------------------------------------------------
fout = spirouImage.ReadFlatFile(p, hdr, return_header=True)
p, loc['FLAT'], flathdr = fout
loc.set_source('FLAT',
__NAME__ + '/main() + /spirouImage.ReadFlatFile')
# get flat extraction mode
if p['KW_E2DS_EXTM'][0] in flathdr:
flat_ext_mode = flathdr[p['KW_E2DS_EXTM'][0]]
else:
flat_ext_mode = None
# ------------------------------------------------------------------
# Check extraction method is same as flat extraction method
# ------------------------------------------------------------------
# get extraction method and function
extmethod, extfunc = spirouEXTOR.GetExtMethod(p, p['IC_EXTRACT_TYPE'])
if not DEBUG:
# compare flat extraction mode to extraction mode
spirouEXTOR.CompareExtMethod(p, flat_ext_mode, extmethod, 'FLAT',
'EXTRACTION')
# ------------------------------------------------------------------
# Read Blaze file
# ------------------------------------------------------------------
p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hdr)
blazesource = __NAME__ + '/main() + /spirouImage.ReadBlazeFile'
loc.set_source('BLAZE', blazesource)
# ------------------------------------------------------------------
# Get fiber specific parameters from loc_fibers
# ------------------------------------------------------------------
# get this fibers parameters
p = spirouImage.FiberParams(p, p['FIBER'], merge=True)
# get localisation parameters
for key in loc_fibers[fiber]:
loc[key] = loc_fibers[fiber][key]
loc.set_source(key, loc_fibers[fiber].sources[key])
# get locofile source
p['LOCOFILE'] = loc['LOCOFILE']
p.set_source('LOCOFILE', loc.sources['LOCOFILE'])
# get the order_profile
order_profile = loc_fibers[fiber]['ORDER_PROFILE']
# ------------------------------------------------------------------
# Set up Extract storage
# ------------------------------------------------------------------
# Create array to store extraction (for each order and each pixel
# along order)
loc['E2DS'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
loc['E2DSFF'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
loc['E2DSLL'] = []
loc['SPE1'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
loc['SPE3'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
loc['SPE4'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
loc['SPE5'] = np.zeros((loc['NUMBER_ORDERS'], data2.shape[1]))
# Create array to store the signal to noise ratios for each order
loc['SNR'] = np.zeros(loc['NUMBER_ORDERS'])
# ------------------------------------------------------------------
# Extract orders
# ------------------------------------------------------------------
# source for parameter dictionary
source = __NAME__ + '/main()'
# get limits of order extraction
valid_orders = spirouEXTOR.GetValidOrders(p, loc)
# loop around each order
for order_num in valid_orders:
# -------------------------------------------------------------
# IC_EXTRACT_TYPE decides the extraction routine
# -------------------------------------------------------------
eargs = [p, loc, data2, order_num]
ekwargs = dict(mode=p['IC_EXTRACT_TYPE'],
order_profile=order_profile)
with warnings.catch_warnings(record=True) as w:
eout = spirouEXTOR.Extraction(*eargs, **ekwargs)
# deal with different return
if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES:
e2ds, e2dsll, cpt = eout
else:
e2ds, cpt = eout
e2dsll = None
# -------------------------------------------------------------
# calculate the noise
range1, range2 = p['IC_EXT_RANGE1'], p['IC_EXT_RANGE2']
# set the noise
noise = p['SIGDET'] * np.sqrt(range1 + range2)
# get window size
blaze_win1 = int(data2.shape[0] / 2) - p['IC_EXTFBLAZ']
blaze_win2 = int(data2.shape[0] / 2) + p['IC_EXTFBLAZ']
# get average flux per pixel
flux = np.nansum(
e2ds[blaze_win1:blaze_win2]) / (2 * p['IC_EXTFBLAZ'])
# calculate signal to noise ratio = flux/sqrt(flux + noise^2)
snr = flux / np.sqrt(flux + noise**2)
# log the SNR RMS
wmsg = 'On fiber {0} order {1}: S/N= {2:.1f} Nbcosmic= {3}'
wargs = [p['FIBER'], order_num, snr, cpt]
WLOG(p, '', wmsg.format(*wargs))
# add calculations to storage
loc['E2DS'][order_num] = e2ds
loc['E2DSFF'][order_num] = e2ds / loc['FLAT'][order_num]
loc['SNR'][order_num] = snr
# save the longfile
if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES:
loc['E2DSLL'].append(e2dsll)
# set sources
loc.set_sources(['e2ds', 'SNR'], source)
# Log if saturation level reached
satvalue = (flux / p['GAIN']) / (range1 + range2)
if satvalue > (p['QC_LOC_FLUMAX'] * p['NBFRAMES']):
wmsg = 'SATURATION LEVEL REACHED on Fiber {0} order={1}'
WLOG(p, 'warning', wmsg.format(fiber, order_num))
# ------------------------------------------------------------------
# Thermal correction
# ------------------------------------------------------------------
# get fiber type
if fiber in ['AB', 'A', 'B']:
fibertype = p['DPRTYPE'].split('_')[0]
else:
fibertype = p['DPRTYPE'].split('_')[1]
# apply thermal correction based on fiber type
if fibertype in p['THERMAL_CORRECTION_TYPE1']:
# log progress
wmsg = 'Correcting thermal background for {0}={1} mode={2}'
wargs = [fiber, fibertype, 1]
WLOG(p, 'info', wmsg.format(*wargs))
# correct E2DS
tkwargs = dict(image=loc['E2DS'], mode=1, fiber=fiber, hdr=hdr)
p, loc['E2DS'] = spirouBACK.ThermalCorrect(p, **tkwargs)
# correct E2DSFF
tkwargs = dict(image=loc['E2DSFF'],
mode=1,
fiber=fiber,
hdr=hdr,
flat=loc['FLAT'])
p, loc['E2DSFF'] = spirouBACK.ThermalCorrect(p, **tkwargs)
elif fibertype in p['THERMAL_CORRECTION_TYPE2']:
# log progress
wmsg = 'Correcting thermal background for {0}={1} mode={2}'
wargs = [fiber, fibertype, 2]
WLOG(p, 'info', wmsg.format(*wargs))
# correct E2DS
tkwargs = dict(image=loc['E2DS'], mode=2, fiber=fiber, hdr=hdr)
p, loc['E2DS'] = spirouBACK.ThermalCorrect(p, **tkwargs)
# correct E2DSFF
tkwargs = dict(image=loc['E2DSFF'],
mode=2,
fiber=fiber,
hdr=hdr,
flat=loc['FLAT'])
p, loc['E2DSFF'] = spirouBACK.ThermalCorrect(p, **tkwargs)
else:
# log progress
wmsg = 'Not correcting thermal background for {0}={1}'
wargs = [fiber, fibertype]
WLOG(p, 'info', wmsg.format(*wargs))
# set filename for output
outfile = 'THERMALFILE_{0}'.format(fiber)
p[outfile] = 'None'
p.set_source(outfile, __NAME__ + '.main()')
# ------------------------------------------------------------------
# Plots
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# start interactive session if needed
sPlt.start_interactive_session(p)
# plot all orders or one order
if p['IC_FF_PLOT_ALL_ORDERS']:
# plot image with all order fits (slower)
sPlt.ext_aorder_fit(p, loc, data1, max_signal / 10.)
else:
# plot image with selected order fit and edge fit (faster)
sPlt.ext_sorder_fit(p, loc, data1, max_signal / 10.)
# plot e2ds against wavelength
sPlt.ext_spectral_order_plot(p, loc)
if p['IC_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES:
sPlt.ext_debanana_plot(p, loc, data2, max_signal / 10.)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# if array is completely NaNs it shouldn't pass
if np.sum(np.isfinite(loc['E2DS'])) == 0:
fail_msg.append('E2DS image is all NaNs')
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append('NaN')
qc_names.append('image')
qc_logic.append('image is all NaN')
# ----------------------------------------------------------------------
# saturation check: check that the max_signal is lower than
# qc_max_signal
max_qcflux = p['QC_MAX_SIGNAL'] * p['NBFRAMES']
if max_signal > max_qcflux:
fmsg = 'Too much flux in the image ({0:.2f} > {1:.2f})'
fail_msg.append(fmsg.format(max_signal, max_qcflux))
passed = False
# Question: Why is this test ignored?
# For some reason this test is ignored in old code
passed = True
WLOG(p, 'info', fail_msg[-1])
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(max_signal)
qc_names.append('max_signal')
qc_logic.append('QC_MAX_SIGNAL > {0:.3f}'.format(max_qcflux))
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ------------------------------------------------------------------
# Store extraction in file(s)
# ------------------------------------------------------------------
raw_ext_file = os.path.basename(p['FITSFILENAME'])
# construct filename
e2dsfits, tag1 = spirouConfig.Constants.EXTRACT_E2DS_FILE(p)
e2dsfitsname = os.path.split(e2dsfits)[-1]
e2dsfffits, tag2 = spirouConfig.Constants.EXTRACT_E2DSFF_FILE(p)
e2dsfffitsname = os.path.split(e2dsfffits)[-1]
e2dsllfits, tag4 = spirouConfig.Constants.EXTRACT_E2DSLL_FILE(p)
e2dsfllitsname = os.path.split(e2dsllfits)[-1]
# log that we are saving E2DS spectrum
wmsg = 'Saving E2DS spectrum of Fiber {0} in {1}'
WLOG(p, '', wmsg.format(p['FIBER'], e2dsfitsname))
wmsg = 'Saving E2DSFF spectrum of Fiber {0} in {1}'
WLOG(p, '', wmsg.format(p['FIBER'], e2dsfffitsname))
# add keys from original header file
hdict = spirouImage.CopyOriginalKeys(hdr)
# set the version
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_FIBER'], value=p['FIBER'])
# set the input files
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBDARK'],
value=p['DARKFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBBAD'],
value=p['BADPFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBLOCO'],
value=p['LOCOFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBBACK'],
value=p['BKGRDFILE'])
if p['IC_EXTRACT_TYPE'] not in EXTRACT_SHAPE_TYPES:
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBTILT'],
value=p['TILTFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBBLAZE'],
value=p['BLAZFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBFLAT'],
value=p['FLATFILE'])
if p['IC_EXTRACT_TYPE'] in EXTRACT_SHAPE_TYPES:
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBSHAPEX'],
value=p['SHAPEXFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBSHAPEY'],
value=p['SHAPEYFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBSHAPE'],
value=p['SHAPEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBFPMASTER'],
value=p['FPMASTERFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBTHERMAL'],
value=p['THERMALFILE_{0}'.format(fiber)])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_CDBWAVE'],
value=loc['WAVEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVESOURCE'],
value=loc['WSOURCE'])
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='file',
values=p['ARG_FILE_NAMES'])
# construct loco filename
locofile, _ = spirouConfig.Constants.EXTRACT_LOCO_FILE(p)
locofilename = os.path.basename(locofile)
# add barycentric keys to header
hdict = spirouImage.AddKey(p, hdict, p['KW_BERV'], value=loc['BERV'])
hdict = spirouImage.AddKey(p, hdict, p['KW_BJD'], value=loc['BJD'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_BERV_MAX'],
value=loc['BERV_MAX'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_B_OBS_HOUR'],
value=loc['BERVHOUR'])
# add barycentric estimate keys to header
hdict = spirouImage.AddKey(p,
hdict,
p['KW_BERV_EST'],
value=loc['BERV_EST'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_BJD_EST'],
value=loc['BJD_EST'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_BERV_MAX_EST'],
value=loc['BERV_MAX_EST'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_BERV_SOURCE'],
value=loc['BERV_SOURCE'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# copy extraction method and function to header
# (for reproducibility)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_E2DS_EXTM'],
value=extmethod)
hdict = spirouImage.AddKey(p, hdict, p['KW_E2DS_FUNC'], value=extfunc)
# add localization file name to header
hdict = spirouImage.AddKey(p,
hdict,
p['KW_LOCO_FILE'],
value=locofilename)
# add wave solution date
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_TIME1'],
value=loc['WAVE_ACQTIMES'][0])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_TIME2'],
value=loc['WAVE_ACQTIMES'][1])
# add wave solution number of orders
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_ORD_N'],
value=loc['WAVEPARAMS'].shape[0])
# add wave solution degree of fit
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WAVE_LL_DEG'],
value=loc['WAVEPARAMS'].shape[1] - 1)
# -------------------------------------------------------------------------
# add keys of the wave solution FP CCF
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_FILE'],
value=loc['WAVEFILE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_DRIFT'],
value=p['WFP_DRIFT'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_FWHM'],
value=p['WFP_FWHM'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_CONTRAST'],
value=p['WFP_CONTRAST'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_MAXCPP'],
value=p['WFP_MAXCPP'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_MASK'],
value=p['WFP_MASK'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_LINES'],
value=p['WFP_LINES'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_TARG_RV'],
value=p['WFP_TARG_RV'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_WIDTH'],
value=p['WFP_WIDTH'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_WFP_STEP'],
value=p['WFP_STEP'])
# write 1D list of the SNR
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_E2DS_SNR'],
values=loc['SNR'])
# add localization file keys to header
root = p['KW_ROOT_DRS_LOC'][0]
hdict = spirouImage.CopyRootKeys(p, hdict, locofile, root=root)
# add wave solution coefficients
hdict = spirouImage.AddKey2DList(p,
hdict,
p['KW_WAVE_PARAM'],
values=loc['WAVEPARAMS'])
# Save E2DS file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag1)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_EXT_TYPE'],
value=p['DPRTYPE'])
p = spirouImage.WriteImage(p, e2dsfits, loc['E2DS'], hdict)
# Save E2DSFF file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag2)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_EXT_TYPE'],
value=p['DPRTYPE'])
p = spirouImage.WriteImage(p, e2dsfffits, loc['E2DSFF'], hdict)
# Save E2DSLL file
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_EXT_TYPE'],
value=p['DPRTYPE'])
if p['IC_EXTRACT_TYPE'] in EXTRACT_LL_TYPES:
llstack = np.vstack(loc['E2DSLL'])
p = spirouImage.WriteImage(p, e2dsllfits, llstack, hdict)
# ------------------------------------------------------------------
# 1-dimension spectral S1D (uniform in wavelength)
# ------------------------------------------------------------------
# get arguments for E2DS to S1D
e2dsargs = [loc['WAVE'], loc['E2DSFF'], loc['BLAZE']]
# get 1D spectrum
xs1d1, ys1d1 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='wave')
# Plot the 1D spectrum
if p['DRS_PLOT'] > 0:
sPlt.ext_1d_spectrum_plot(p, xs1d1, ys1d1)
# construct file name
s1dfile1, tag3 = spirouConfig.Constants.EXTRACT_S1D_FILE1(p)
s1dfilename1 = os.path.basename(s1dfile1)
# add header keys
# set the version
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag3)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_EXT_TYPE'],
value=p['DPRTYPE'])
# log writing to file
wmsg = 'Saving 1D spectrum (uniform in wavelength) for Fiber {0} in {1}'
WLOG(p, '', wmsg.format(p['FIBER'], s1dfilename1))
# Write to file
columns = ['wavelength', 'flux', 'eflux']
values = [xs1d1, ys1d1, np.zeros_like(ys1d1)]
units = ['nm', None, None]
s1d1 = spirouImage.MakeTable(p, columns, values, units=units)
spirouImage.WriteTable(p, s1d1, s1dfile1, header=hdict)
# ------------------------------------------------------------------
# 1-dimension spectral S1D (uniform in velocity)
# ------------------------------------------------------------------
# get arguments for E2DS to S1D
e2dsargs = [loc['WAVE'], loc['E2DSFF'], loc['BLAZE']]
# get 1D spectrum
xs1d2, ys1d2 = spirouImage.E2DStoS1D(p, *e2dsargs, wgrid='velocity')
# Plot the 1D spectrum
if p['DRS_PLOT'] > 0:
sPlt.ext_1d_spectrum_plot(p, xs1d2, ys1d2)
# construct file name
s1dfile2, tag4 = spirouConfig.Constants.EXTRACT_S1D_FILE2(p)
s1dfilename2 = os.path.basename(s1dfile2)
# add header keys
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DRS_DATE'],
value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DATE_NOW'],
value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag4)
hdict = spirouImage.AddKey(p,
hdict,
p['KW_EXT_TYPE'],
value=p['DPRTYPE'])
# log writing to file
wmsg = 'Saving 1D spectrum (uniform in velocity) for Fiber {0} in {1}'
WLOG(p, '', wmsg.format(p['FIBER'], s1dfilename2))
# Write to file
columns = ['wavelength', 'flux', 'eflux']
values = [xs1d2, ys1d2, np.zeros_like(ys1d2)]
units = ['nm', None, None]
s1d2 = spirouImage.MakeTable(p, columns, values, units=units)
spirouImage.WriteTable(p, s1d2, s1dfile2, header=hdict)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
"""
cal_DARK_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_DARK_spirou.py [night_directory] [fitsfilename]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files)
p = spirouStartup.InitialFileSetup(p)
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='average')
# ----------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
data = spirouImage.FixNonPreProcess(p, data)
# ----------------------------------------------------------------------
# Get basic image properties
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# ----------------------------------------------------------------------
# Dark exposure time check
# ----------------------------------------------------------------------
# log the Dark exposure time
WLOG(p, 'info', 'Dark Time = {0:.3f} s'.format(p['EXPTIME']))
# Quality control: make sure the exposure time is longer than qc_dark_time
if p['EXPTIME'] < p['QC_DARK_TIME']:
emsg = 'Dark exposure time too short (< {0:.1f} s)'
WLOG(p, 'error', emsg.format(p['QC_DARK_TIME']))
# ----------------------------------------------------------------------
# Resize image
# ----------------------------------------------------------------------
# # rotate the image and conver from ADU/s to e-
# data = data[::-1, ::-1] * p['exptime'] * p['gain']
# convert NaN to zeros
nanmask = ~np.isfinite(data)
data0 = np.where(nanmask, np.zeros_like(data), data)
# resize blue image
bkwargs = dict(xlow=p['IC_CCDX_BLUE_LOW'],
xhigh=p['IC_CCDX_BLUE_HIGH'],
ylow=p['IC_CCDY_BLUE_LOW'],
yhigh=p['IC_CCDY_BLUE_HIGH'])
datablue, nx2, ny2 = spirouImage.ResizeImage(p, data, **bkwargs)
# Make sure we have data in the blue image
if nx2 == 0 or ny2 == 0:
WLOG(p, 'error', ('IC_CCD(X/Y)_BLUE_(LOW/HIGH) remove '
'all pixels from image.'))
# resize red image
rkwargs = dict(xlow=p['IC_CCDX_RED_LOW'],
xhigh=p['IC_CCDX_RED_HIGH'],
ylow=p['IC_CCDY_RED_LOW'],
yhigh=p['IC_CCDY_RED_HIGH'])
datared, nx3, ny3 = spirouImage.ResizeImage(p, data, **rkwargs)
# Make sure we have data in the red image
if nx3 == 0 or ny3 == 0:
WLOG(p, 'error', ('IC_CCD(X/Y)_RED_(LOW/HIGH) remove '
'all pixels from image.'))
# ----------------------------------------------------------------------
# Dark Measurement
# ----------------------------------------------------------------------
# Log that we are doing dark measurement
WLOG(p, '', 'Doing Dark measurement')
# measure dark for whole frame
p = spirouImage.MeasureDark(p, data, 'Whole det', 'full')
# measure dark for blue part
p = spirouImage.MeasureDark(p, datablue, 'Blue part', 'blue')
# measure dark for rede part
p = spirouImage.MeasureDark(p, datared, 'Red part', 'red')
# ----------------------------------------------------------------------
# Identification of bad pixels
# ----------------------------------------------------------------------
# get number of bad dark pixels (as a fraction of total pixels)
with warnings.catch_warnings(record=True) as w:
baddark = 100.0 * np.sum(data0 > p['DARK_CUTLIMIT'])
baddark /= np.product(data0.shape)
# log the fraction of bad dark pixels
wmsg = 'Frac pixels with DARK > {0:.2f} ADU/s = {1:.3f} %'
WLOG(p, 'info', wmsg.format(p['DARK_CUTLIMIT'], baddark))
# define mask for values above cut limit or NaN
with warnings.catch_warnings(record=True) as w:
datacutmask = ~((data0 > p['DARK_CUTLIMIT']) | (~np.isfinite(data)))
spirouCore.spirouLog.warninglogger(p, w)
# get number of pixels above cut limit or NaN
n_bad_pix = np.product(data.shape) - np.nansum(datacutmask)
# work out fraction of dead pixels + dark > cut, as percentage
p['DADEADALL'] = n_bad_pix * 100 / np.product(data.shape)
p.set_source('DADEADALL', __NAME__ + '/main()')
# log fraction of dead pixels + dark > cut
logargs = [p['DARK_CUTLIMIT'], p['DADEADALL']]
WLOG(p, 'info', ('Total Frac dead pixels (N.A.N) + DARK > '
'{0:.2f} ADU/s = {1:.3f} %').format(*logargs))
# ----------------------------------------------------------------------
# Plots
# ----------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# start interactive plot
sPlt.start_interactive_session(p)
# plot the image with blue and red regions
sPlt.darkplot_image_and_regions(p, data)
# plot histograms
sPlt.darkplot_histograms(p)
# end interactive session
sPlt.end_interactive_session(p)
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# ----------------------------------------------------------------------
# check that med < qc_max_darklevel
if p['MED_FULL'] > p['QC_MAX_DARKLEVEL']:
# add failed message to fail message list
fmsg = 'Unexpected Median Dark level ({0:5.2f} > {1:5.2f} ADU/s)'
fail_msg.append(fmsg.format(p['MED_FULL'], p['QC_MAX_DARKLEVEL']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(p['MED_FULL'])
qc_names.append('MED_FULL')
qc_logic.append('MED_FULL > {0:.2f}'.format(p['QC_MAX_DARKLEVEL']))
# ----------------------------------------------------------------------
# check that fraction of dead pixels < qc_max_dead
if p['DADEADALL'] > p['QC_MAX_DEAD']:
# add failed message to fail message list
fmsg = 'Unexpected Fraction of dead pixels ({0:5.2f} > {1:5.2f} %)'
fail_msg.append(fmsg.format(p['DADEADALL'], p['QC_MAX_DEAD']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(p['DADEADALL'])
qc_names.append('DADEADALL')
qc_logic.append('DADEADALL > {0:.2f}'.format(p['QC_MAX_DEAD']))
# ----------------------------------------------------------------------
# checl that the precentage of dark pixels < qc_max_dark
if baddark > p['QC_MAX_DARK']:
fmsg = ('Unexpected Fraction of dark pixels > {0:.2f} ADU/s '
'({1:.2f} > {2:.2f}')
fail_msg.append(
fmsg.format(p['DARK_CUTLIMIT'], baddark, p['QC_MAX_DARK']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(baddark)
qc_names.append('baddark')
qc_logic.append('baddark > {0:.2f}'.format(p['QC_MAX_DARK']))
# ----------------------------------------------------------------------
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ----------------------------------------------------------------------
# Save dark to file
# ----------------------------------------------------------------------
# get raw dark filename
rawdarkfile = os.path.basename(p['FITSFILENAME'])
# construct folder and filename
darkfits, tag = spirouConfig.Constants.DARK_FILE(p)
darkfitsname = os.path.basename(darkfits)
# log saving dark frame
WLOG(p, '', 'Saving Dark frame in ' + darkfitsname)
# add keys from original header file
hdict = spirouImage.CopyOriginalKeys(hdr)
# define new keys to add
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey1DList(p,
hdict,
p['KW_INFILE1'],
dim1name='file',
values=p['ARG_FILE_NAMES'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DARK_DEAD'],
value=p['DADEAD_FULL'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DARK_MED'], value=p['MED_FULL'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DARK_B_DEAD'],
value=p['DADEAD_BLUE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DARK_B_MED'],
value=p['MED_BLUE'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DARK_R_DEAD'],
value=p['DADEAD_RED'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DARK_R_MED'],
value=p['MED_RED'])
hdict = spirouImage.AddKey(p,
hdict,
p['KW_DARK_CUT'],
value=p['DARK_CUTLIMIT'])
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# Set to zero dark value > dark_cutlimit
cutmask = data0 > p['DARK_CUTLIMIT']
data0c = np.where(cutmask, np.zeros_like(data0), data0)
# write image and add header keys (via hdict)
p = spirouImage.WriteImage(p, darkfits, data0c, hdict)
# ----------------------------------------------------------------------
# Save bad pixel mask
# ----------------------------------------------------------------------
# construct bad pixel file name
badpixelfits, tag = spirouConfig.Constants.DARK_BADPIX_FILE(p)
badpixelfitsname = os.path.split(badpixelfits)[-1]
# log that we are saving bad pixel map in dir
WLOG(p, '', 'Saving Bad Pixel Map in ' + badpixelfitsname)
# add keys from original header file
hdict = spirouImage.CopyOriginalKeys(hdr)
# define new keys to add
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
hdict['DACUT'] = (p['DARK_CUTLIMIT'],
'Threshold of dark level retain [ADU/s]')
# write to file
datacutmask = np.array(datacutmask, dtype=float)
p = spirouImage.WriteImage(p,
badpixelfits,
datacutmask,
hdict,
dtype='float64')
# ----------------------------------------------------------------------
# Move to calibDB and update calibDB
# ----------------------------------------------------------------------
if p['QC']:
# set dark key
if p['DPRTYPE'] == 'DARK_DARK':
keydb = 'DARK'
elif p['USE_SKYDARK_CORRECTION']:
keydb = 'SKYDARK'
else:
emsg = 'Error: Currently {0} only supports DARK_DARK and OBJ_DARK'
WLOG(p, 'error', emsg.format(__NAME__))
# copy dark fits file to the calibDB folder
spirouDB.PutCalibFile(p, darkfits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, darkfitsname, hdr)
# # set badpix key
# keydb = 'BADPIX_OLD'
# # copy badpix fits file to calibDB folder
# spirouDB.PutCalibFile(p, badpixelfits)
# # update the master calib DB file with new key
# spirouDB.UpdateCalibMaster(p, keydb, badpixelfitsname, hdr)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, ufile=None, xsize=None, ysize=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
# need custom args (to accept full path or wild card
if ufile is None or xsize is None or ysize is None:
pos = [0, 1, 2]
names, types = ['ufile', 'xsize', 'ysize'], [str, int, int]
customargs = spirouStartup.GetCustomFromRuntime(p,
pos,
types,
names,
last_multi=True)
else:
customargs = dict(ufile=ufile, xsize=xsize, ysize=ysize)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p, night_name, customargs=customargs)
# add constants not currently in constants file
# ------------------------------------------------------------------
# Read image file
# ------------------------------------------------------------------
# read the image data
rout = spirouImage.ReadImage(p, filename=p['UFILE'])
image, hdr, nx, ny = rout
# ----------------------------------------------------------------------
# un-Resize image
# ----------------------------------------------------------------------
# create an array of given size
size = np.product([p['YSIZE'], p['XSIZE']])
newimage = np.repeat(np.nan, size).reshape(p['YSIZE'], p['XSIZE'])
# insert image at given pixels
xlow, xhigh = p['IC_CCDX_LOW'], p['IC_CCDX_HIGH']
ylow, yhigh = p['IC_CCDY_LOW'], p['IC_CCDY_HIGH']
newimage[ylow:yhigh, xlow:xhigh] = image
# rotate the image
newimage = spirouImage.FlipImage(p, newimage)
# ------------------------------------------------------------------
# Save rotated image
# ------------------------------------------------------------------
# construct rotated file name
outfits = ufile.replace('.fits', '_old.fits')
outfitsname = os.path.split(outfits)[-1]
# log that we are saving rotated image
WLOG(p, '', 'Saving Rotated Image in ' + outfitsname)
# add keys from original header file
hdict = spirouImage.CopyOriginalKeys(hdr)
# write to file
p = spirouImage.WriteImage(p, outfits, newimage, hdict)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
wmsg = 'Recipe {0} has been successfully completed'
WLOG(p, 'info', wmsg.format(p['PROGRAM']))
# return a copy of locally defined variables in the memory
return dict(locals())
def main(night_name=None, files=None):
"""
cal_SLIT_spirou.py main function, if night_name and files are None uses
arguments from run time i.e.:
cal_SLIT_spirou.py [night_directory] [files]
:param night_name: string or None, the folder within data raw directory
containing files (also reduced directory) i.e.
/data/raw/20170710 would be "20170710" but
/data/raw/AT5/20180409 would be "AT5/20180409"
:param files: string, list or None, the list of files to use for
arg_file_names and fitsfilename
(if None assumes arg_file_names was set from run time)
:return ll: dictionary, containing all the local variables defined in
main
"""
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p, night_name, files)
p = spirouStartup.InitialFileSetup(p, calibdb=True)
# set the fiber type
p['FIB_TYP'] = 'AB'
p.set_source('FIB_TYP', __NAME__ + '/main()')
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read the image data
p, data, hdr = spirouImage.ReadImageAndCombine(p, framemath='add')
# ----------------------------------------------------------------------
# fix for un-preprocessed files
# ----------------------------------------------------------------------
data = spirouImage.FixNonPreProcess(p, data)
# ----------------------------------------------------------------------
# Get basic image properties
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hdr, name='gain')
# ----------------------------------------------------------------------
# Correction of DARK
# ----------------------------------------------------------------------
p, datac = spirouImage.CorrectForDark(p, data, hdr)
# ----------------------------------------------------------------------
# Resize image
# ----------------------------------------------------------------------
# rotate the image and convert from ADU/s to e-
data = spirouImage.ConvertToE(spirouImage.FlipImage(p, datac), p=p)
# convert NaN to zeros
data0 = np.where(~np.isfinite(data), np.zeros_like(data), data)
# resize image
bkwargs = dict(xlow=p['IC_CCDX_LOW'], xhigh=p['IC_CCDX_HIGH'],
ylow=p['IC_CCDY_LOW'], yhigh=p['IC_CCDY_HIGH'],
getshape=False)
data2 = spirouImage.ResizeImage(p, data0, **bkwargs)
# log change in data size
WLOG(p, '', ('Image format changed to '
'{0}x{1}').format(*data2.shape))
# ----------------------------------------------------------------------
# Correct for the BADPIX mask (set all bad pixels to zero)
# ----------------------------------------------------------------------
p, data2 = spirouImage.CorrectForBadPix(p, data2, hdr)
# ----------------------------------------------------------------------
# Background computation
# ----------------------------------------------------------------------
if p['IC_DO_BKGR_SUBTRACTION']:
# log that we are doing background measurement
WLOG(p, '', 'Doing background measurement on raw frame')
# get the bkgr measurement
bargs = [p, data2, hdr]
# background, xc, yc, minlevel = spirouBACK.MeasureBackgroundFF(*bargs)
p, background = spirouBACK.MeasureBackgroundMap(*bargs)
else:
background = np.zeros_like(data2)
p['BKGRDFILE'] = 'None'
p.set_source('BKGRDFILE', __NAME__ + '.main()')
# correct data2 with background
data2 = data2 - background
# ----------------------------------------------------------------------
# Log the number of dead pixels
# ----------------------------------------------------------------------
# get the number of bad pixels
n_bad_pix = np.nansum(~np.isfinite(data2))
n_bad_pix_frac = n_bad_pix * 100 / np.product(data2.shape)
# Log number
wmsg = 'Nb dead pixels = {0} / {1:.2f} %'
WLOG(p, 'info', wmsg.format(int(n_bad_pix), n_bad_pix_frac))
# ----------------------------------------------------------------------
# Log the number of dead pixels
# ----------------------------------------------------------------------
loc = ParamDict()
# ----------------------------------------------------------------------
# Loop around fiber types
# ----------------------------------------------------------------------
# set fiber
p['FIBER'] = p['FIB_TYP']
# ------------------------------------------------------------------
# Get localisation coefficients
# ------------------------------------------------------------------
# original there is a loop but it is not used --> removed
p = spirouImage.FiberParams(p, p['FIBER'], merge=True)
# get localisation fit coefficients
p, loc = spirouLOCOR.GetCoeffs(p, hdr, loc)
# ------------------------------------------------------------------
# Calculating the tilt
# ------------------------------------------------------------------
# get the tilt by extracting the AB fibers and correlating them
loc = spirouImage.GetTilt(p, loc, data2)
# fit the tilt with a polynomial
loc = spirouImage.FitTilt(p, loc)
# log the tilt dispersion
wmsg = 'Tilt dispersion = {0:.3f} deg'
WLOG(p, 'info', wmsg.format(loc['RMS_TILT']))
# ------------------------------------------------------------------
# Plotting
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# plots setup: start interactive plot
sPlt.start_interactive_session(p)
# plot image with selected order shown
sPlt.slit_sorder_plot(p, loc, data2)
# plot slit tilt angle and fit
sPlt.slit_tilt_angle_and_fit_plot(p, loc)
# end interactive section
sPlt.end_interactive_session(p)
# ------------------------------------------------------------------
# Replace tilt by the global fit
# ------------------------------------------------------------------
loc['TILT'] = loc['YFIT_TILT']
oldsource = loc.get_source('tilt')
loc.set_source('TILT', oldsource + '+{0}/main()'.format(__NAME__))
# ----------------------------------------------------------------------
# Quality control
# ----------------------------------------------------------------------
# set passed variable and fail message list
passed, fail_msg = True, []
qc_values, qc_names, qc_logic, qc_pass = [], [], [], []
# check that tilt rms is below required
if loc['RMS_TILT'] > p['QC_SLIT_RMS']:
# add failed message to fail message list
fmsg = 'abnormal RMS of SLIT angle ({0:.2f} > {1:.2f} deg)'
fail_msg.append(fmsg.format(loc['RMS_TILT'], p['QC_SLIT_RMS']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(loc['RMS_TILT'])
qc_names.append('RMS_TILT')
qc_logic.append('RMS_TILT > {0:.2f}'.format(p['QC_SLIT_RMS']))
# ----------------------------------------------------------------------
# check that tilt is less than max tilt required
max_tilt = np.max(loc['TILT'])
if max_tilt > p['QC_SLIT_MAX']:
# add failed message to fail message list
fmsg = 'abnormal SLIT angle ({0:.2f} > {1:.2f} deg)'
fail_msg.append(fmsg.format(max_tilt, p['QC_SLIT_MAX']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(max_tilt)
qc_names.append('max_tilt')
qc_logic.append('max_tilt > {0:.2f}'.format(p['QC_SLIT_MAX']))
# ----------------------------------------------------------------------
# check that tilt is greater than min tilt required
min_tilt = np.min(loc['TILT'])
if min_tilt < p['QC_SLIT_MIN']:
# add failed message to fail message list
fmsg = 'abnormal SLIT angle ({0:.2f} < {1:.2f} deg)'
fail_msg.append(fmsg.format(max_tilt, p['QC_SLIT_MIN']))
passed = False
qc_pass.append(0)
else:
qc_pass.append(1)
# add to qc header lists
qc_values.append(min_tilt)
qc_names.append('min_tilt')
qc_logic.append('min_tilt > {0:.2f}'.format(p['QC_SLIT_MIN']))
# ----------------------------------------------------------------------
# finally log the failed messages and set QC = 1 if we pass the
# quality control QC = 0 if we fail quality control
if passed:
WLOG(p, 'info', 'QUALITY CONTROL SUCCESSFUL - Well Done -')
p['QC'] = 1
p.set_source('QC', __NAME__ + '/main()')
else:
for farg in fail_msg:
wmsg = 'QUALITY CONTROL FAILED: {0}'
WLOG(p, 'warning', wmsg.format(farg))
p['QC'] = 0
p.set_source('QC', __NAME__ + '/main()')
# store in qc_params
qc_params = [qc_names, qc_values, qc_logic, qc_pass]
# ----------------------------------------------------------------------
# Save and record of tilt table
# ----------------------------------------------------------------------
# copy the tilt along the orders
tiltima = np.ones((int(loc['NUMBER_ORDERS']/2), data2.shape[1]))
tiltima *= loc['TILT'][:, None]
# get the raw tilt file name
raw_tilt_file = os.path.basename(p['FITSFILENAME'])
# construct file name and path
tiltfits, tag = spirouConfig.Constants.SLIT_TILT_FILE(p)
tiltfitsname = os.path.basename(tiltfits)
# Log that we are saving tilt file
wmsg = 'Saving tilt information in file: {0}'
WLOG(p, '', wmsg.format(tiltfitsname))
# Copy keys from fits file
hdict = spirouImage.CopyOriginalKeys(hdr)
# add version number
hdict = spirouImage.AddKey(p, hdict, p['KW_VERSION'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_DATE'], value=p['DRS_DATE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_DATE_NOW'], value=p['DATE_NOW'])
hdict = spirouImage.AddKey(p, hdict, p['KW_PID'], value=p['PID'])
hdict = spirouImage.AddKey(p, hdict, p['KW_OUTPUT'], value=tag)
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBDARK'],
value=p['DARKFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBAD'],
value=p['BADPFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBLOCO'], value=p['LOCOFILE'])
hdict = spirouImage.AddKey(p, hdict, p['KW_CDBBACK'],
value=p['BKGRDFILE'])
hdict = spirouImage.AddKey1DList(p, hdict, p['KW_INFILE1'], dim1name='file',
values=p['ARG_FILE_NAMES'])
# add qc parameters
# add qc parameters
hdict = spirouImage.AddKey(p, hdict, p['KW_DRS_QC'], value=p['QC'])
hdict = spirouImage.AddQCKeys(p, hdict, qc_params)
# add tilt parameters as 1d list
hdict = spirouImage.AddKey1DList(p, hdict, p['KW_TILT'], values=loc['TILT'])
# write tilt file to file
p = spirouImage.WriteImage(p, tiltfits, tiltima, hdict)
# ----------------------------------------------------------------------
# Update the calibration data base
# ----------------------------------------------------------------------
if p['QC']:
keydb = 'TILT'
# copy localisation file to the calibDB folder
spirouDB.PutCalibFile(p, tiltfits)
# update the master calib DB file with new key
spirouDB.UpdateCalibMaster(p, keydb, tiltfitsname, hdr)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
from SpirouDRS import spirouCore
from SpirouDRS import spirouImage
from SpirouDRS import spirouStartup
from debanananificator import *
# FP file for tilt/dx determination
slope_file = '2295305a_pp.fits'
outname = (slope_file.split('_'))[0] + '_dxmap.fits'
hdr_wave = pyfits.getheader("2295305a_pp_e2dsff_AB.fits")
# width of the ABC fibres.
wpix = 55
__NAME__ = 'anything'
p = spirouStartup.Begin(recipe=__NAME__)
p = spirouStartup.LoadArguments(p)
n_ord = hdr_wave["LONBO"] // 2
n_coeff = len(hdr_wave['LOFW[0-9]*']) // (n_ord * 2)
ordfit = 5
poly_c = np.zeros([n_coeff, n_ord * 2])
LOCTR = hdr_wave['LOCTR*']
for ord in range(0, n_ord * 2, 2):
for coeff in range(ordfit):
i = (ord * n_coeff) + coeff
poly_c[i % n_coeff, i // n_coeff] = hdr_wave["LOCTR" + str(i)]
poly_c = poly_c[:, poly_c[0, :] != 0]
def main(night_name=None, fpfile=None, hcfiles=None):
# ----------------------------------------------------------------------
# Set up
# ----------------------------------------------------------------------
# get parameters from config files/run time args/load paths + calibdb
p = spirouStartup.Begin(recipe=__NAME__)
if hcfiles is None or fpfile is None:
names, types = ['fpfile', 'hcfiles'], [str, str]
customargs = spirouStartup.GetCustomFromRuntime(p, [0, 1],
types,
names,
last_multi=True)
else:
customargs = dict(hcfiles=hcfiles, fpfile=fpfile)
# get parameters from configuration files and run time arguments
p = spirouStartup.LoadArguments(p,
night_name,
customargs=customargs,
mainfitsdir='reduced',
mainfitsfile='hcfiles')
# ----------------------------------------------------------------------
# Construct reference filename and get fiber type
# ----------------------------------------------------------------------
p, fpfitsfilename = spirouStartup.SingleFileSetup(p, filename=p['FPFILE'])
fiber1 = str(p['FIBER'])
p, hcfilenames = spirouStartup.MultiFileSetup(p, files=p['HCFILES'])
fiber2 = str(p['FIBER'])
# set the hcfilename to the first hcfilenames
hcfitsfilename = hcfilenames[0]
# ----------------------------------------------------------------------
# Once we have checked the e2dsfile we can load calibDB
# ----------------------------------------------------------------------
# as we have custom arguments need to load the calibration database
p = spirouStartup.LoadCalibDB(p)
# ----------------------------------------------------------------------
# Have to check that the fibers match
# ----------------------------------------------------------------------
if fiber1 == fiber2:
p['FIBER'] = fiber1
fsource = __NAME__ + '/main() & spirouStartup.GetFiberType()'
p.set_source('FIBER', fsource)
else:
emsg = 'Fiber not matching for {0} and {1}, should be the same'
eargs = [hcfitsfilename, fpfitsfilename]
WLOG(p, 'error', emsg.format(*eargs))
# set the fiber type
p['FIB_TYP'] = [p['FIBER']]
p.set_source('FIB_TYP', __NAME__ + '/main()')
# set find line mode
find_lines_mode = p['HC_FIND_LINES_MODE']
# ----------------------------------------------------------------------
# Read image file
# ----------------------------------------------------------------------
# read and combine all HC files except the first (fpfitsfilename)
rargs = [p, 'add', hcfitsfilename, hcfilenames[1:]]
p, hcdata, hchdr = spirouImage.ReadImageAndCombine(*rargs)
# read first file (fpfitsfilename)
fpdata, fphdr, _, _ = spirouImage.ReadImage(p, fpfitsfilename)
# add data and hdr to loc
loc = ParamDict()
loc['HCDATA'], loc['HCHDR'] = hcdata, hchdr
loc['FPDATA'], loc['FPHDR'] = fpdata, fphdr
# set the source
sources = ['HCDATA', 'HCHDR']
loc.set_sources(sources, 'spirouImage.ReadImageAndCombine()')
sources = ['FPDATA', 'FPHDR']
loc.set_sources(sources, 'spirouImage.ReadImage()')
# ----------------------------------------------------------------------
# Get basic image properties for reference file
# ----------------------------------------------------------------------
# get sig det value
p = spirouImage.GetSigdet(p, hchdr, name='sigdet')
# get exposure time
p = spirouImage.GetExpTime(p, hchdr, name='exptime')
# get gain
p = spirouImage.GetGain(p, hchdr, name='gain')
# get acquisition time
p = spirouImage.GetAcqTime(p, hchdr, name='acqtime', kind='julian')
bjdref = p['ACQTIME']
# set sigdet and conad keywords (sigdet is changed later)
p['KW_CCD_SIGDET'][1] = p['SIGDET']
p['KW_CCD_CONAD'][1] = p['GAIN']
# get lamp parameters
p = spirouTHORCA.GetLampParams(p, hchdr)
# ----------------------------------------------------------------------
# Obtain the flat
# ----------------------------------------------------------------------
# get the flat
p, loc = spirouFLAT.GetFlat(p, loc, hchdr)
# ----------------------------------------------------------------------
# Read blaze
# ----------------------------------------------------------------------
# get tilts
p, loc['BLAZE'] = spirouImage.ReadBlazeFile(p, hchdr)
loc.set_source('BLAZE', __NAME__ + '/main() + /spirouImage.ReadBlazeFile')
# correct the data with the flat
# TODO: Should this be used?
# log
# WLOG(p, '', 'Applying flat correction')
# loc['HCDATA'] = loc['HCDATA']/loc['FLAT']
# loc['FPDATA'] = loc['FPDATA']/loc['FLAT']
# ----------------------------------------------------------------------
# Start plotting session
# ----------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# start interactive plot
sPlt.start_interactive_session(p)
# ----------------------------------------------------------------------
# loop around fiber type
# ----------------------------------------------------------------------
for fiber in p['FIB_TYP']:
# set fiber type for inside loop
p['FIBER'] = fiber
# ------------------------------------------------------------------
# Instrumental drift computation (if previous solution exists)
# ------------------------------------------------------------------
# get key
keydb = 'HCREF_{0}'.format(p['FIBER'])
# check for key in calibDB
if keydb in p['CALIBDB'].keys():
# log process
wmsg = ('Doing Instrumental drift computation from previous '
'calibration')
WLOG(p, '', wmsg)
# calculate instrument drift
loc = spirouTHORCA.CalcInstrumentDrift(p, loc)
# ------------------------------------------------------------------
# Wave solution
# ------------------------------------------------------------------
# log message for loop
wmsg = 'Processing Wavelength Calibration for Fiber {0}'
WLOG(p, 'info', wmsg.format(p['FIBER']))
# ------------------------------------------------------------------
# Part 1 of cal_HC
# ------------------------------------------------------------------
p, loc = cal_HC_E2DS_spirou.part1(p, loc, mode=find_lines_mode)
# ------------------------------------------------------------------
# FP solution
# ------------------------------------------------------------------
# log message
wmsg = 'Calculating FP wave solution'
WLOG(p, '', wmsg)
# calculate FP wave solution
# spirouTHORCA.FPWaveSolution(p, loc, mode=find_lines_mode)
spirouTHORCA.FPWaveSolutionNew(p, loc)
# ------------------------------------------------------------------
# FP solution plots
# ------------------------------------------------------------------
if p['DRS_PLOT'] > 0:
# Plot the FP extracted spectrum against wavelength solution
sPlt.wave_plot_final_fp_order(p, loc, iteration=1)
# Plot the measured FP cavity width offset against line number
sPlt.wave_local_width_offset_plot(p, loc)
# Plot the FP line wavelength residuals
sPlt.wave_fp_wavelength_residuals(p, loc)
# ------------------------------------------------------------------
# Part 2 of cal_HC
# ------------------------------------------------------------------
# set params for part2
p['QC_RMS_LITTROW_MAX'] = p['QC_WAVE_RMS_LITTROW_MAX']
p['QC_DEV_LITTROW_MAX'] = p['QC_WAVE_DEV_LITTROW_MAX']
p['IC_HC_N_ORD_START_2'] = min(p['IC_HC_N_ORD_START_2'],
p['IC_FP_N_ORD_START'])
p['IC_HC_N_ORD_FINAL_2'] = max(p['IC_HC_N_ORD_FINAL_2'],
p['IC_FP_N_ORD_FINAL'])
# run part 2
# p, loc = part2test(p, loc)
p, loc = cal_HC_E2DS_spirou.part2(p, loc)
# ----------------------------------------------------------------------
# End plotting session
# ----------------------------------------------------------------------
# end interactive session
sPlt.end_interactive_session(p)
# ----------------------------------------------------------------------
# End Message
# ----------------------------------------------------------------------
p = spirouStartup.End(p)
# return a copy of locally defined variables in the memory
return dict(locals())
