def run(path0, silent=False, verbose=True):
'''
Main function to run for a given path containing the raw GNIRS data
Parameters
----------
path0 : str
Path to raw FITS file. Must include '/' at the end
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 22 March 2017
Modified by Chun Ly, 23 March 2017
- Call dir_check.main() to handle multiple date directories
Modified by Chun Ly, 8 January 2018
- Import glog and call for stdout and ASCII logging
- Pass mylogger to get_files()
Modified by Chun Ly, 7 May 2018
- Bug fix: Call check_path() to add '/' in path0
'''
rawdir = check_path(path0) # + on 07/05/2018
# + on 08/01/2018
logfile = path0+'symlink.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin run : '+systime())
# + on 23/03/2017
dir_list, list_path = dir_check.main(path0, mylogger=mylogger,
silent=silent, verbose=verbose)
# Mod on 23/03/2017
for path in list_path:
files, n_files = get_files(path, mylogger=mylogger, silent=silent,
verbose=verbose)
for nn in xrange(n_files):
c_file = path+'c'+files[nn]
if exists(c_file):
mylogger.warn('File exists : '+c_file)
else:
cmd0 = 'ln -fs '+files[nn]+' '+c_file
if silent == False:
mylogger.info(cmd0)
os.system(cmd0)
#endfor
#endfor
if silent == False: mylogger.info('### End run : '+systime())
def run(path0, clean_file='', out_script='', silent=False, verbose=True,
overwrite=False):
'''
Create a .sh script to run cleanir.py for a set of files
Parameters
----------
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
overwrite : boolean
Overwrite files if they exists. Default: False
Returns
-------
Notes
-----
Created by Chun Ly, 7 March 2017
Modified by Chun Ly, 15 May 2017
- Call dir_check.main() to handle multiple date directories
Modified by Chun Ly, 30 May 2017
- Added overwrite option. Default is to not overwrite .sh files
- Fix bug when only one file is found
Modified by Chun Ly, 18 June 2017
- Fix to work with no date directory
Modified by Chun Ly, 22 January 2018
- Import glog and call for stdout and ASCII logging
- Pass mylogger to dir_check.main()
Modified by Chun Ly, 20 April 2018
- Change afq to rfq - row filtering generally produces better results
'''
logfile = path0+'cleanir_script.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin run : '+systime())
if clean_file == '': clean_file = 'clean.lis'
# + on 15/05/2017
dir_list, list_path = dir_check.main(path0, mylogger=mylogger, silent=silent,
verbose=verbose)
# Mod on 15/05/2017
for date,path in zip(dir_list,list_path):
clean_file0 = path+clean_file
if not exists(clean_file0):
mylogger.warn('File does not exist!!!')
mylogger.warn(clean_file0)
else:
if silent == False: mylogger.info('Reading : '+clean_file0)
files = np.loadtxt(clean_file0, dtype=type(str)).tolist()
if type(files) == str: files = [files] # Bug fix. Mod on 30/05/2017
out_script0 = path+'run_cleanir.sh' if out_script == '' \
else out_script
if date != '': # Mod on 18/06/2017
out_script0 = out_script0.replace('.sh', '.'+date+'.sh')
# Mod on 30/05/2017
if overwrite == False and exists(out_script0):
log.warn('## File found!!! : '+out_script0)
log.warn('## Will not overwrite!!!')
else:
if silent == False:
stat0 = 'Overwriting' if exists(out_script0) else 'Writing'
mylogger.info(stat0+' : '+out_script0)
f = open(out_script0, 'w')
for ii in xrange(len(files)):
cmd1 = cmd0+' -rfqo '+path+'c'+files[ii]+' '+path+files[ii]
f.write(cmd1+'\n')
f.close()
#endelse
#endelse
#endfor
if silent == False: mylogger.info('End run : '+systime())
def main(path0, out_pdf='', silent=False, verbose=True, overwrite=False):
'''
Main function to generate PDF illustrating alignment on target
Parameters
----------
path0 : str
Path to FITS file. Must include '/' at the end
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
overwrite : boolean
Overwrite files if they exists. Default: False
Returns
-------
Notes
-----
Created by Chun Ly, 24 March 2017
Modified by Chun Ly, 01 April 2017
- Handle CRs and bad pixels using cosmicrays_lacosmic
Modified by Chun Ly, 04 April 2017
- Use find_gnirs_window_mean to find center
Modified by Chun Ly, 04-05 April 2017
- Adjust greyscale limits to handle slit image (make it black),
and faint sources
Use find_gnirs_window_mean to find center
Modified by Chun Ly, 05 April 2017
- Handle alignment sequences with more than just 4 frames
- Handle excess subplots for individual PDF pages (remove axes)
- Compute seeing FWHM for acquisition images
Modified by Chun Ly, 06 April 2017
- Get coordinates for slit in cutout
Modified by Chun Ly, 11 May 2017
- Use slit image to find center when telluric data is only available
Modified by Chun Ly, 3 July 2017
- Add overwrite option to prevent overwriting file
Modified by Chun Ly, 9 January 2018
- Import glog and call for stdout and ASCII logging
- Pass mylogger to find_gnirs_window_mean(), find_gnirs_window()
Modified by Chun Ly, 20 April 2018
- Pass mylogger to gauss2d_fit()
- Switch print statements to mylogger calls
Modified by Chun Ly, 22 April 2018
- Bug fix: mylogger calls mistakes
- Bug fix: mylogger calls mistakes (cont'd)
'''
# + on 09/01/2018
logfile = path0 + 'align_check.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin main : ' + systime())
dir_list, list_path = dir_check.main(path0,
mylogger=mylogger,
silent=silent,
verbose=verbose)
out_pdf_default = out_pdf
for path in list_path:
infile = path + 'hdr_info.QA.tbl'
if not exists(infile):
mylogger.warning('File does not exist : ' + infile)
mylogger.warning('Exiting!!! ' + systime())
return
out_pdf = path + 'align_check.pdf' if out_pdf == '' else path + out_pdf
# Mod on 03/07/2017
if overwrite == False and exists(out_pdf):
mylogger.warn('File exists!! Will not overwrite ' + out_pdf)
else:
pp = PdfPages(out_pdf)
if silent == False: mylogger.info('Reading: ' + infile)
tab0 = asc.read(infile, format='fixed_width_two_line')
align = [ii for ii in xrange(len(tab0)) if tab0['QA'][ii] == 'N/A']
if silent == False:
mylogger.info('Number of alignment images found : ' +
str(len(align)))
ID = tab0['object'][align]
ID0 = list(set(ID)) #Unique ID's
if silent == False:
mylogger.info('Sources found : ' + ', '.join(ID0))
# + on 04/04/2017
win_ref_idx = [
tt for tt in xrange(len(tab0))
if (tab0['QA'][tt] == 'N/A') and ('Acq' in tab0['slit'][tt])
and ('HIP' not in tab0['object'][tt]) and (
'HD' not in tab0['object'][tt])
]
# Mod on 11/05/2017
if len(win_ref_idx) > 0:
win_ref_file = path + tab0['filename'][win_ref_idx[0]]
mylogger.info('Reference image for finding GNIRS window : ' +
win_ref_file)
x_min, x_max, y_min, y_max, x_cen, \
y_cen = find_gnirs_window_mean(win_ref_file, mylogger=mylogger)
else:
mylogger.info('Using telluric image as reference')
win_ref_file = path + tab0['filename'][0]
slit_x0, slit_y0_lo, slit_y0_hi = get_slit_trace(win_ref_file)
x_min, x_max = min(slit_x0), max(slit_x0)
x_cen = (x_min + x_max) / 2.0
y_cen = (np.median(slit_y0_lo) + np.median(slit_y0_hi)) / 2.0
y_min, y_max = y_cen - size2d[0].value / 2.0, y_cen + size2d[
0].value / 2.0
pos_cen = (x_cen, y_cen)
new_size = u.Quantity((y_max - y_min, x_max - x_min), u.pixel)
# + on 20/04/2018, Mod on 22/04/2018
mylogger.info('pos_cen : (%f, %f) ' % (pos_cen[0], pos_cen[1]))
mylogger.info('new_size : [%f, %f] pix ' %
(new_size[0].value, new_size[1].value))
for ii in xrange(len(ID0)):
t_idx = [
tt for tt in xrange(len(tab0)) if
(tab0['object'][tt] == ID0[ii] and tab0['QA'][tt] == 'N/A')
]
t_files = [path + a for a in tab0['filename'][t_idx]]
ncols = 2.0
nrows = 2 # np.ceil(len(t_idx)/ncols)
ncols, nrows = np.int(ncols), np.int(nrows)
# Mod on 05/04/2017
if len(t_idx) <= nrows * ncols:
fig, ax_arr = plt.subplots(nrows=nrows, ncols=ncols)
#med0, x_min, x_max, y_min, \
# y_max, x_cen, y_cen = find_gnirs_window(t_files[1], mylogger=mylogger)
# Later + on 24/03/2017 | Mod on 04/04/2017
xcen, ycen = find_star(t_files[-1],
pos=pos_cen,
find_size2d=new_size)
# Fix to get relative coordinate for Cutout2D image
#xcen -= pos_cen[0]-new_size[1].value/2.0
#ycen -= pos_cen[1]-new_size[0].value/2.0
slit_x0, slit_y0_lo, slit_y0_hi = get_slit_trace(
t_files[0]) #, x_min, x_max)
# Adjust values for offset that is applied
# Bug: Mod on 04/04/2017 to get proper coordinate
slit_x0 -= np.int64(pos_cen[0] - size2d[1].value / 2.0)
slit_y0_lo -= pos_cen[1] - size2d[0].value / 2.0
slit_y0_hi -= pos_cen[1] - size2d[0].value / 2.0
for jj in xrange(len(t_idx)):
jj_idx = t_idx[jj]
# + on 05/04/2017
if len(t_idx) > (nrows * ncols):
if jj % (nrows * ncols) == 0:
fig, ax_arr = plt.subplots(nrows=nrows,
ncols=ncols)
im0 = fits.getdata(t_files[jj])
hdr0 = fits.getheader(t_files[jj], ext=0) # Get WCS header
# + 01/04/2017
im0_clean = cosmicray_lacosmic(im0, sigclip=10)[0]
cutout = Cutout2D(im0_clean,
pos_cen,
size2d,
mode='partial',
fill_value=np.nan)
t_col, t_row = jj % ncols, (jj / ncols) % nrows
# Mod on 04/04/2017 to handle bright and faint stars
max0 = np.max(cutout.data)
# Compute median within GNIRS window
# + on 04-05/04/2017
temp = im0_clean[-50:-1, :]
bgd0, sig0 = np.median(temp), np.std(temp)
idx_y, idx_x = np.where(im0_clean > (bgd0 + 5 * sig0))
med0 = np.median(im0_clean[idx_y, idx_x])
mylogger.info('## max0 : %f med0 : %f ' % (max0, med0))
if max0 > 50000:
z1, z2 = zscale.get_limits(cutout.data)
z2 = max0 # Change for better stretch for telluric star
else:
if ('Acq_' not in tab0['slit'][jj_idx]) and \
(tab0['exptime'][jj_idx] == 3):
# First frame that will show the longslit
z1, z2 = 0.0, 0.5 * max0
else:
# This should handle faint and bright stars
z1, z2 = 0.5 * med0, max0
norm = ImageNormalize(vmin=z1, vmax=z2)
t_ax = ax_arr[t_row, t_col]
t_ax.imshow(cutout.data,
cmap='Greys',
origin='lower',
norm=norm)
#aplpy.FITSFigure(cutout)
# Draw trace of slit
t_ax.plot(slit_x0, slit_y0_lo, 'r-')
t_ax.plot(slit_x0, slit_y0_hi, 'r-')
t_ax.xaxis.set_ticklabels([])
t_ax.yaxis.set_ticklabels([])
fig.suptitle(path, fontsize=14)
txt0 = tab0['filename'][jj_idx] + '\n'
txt0 += tab0['datelabel'][jj_idx] + '\n'
txt0 += tab0['UT_date'][jj_idx] + '\n'
txt0 += tab0['object'][jj_idx]
t_ax.annotate(txt0, [0.025, 0.95],
xycoords='axes fraction',
ha='left',
va='top')
# Plot inset | Later + on 24/03/2017
axins = zoomed_inset_axes(t_ax, 6, loc=4)
norm2 = ImageNormalize(vmin=z1, vmax=z2)
axins.imshow(cutout.data,
cmap='Greys',
origin='lower',
norm=norm2)
# Draw trace of slit
axins.plot(slit_x0, slit_y0_lo, 'r-')
axins.plot(slit_x0, slit_y0_hi, 'r-')
# Mod on 04/04/2017 to get Cutout2d coordinates
c_xcen = xcen - (pos_cen[0] - size2d[1].value / 2.0)
c_ycen = ycen - (pos_cen[1] - size2d[0].value / 2.0)
x1, x2, y1, y2 = c_xcen - 20, c_xcen + 20, c_ycen - 20, c_ycen + 20
axins.set_xlim([x1, x2])
axins.set_ylim([y1, y2])
axins.xaxis.set_ticklabels([])
axins.yaxis.set_ticklabels([])
mark_inset(t_ax,
axins,
loc1=1,
loc2=3,
fc="none",
ec="b",
ls='dotted',
lw=0.5)
# Compute FWHM of alignment star | + on 05/04/2017
if ('Acq_' not in tab0['slit'][jj_idx]) and \
(tab0['exptime'][jj_idx] == 3):
mylogger.info('No source in slit : ' +
tab0['filename'][jj_idx])
else:
# + on 06/04/2017
c_size2d = u.Quantity((40, 40), u.pixel)
c_slit_x0 = slit_x0 - (c_xcen -
c_size2d[1].value / 2.0)
c_slit_y0_lo = slit_y0_lo - (c_ycen -
c_size2d[0].value / 2.0)
c_slit_y0_hi = slit_y0_hi - (c_ycen -
c_size2d[0].value / 2.0)
im0_crop = Cutout2D(cutout.data, (c_xcen, c_ycen),
c_size2d,
mode='partial',
fill_value=np.nan)
gauss2d_fit(im0_crop.data,
hdr0,
t_ax,
c_slit_x0,
c_slit_y0_lo,
c_slit_y0_hi,
mylogger=mylogger) # Mod on 06/04/2017
# Write each page separately | + on 05/04/2017
if len(t_idx) > (nrows * ncols):
# Mod later on 05/04/2017 to handle excess subplots
if jj == len(t_idx) - 1:
rem0 = len(t_idx) % (nrows * ncols) # remainder
if rem0 != 0:
for rr in range(rem0, nrows * ncols, 1):
t_col, t_row = rr % ncols, (rr /
ncols) % nrows
ax_arr[t_row, t_col].axis('off')
if (jj % (nrows * ncols) == nrows*ncols-1) or \
(jj == len(t_idx)-1):
subplots_adjust(left=0.02,
bottom=0.02,
top=0.95,
right=0.98,
wspace=0.02,
hspace=0.02)
fig.set_size_inches(11, 8)
fig.savefig(pp, format='pdf')
#endfor
# Mod on 05/04/2017
if len(t_idx) <= nrows * ncols:
# Mod later on 05/04/2017 to handle excess subplots
for rr in range(len(t_idx), nrows * ncols):
t_col, t_row = rr % ncols, (rr / ncols) % nrows
ax_arr[t_row, t_col].axis('off')
subplots_adjust(left=0.02,
bottom=0.02,
top=0.95,
right=0.98,
wspace=0.02,
hspace=0.02)
fig.set_size_inches(11, 8)
fig.savefig(pp, format='pdf')
#endfor
pp.close()
#endelse
out_pdf = out_pdf_default
if silent == False: mylogger.info('### End main : ' + systime())
def main(rawdir, out_pdf='', gz=False, silent=False, verbose=True):
'''
Main function to compute and plot statistics
Parameters
----------
rawdir : str
Path to raw files.
gz : boolean
Indicate using gz-compressed files (mostly for debugging). Default: False
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 8 May 2018
- Change output PDF filename
- Plot averages of statistic measurements
- Plot expected Poissonian level
- Plot aesthetics (legend, limits, x/ylabel)
- Compute and plot rms of stacked image
Modified by Chun Ly, 21 May 2018
- Bug fix: Mistake in computing RMS for combined 2-D data
- Write RMS to mylogger; Label 'Stack' RMS on right side of plot
- Add gz keyword option to use with compressed files
Modified by Chun Ly, 28 May 2018
- Add '/' after rawdir if not provided
'''
if rawdir[-1] != '/': rawdir = rawdir + '/'
logfile = rawdir + 'image_stats.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin run ! ')
dir_list, list_path = dir_check.main(rawdir,
mylogger=mylogger,
silent=silent,
verbose=verbose)
out_pdf_default = out_pdf
for path in list_path:
file_lis = path + 'obj.lis'
if not exists(file_lis):
if silent == False: mylogger.info('File not found : ' + file_lis)
else:
if silent == False: mylogger.info('Reading : ' + file_lis)
t_files = np.loadtxt(file_lis, dtype=type(str)).tolist()
files0 = [path + 'tfrbnc' + t_file for t_file in t_files]
if gz == True: # + on 21/05/2018
files0 = [t_file + '.gz' for t_file in files0]
n_files0 = len(files0)
avg_arr = np.zeros(n_files0)
med_arr = np.zeros(n_files0)
sig_arr = np.zeros(n_files0)
for nn in range(n_files0):
im0 = fits.getdata(files0[nn], extname='SCI')
c_mean, c_med, c_sig = sigma_clipped_stats(im0,
sigma=2.0,
iters=10)
avg_arr[nn] = c_mean
med_arr[nn] = c_med
sig_arr[nn] = c_sig
#endfor
#print avg_arr, med_arr, sig_arr
if out_pdf == '':
out_pdf = path + 'image_stats.pdf'
else:
out_pdf = path + out_pdf
fig, ax_arr = plt.subplots(nrows=2)
num0 = 1 + np.arange(n_files0)
ax_arr[0].scatter(num0,
avg_arr,
marker='o',
s=50,
facecolor='none',
edgecolor='k',
label='Mean')
avg_avg = np.average(avg_arr)
ax_arr[0].axhline(y=avg_avg, c='k', linestyle='dashed')
ax_arr[0].scatter(num0,
med_arr,
marker='x',
s=25,
color='b',
label='Median')
avg_med = np.average(med_arr)
ax_arr[0].axhline(y=avg_med, c='b', linestyle='dotted')
ax_arr[0].legend(loc='lower right', fontsize=8)
ax_arr[1].scatter(num0,
sig_arr,
marker='o',
s=50,
facecolor='none',
edgecolor='k')
avg_sig = np.average(sig_arr)
ax_arr[1].axhline(y=avg_sig, c='k', linestyle='dashed')
# Expected Poissionan level
sig_poisson = avg_sig / np.sqrt(n_files0)
mylogger.info('Expected (Poisson) RMS : %.4f' %
sig_poisson) # + on 21/05/2018
ax_arr[1].axhline(y=sig_poisson, c='b', linestyle='dashed')
ax_arr[1].text(0,
sig_poisson,
'Poisson',
color='b',
ha='left',
va='bottom')
ax_arr[0].minorticks_on()
ax_arr[1].minorticks_on()
ax_arr[0].set_xticklabels([])
ax_arr[0].set_xlim([-0.25, n_files0 + 1])
ax_arr[1].set_xlim([-0.25, n_files0 + 1])
ax_arr[1].set_ylabel(r'$\sigma$')
ax_arr[1].set_xlabel('Frame No.')
# Plot rms of stacked image
comb_file = glob.glob(path + 'obj_comb.fits')
if len(comb_file) != 0:
mylogger.info('Reading : ' + comb_file[0])
comb_data = fits.getdata(comb_file[0],
extname='SCI') # Mod on 21/05/2018
# Mod on 21/05/2018
c_mean, c_med, c_sig = sigma_clipped_stats(comb_data,
sigma=2.0,
iters=10)
mylogger.info('Measured RMS in stack : %.4f' %
c_sig) # + on 21/05/2018
ax_arr[1].axhline(y=c_sig, c='g', linestyle='dashed')
ax_arr[1].text(n_files0 + 1,
c_sig,
'Stack',
color='g',
ha='right',
va='bottom') # Mod on 21/05/2018
#fig, ax_arr = plt.subplots(ncols=2)
#ax_arr[0].hist(avg_arr, bins=5, align='mid', color='b',
# linestyle='solid', alpha=0.5, edgecolor='b',
# histtype='step', label='Average')
#ax_arr[0].hist(med_arr, bins=5, align='mid', # color='g',
# linestyle='dashed', edgecolor='k',
# histtype='step', label='Median')
#ax_arr[1].hist(sig_arr, bins=5, align='mid', color='k',
# linestyle='solid', alpha=0.5, edgecolor='k',
# histtype='step')
mylogger.info('Writing : ' + out_pdf)
fig.set_size_inches(8, 4)
fig.savefig(out_pdf, bbox_inches='tight')
plt.close()
out_pdf = out_pdf_default
#endfor
if silent == False: mylogger.info('### End run ! ')
def main(path0, targets, outfile=None, silent=False, verbose=True):
'''
Generate ASCII file summarizing observations
Parameters
----------
path0 : str
Parent path for all files
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 25 April 2017
Modified by Chun Ly, 5 May 2017
- Handle overwriting file
Modified by Chun Ly, 3 June 2017
- Bug fix: Check if hdr_info.QA.tbl exists
'''
if silent == False: log.info('### Begin main : ' + systime())
Targets, ObsDate, ObsSet = [], [], []
TotalTime, gratwave, Airmass = [], [], []
TellStar, TellSet, TellAM = [], [], [] # Later Mod on 25/04/2017
for tt in range(len(targets)):
tt_path = path0 + targets[tt] + '/'
dir_list, list_path = dir_check.main(tt_path,
silent=True,
verbose=False)
cnt = 0
for path in list_path:
# Mod on 25/04/2017
txt = targets[tt] if cnt == 0 else '...'
Targets.append(txt) #targets[tt])
QA_file = path + '/hdr_info.QA.tbl'
# Mod on 03/06/2017
if not exists(QA_file):
log.warn('## File not found! ' + QA_file)
ObsDate.append('N/A')
gratwave.append('N/A')
ObsSet.append('N/A')
TotalTime.append('N/A')
Airmass.append('N/A')
TellStar.append('N/A')
TellSet.append('N/A')
TellAM.append('N/A')
else:
QA_tab = asc.read(QA_file, format='fixed_width_two_line')
# All science targets
idx = [
xx for xx in range(len(QA_tab))
if ('obj' in QA_tab['QA'][xx]) or (
'sky' in QA_tab['QA'][xx])
]
# Later Mod on 25/04/2017
tab_ref = QA_tab[idx][0]
t_date = tab_ref['UT_date'].split('T')[0]
exptime = tab_ref['exptime']
ObsDate.append(t_date)
gratwave.append(tab_ref['gratwave'])
ObsSet.append(str(len(idx)) + 'x' + str(exptime) + 's')
TotalTime.append('%.2f' % (len(idx) * exptime / 60.0))
AM0 = QA_tab['airmass'][idx]
Airmass.append('%.3f-%.3f' % (np.min(AM0), np.max(AM0)))
t_idx = [
xx for xx in range(len(QA_tab))
if ('telluric' in QA_tab['QA'][xx])
]
t_names = list(set(QA_tab['object'][t_idx]))
telstar, telset, telAM = get_telluric_info(
QA_tab, t_idx, t_names)
TellStar.append(telstar)
TellSet.append(telset)
TellAM.append(telAM) # Later + on 25/04/2017
cnt += 1
#endelse
#endfor
#endfor
arr0 = [
Targets, ObsDate, ObsSet, TotalTime, gratwave, Airmass, TellStar,
TellSet, TellAM
]
names0 = ('Name', 'UT_Date', 'Sequence', 'Int_Time', 'Grating_Wave',
'Airmass', 'Telluric_Star', 'Telluric_Seq', 'Telluric_AM')
tab0 = Table(arr0, names=names0)
print tab0
if outfile == None: outfile = path0 + 'obs_summary.txt'
# Mod on 06/05/2017
if silent == False:
stat0 = 'Overwriting : ' if exists(outfile) else 'Writing : '
log.info(stat0 + outfile)
asc.write(tab0,
output=outfile,
format='fixed_width_two_line',
overwrite=True)
if silent == False: log.info('### End main : ' + systime())
def main(rawdir, silent=False, verbose=True):
'''
Main function for distort_trace
Parameters
----------
rawdir : str
Path to FITS file. Must include '/' at the end
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 27 June 2018
- Write multi-page PDF file
- Remove center value for middle of spectra
- Plot offsets
- Bug fix for curve_fit (use bb_med0); plot aesthetics (legend)
- Plot aesthetics (axes labeling), margin adjustments
- Call group() to get matplotlib markers and colors
- Write npz file using np.savez and np.load when available
- Do linear regression fit and save to npz file
Modified by Chun Ly, 28 June 2018
- Change linear regression to 2nd order, plot fit
Modified by Chun Ly, 29 June 2018
- Write PDF file in each datedir
- Minor changes to mylogger calls
Modified by Chun Ly, 31 July 2018
- Use sigma_clipped_stats; Search for multiple peaks
Modified by Chun Ly, 1 August 2018
- Smooth median signal (boxcar); mask for peaks
- Call group_index(); Require at leat 5 pixels for peak identification
- Handle multiple peaks
- Compute number of peaks only once
Modified by Chun Ly, 2 August 2018
- Use combine stack for peak identification (more sensitive)
- Get quick peak centers from combine stack
- Use peak in med0 if no combine stack or telluric spec
- Handle peak finding for both telluric and science data
- Handle multiple peaks in plotting
- Restrict fitting to within 10 pixels
Modified by Chun Ly, 3 August 2018
- Simplify code (x0_bb -> x0)
- Fix ValueError: Invalid rgba arg ""
- Fix bugs with n_files and use of 'x' datapoints
- Use median for x_cen_middle to deal with outliers
- Compute median using sigma_clipped_stats, exclude outliers from polyfit
- Switch flag to flag0 to avoid conflict
- Use peak when combine stack has single line
- Switch from curve_fit to weighted centering
- Define p_idx for single peak
Modified by Chun Ly, 7 August 2018
- Bug fix: Incorrect x0_diff
- Handle case when not all skysubtracted frames are used
(check for obj_rev.lis file)
- Force integer for index
- Avoid right edge issue (specific hack for one target)
- Use DQ array to masked bad pixels and edges
- Limit weighted computation within expected location
- Shift x0_max1 for sub-indexing
- Force x-limit range (zoom-in)
- Plot aesthetics: ax annotation
Modified by Chun Ly, 8 August 2018
- Plot fitting results
- Bug fix with savefig location, change labeling
Modified by Chun Ly, 9 August 2018
- Fix typo with wrong ax subplots use (row determination)
- Annotate plot with center
- Plot aesthetics: toplabel, white space, ylim
- Changes for right center for telluric and single peak science data cases
- Adjust ax.plot xrange
Modified by Chun Ly, 10 August 2018
- Compute median of trace_arr in each bin, best fit polynomial fit
- Plot best fit to median of trace_arr
- Update npz savefile with best_fit results
- Annotate plot with best fit
- Fix IndexError issue (limit v1 >= 0)
- Change fig suptitle (just the filename)
'''
if rawdir[-1] != '/': rawdir += '/'
logfile = rawdir + 'distort_trace.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin ! ')
bin_size = 25
dir_list, list_path = dir_check.main(rawdir,
mylogger=mylogger,
silent=silent,
verbose=verbose)
for path in list_path:
mylogger.info('Working on : ' + path.split('/')[-2])
out_pdf = path + 'distort_trace.pdf'
out_pdf1 = path + 'distort_trace_fit.pdf'
npz_file = path + 'distort_trace.npz'
obj_file = path + 'obj_rev.lis'
if not exists(npz_file):
if not exists(obj_file):
files = glob(path + 'tfrbncN????????S????.fits')
else:
mylogger.info('File found : ' + obj_file)
npfiles = np.loadtxt(obj_file, dtype=type(str))
files = [path + 'tfrbnc' + t_file for t_file in npfiles]
npfilesT = np.loadtxt(path + 'telluric.lis', dtype=type(str))
filesT = [path + 'tfrbnc' + t_file for t_file in npfilesT]
files += filesT
files.sort()
n_files = len(files)
mylogger.info('Number of files found : %i ' % n_files)
if n_files > 0:
pdf_pp = PdfPages(out_pdf1)
hdr0 = fits.getheader(files[0], extname='SCI')
n_bins = np.int(np.ceil(np.float(hdr0['NAXIS2'])) / bin_size)
x0 = np.arange(hdr0['NAXIS1'])
y0 = bin_size / 2.0 + bin_size * np.arange(n_bins)
no_c_file = 0
c_file = glob(path + 'obj_comb.fits')
if len(c_file) == 0:
mylogger.warn('Combine frame not found! Using single peak')
no_c_file = 1
n_peaks = 1
else:
t_c_im = fits.getdata(c_file[0], extname='SCI')
c_med0 = np.median(t_c_im, axis=0)
# Find peaks | + on 31/07/2018
x0_max = np.argmax(c_med0)
x0_min = np.argmin(c_med0)
idx_mask = np.where((np.absolute(x0 - x0_max) >= 10)
& (np.absolute(x0 - x0_min) >= 10))[0]
sm_c_med0 = convolve(c_med0, box_kernel)
t_mean, t_med, t_std = sigma_clipped_stats(
sm_c_med0[idx_mask], sigma=2, iters=20)
idx_det = np.where((sm_c_med0 - t_med) / t_std >= 5)[0]
list_peak = np.array(list(group_index(idx_det)))
peak_idx = [
xx for xx in range(len(list_peak))
if (list_peak[xx][1] -
list_peak[xx][0] >= 5 and list_peak[xx][0] < 625)
] # Avoid right edge issues
list_peak = list_peak[peak_idx]
n_peaks = len(peak_idx)
mylogger.info('Number of peaks found : ' + str(n_peaks))
peak_ctr = np.zeros(n_peaks)
for pp in range(n_peaks):
i1, i2 = list_peak[pp][0], list_peak[pp][1]
peak_ctr[pp] = i1 + np.argmax(c_med0[i1:i2])
trace_arr = np.zeros((n_peaks, n_files, n_bins))
xcen_arr = np.zeros((n_peaks, n_files))
fit_arr = np.zeros((n_peaks, n_files, 3))
for ff in range(n_files):
fig, ax = plt.subplots(nrows=nrows, ncols=ncols)
t_im0, t_hdr = fits.getdata(files[ff],
extname='SCI',
header=True)
t_dq = fits.getdata(files[ff], extname='DQ')
t_im = np.ma.array(t_im0, mask=t_dq)
med0 = np.ma.median(t_im, axis=0)
x0_max0 = np.ma.argmax(med0)
h_obj = t_hdr['OBJECT']
if no_c_file or ('HIP' in h_obj or 'HD' in h_obj):
n_peak, use_peak = 1, 1
else:
n_peak = n_peaks
use_peak = 1 if n_peaks == 1 else 0
x0_diff = peak_ctr[0] - x0_max0
for bb in range(n_bins):
row, col = bb / ncols, bb % ncols
ty1, ty2 = (0 + bb) * bin_size, (1 + bb) * bin_size
bb_med0 = np.ma.median(t_im[ty1:ty2], axis=0)
for pp in range(n_peak):
if use_peak == 1:
if no_c_file or ('HIP' in h_obj
or 'HD' in h_obj):
v1, v2 = x0_max0 - 15, x0_max0 + 15
else:
v1 = np.max([
np.int(list_peak[0][0] - x0_diff - 15),
0
])
v2 = np.int(list_peak[0][1] - x0_diff + 15)
# print ff, bb, pp, v1, v2, x0_max0
x0_max1 = v1 + np.ma.argmax(bb_med0[v1:v2])
p_idx = np.arange(x0_max1 - 10, x0_max1 + 10)
# print ff, bb, pp, x0_max0, x0_max1, p_idx[0], p_idx[-1]
else:
p_idx = np.arange(
np.int(list_peak[pp][0] - x0_diff),
np.int(list_peak[pp][1] - x0_diff))
p_med0 = bb_med0[p_idx]
x_cen = np.sum(p_med0 * p_idx) / np.sum(p_med0)
#p0 = [0.0, y0_max, x0_max, 2.0]
#try:
# popt, pcov = curve_fit(gauss1d, x0, bb_med0, p0=p0)
# x_cen = popt[2]
#except RuntimeError:
# print 'Runtime error'
# x_cen = p0[2]
trace_arr[pp, ff, bb] = x_cen
x_off = p_idx[len(p_idx) / 2]
ax[row, col].plot(p_idx - x_off,
p_med0 / max(p_med0))
ax[row, col].axvline(x=x_cen - x_off)
ax[row, col].annotate('%.1f' % x_cen,
xy=[0.95, 0.95],
ha='right',
va='top',
fontsize=9,
xycoords='axes fraction')
if row != nrows - 1:
ax[row, col].set_xticklabels([])
if col != 0:
ax[row, col].set_yticklabels([])
ax[row, col].set_ylim([-0.05, 1.05])
#endfor
#endfor
plt.subplots_adjust(left=0.08,
right=0.99,
bottom=0.05,
top=0.95,
hspace=0.02,
wspace=0.02)
fig.suptitle(os.path.basename(files[ff]), fontsize=12)
fig.savefig(pdf_pp, format='pdf')
#endfor
mylogger.info('Writing : ' + out_pdf1)
pdf_pp.close()
flag0 = np.ones((n_peaks, n_files, n_bins))
for ff in range(n_files):
for pp in range(n_peaks):
t_me, t_md, t_s = sigma_clipped_stats(trace_arr[pp,
ff],
sigma=3,
iters=10)
x_cen_middle = t_md
xcen_arr[pp, ff] = x_cen_middle
trace_arr[pp, ff] -= x_cen_middle
diff = trace_arr[pp, ff] - ((y0 - 512) * 0.019)
use = np.where(np.absolute(diff) <= 5)[0]
if len(use) > 0: flag0[pp, ff, use] = 0
fit = np.polyfit(y0[use], trace_arr[pp, ff][use], 2)
fit_arr[pp, ff] = fit
#endfor
#endfor
mylogger.info('Writing : ' + npz_file)
np.savez(npz_file,
trace_arr=trace_arr,
xcen_arr=xcen_arr,
fit_arr=fit_arr,
y0=y0,
flag0=flag0)
else:
mylogger.warn('Files not found !')
else:
mylogger.info('Reading : ' + npz_file)
npz = np.load(npz_file)
trace_arr = npz['trace_arr']
xcen_arr = npz['xcen_arr']
y0 = npz['y0']
fit_arr = npz['fit_arr']
flag0 = npz['flag0']
n_files = xcen_arr.shape[1]
n_peaks = xcen_arr.shape[0]
if n_files > 0:
fig, ax = plt.subplots()
xlim = [-10, 10]
ctype, mtype, labels = group(xcen_arr)
x_fit0 = np.zeros(len(y0))
for bb in range(len(y0)):
use = np.where(flag0[:, :, bb] == 0)
x_fit0[bb] = np.median(trace_arr[use[0], use[1], bb])
best_fit = np.polyfit(y0, x_fit0, 2)
mylogger.info('Best fit : [%f, %f, %f]', best_fit[0], best_fit[1],
best_fit[2])
for pp in range(n_peaks):
for ff in range(n_files):
if labels[pp, ff] != '':
fc = ctype[pp, ff] if mtype[pp, ff] == 'x' else 'none'
ax.scatter(trace_arr[pp, ff, :],
y0,
marker=mtype[pp, ff],
alpha=0.5,
edgecolor=ctype[pp, ff],
facecolor=fc,
label=labels[pp, ff])
ax.scatter(trace_arr[pp, ff, flag0[pp, ff] == 1],
y0[flag0[pp, ff] == 1],
marker='x',
color='r')
pd = np.poly1d(fit_arr[pp, ff])
ax.plot(pd(y0),
y0,
color=ctype[pp, ff],
linewidth=0.75,
alpha=0.5)
ax.scatter(x_fit0,
y0,
marker='o',
edgecolor='none',
facecolor='black',
linewidth=2.0,
alpha=0.9)
best_pd = np.poly1d(best_fit)
ax.plot(best_pd(y0), y0, color='black', linewidth=2.0, alpha=0.9)
b_txt = r'x = A y$^2$ + B y + C'+\
'\nA = %.3e\nB = %.3e\nC = %.3f' % (best_fit[0],best_fit[1],
best_fit[2])
ax.annotate(b_txt,
xy=(0.05, 0.75),
xycoords='axes fraction',
ha='left',
va='top')
out_plt = np.where((trace_arr > xlim[1]) | (trace_arr < xlim[0]))
n_out_plt = len(out_plt[0])
flagged = np.where(flag0 == 1)
n_flagged = len(flagged[0])
a_txt = 'N(exclude) = ' + str(n_out_plt) + '\nN(flagged) = ' + str(
n_flagged)
ax.annotate(a_txt,
xy=(0.05, 0.95),
xycoords='axes fraction',
ha='left',
va='top')
ax.legend(loc='lower right')
ax.set_ylabel('Y [pixels]', fontsize=14)
ax.set_xlabel('X relative to center [pixels]', fontsize=14)
ax.set_xlim(xlim) #[-100,100])
ax.set_ylim([-10, 1050])
fig.suptitle(path)
ax.minorticks_on()
fig.set_size_inches(8, 8)
plt.subplots_adjust(left=0.1, right=0.98, bottom=0.07, top=0.98)
mylogger.info('Writing : ' + out_pdf)
fig.savefig(out_pdf, format='pdf')
mylogger.info('Updating : ' + npz_file)
np.savez(npz_file,
trace_arr=trace_arr,
xcen_arr=xcen_arr,
fit_arr=fit_arr,
y0=y0,
flag0=flag0,
best_fit=best_fit)
#endfor
if silent == False: mylogger.info('### End ! ')
def main(path0='',
out_pdf='',
check_quality=True,
skysub=False,
silent=False,
verbose=True,
overwrite=False):
'''
main() function to compute natural seeing (image quality) from bright
alignment star
Parameters
----------
path0 : str
Full path to where output PDF and FITS file are located. Must end
with a '/'
out_pdf : str
Filename for output PDF. Do NOT include full path
check_quality : boolean
Check whether data meets IQ requirements. Default: True
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
overwrite : boolean
Overwrite files if they exists. Default: False
Returns
-------
multi-page PDF plot
Notes
-----
Created by Chun Ly, 10 March 2017
- Later modified to include check_quality keyword option
- Later modified to include inset that shows the stacked line profile
Modified by Chun Ly, 11 April 2017
- Call dir_check.main() to handle multiple date directories
Modified by Chun Ly, 13 April 2017
- Minor bug: Check if file exists first
Modified by Chun Ly, 10 May 2017
- Minor bug: When .lis file contains only one entry, problem for
appending to list
Modified by Chun Ly, 1 June 2017
- Added overwrite keyword option to overwrite file. Default is not to
overwrite .pdf files
- Bug found: No longer need sky.lis since obj.lis includes all
Modified by Chun Ly, 6 June 2017
- Add skysub keyword option to operate on sky-subtracted images
Modified by Chun Ly, 14 July 2017
- Fix tick mark locations
- Fix y limit range for extreme outliers
Modified by Chun Ly, 16 November 2017
- Change prefix: rnc to rbnc
Modified by Chun Ly, 18 December 2017
- Import glog and call for stdout and ASCII logging
- Pass mylogger to compute_fwhm()
Modified by Chun Ly, 11 January 2018
- Pass mylogger to dir_check.main()
Modified by Chun Ly, 18 April 2018
- Compute and report seeing at zenith
- Show FWHM @ Zenith on right y-axis
Modified by Chun Ly, 19 April 2018
- Include airmass info in plots
Modified by Chun Ly, 14 May 2018
- Write ASCII table containing FWHM determination
Modified by Chun Ly, 18 May 2018
- Adjust subplots_adjust to avoid labels being cut off
- Handle case when extra plot window (odd numbers) is available
- Handle case with extra plot window (cont'd)
Modified by Chun Ly, 19 May 2018
- Include QA (PASS/USABLE/FAIL) info in table
Modified by Chun Ly, 28 June 2018
- Bug troubleshooting with ValueError
- Handle ValueError for avg FWHM
'''
# + on 18/12/2017
logfile = path0 + 'IQ_plot.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin main : ' + systime())
# + on 11/04/2017
dir_list, list_path = dir_check.main(path0,
mylogger=mylogger,
silent=silent,
verbose=verbose)
out_pdf_default = out_pdf
# Mod on 11/04/2017
for path in list_path:
files = []
file_lis = ['obj.lis', 'telluric.lis'] # Minor bug fix on 01/06/2017
for file0 in file_lis:
# Mod on 13/04/2017
if exists(path + file0):
if silent == False: mylogger.info('Reading : ' + path + file0)
t_files = np.loadtxt(path + file0, dtype=type(str)).tolist()
# Bug fix - 10/05/2017
if type(t_files) == str: t_files = [t_files]
files += t_files
else:
if silent == False:
mylogger.info('File not found : ' + path + file0)
files.sort()
n_files = len(files)
# + on 14/05/2018
FWHM_avg_arr = np.zeros(n_files) # FWHM from averaging stack0_shift
FWHM_avg_arr_Z = np.zeros(
n_files) # FWHM from averaging stack0_shift at Zenith
FWHM_avg_QA = [
''
] * n_files # QA check from averaging stack0_shift at Zenith
FWHM_med_arr = np.zeros(n_files) # FWHM from median along Y
FWHM_med_arr_Z = np.zeros(
n_files) # FWHM from median along Y at Zenith
FWHM_med_QA = [''] * n_files # QA check from median along Y at Zenith
# Mod on 06/06/2017
if skysub == True:
files = ['brnc' + file0 for file0 in files] # Mod on 16/11/2017
out_pdf = path+'IQ_plot.skysub.pdf' if out_pdf == '' else \
path+out_pdf
else:
out_pdf = path+'IQ_plot.raw.pdf' if out_pdf == '' else \
path+out_pdf
if overwrite == False and exists(out_pdf):
mylogger.warn('File exists!! Will not overwrite ' + out_pdf)
else:
pp = PdfPages(out_pdf)
for nn in xrange(n_files):
if silent == False: mylogger.info('Reading : ' + files[nn])
hdr0 = fits.getheader(path + files[nn])
# Mod on 06/06/2017
if skysub == False:
im0 = fits.getdata(path + files[nn])
else:
im0 = fits.getdata(path + files[nn], 'sci')
airmass = hdr0['AIRMASS'] # + on 18/04/2018
# Mod on 18/12/2017
bins, fwhm0, stack0_shift = compute_fwhm(im0,
mylogger=mylogger)
row = nn % 2
if row == 0: fig, ax0 = plt.subplots(2, 1)
good = np.where(fwhm0 > 0)[0]
ax0[row].plot(bins[good],
fwhm0[good],
marker='o',
alpha=0.5,
mec='none',
mfc='b',
linestyle='none',
zorder=2)
ax0[row].get_yaxis().set_tick_params(which='both',
right=True,
width=1,
direction='in')
ax0[row].get_xaxis().set_tick_params(which='both',
top=True,
width=1,
direction='in')
# Compute average line profile from stack0_shift
# Later + on 10/03/2017
avg_stack = np.average(stack0_shift, axis=0)
x0_avg = np.arange(-25, 25)
if row == 0: axi = fig.add_axes([0.60, 0.81, 0.25, 0.15])
if row == 1: axi = fig.add_axes([0.60, 0.36, 0.25, 0.15])
axi.plot(x0_avg * pscale, avg_stack, 'k-')
axi.set_ylim([-0.4, 1.1])
axi.set_xlim([-2.0, 2.0])
axi.set_xticks(range(-2, 2, 1))
axi.set_xlabel('X [arcsec]', fontsize=8)
axi.tick_params(labelsize=8)
axi.minorticks_on()
p0 = [0.0, 1.0, 0.0, 2.0]
try:
popt, pcov = curve_fit(gauss1d, x0_avg, avg_stack, p0=p0)
fit_good = 1
except ValueError:
print len(np.where(np.isnan(x0_avg))[0])
print len(np.where(np.isnan(avg_stack))[0])
fit_good = 0
if fit_good:
avg_fwhm0 = popt[3] * 2 * np.sqrt(2 * np.log(2)) * pscale
avg_fwhm_Z = avg_fwhm0 / airmass**0.6 # + on 18/04/2018
axi.plot(x0_avg * pscale, gauss1d(x0_avg, *popt), 'r--')
axi.annotate('FWHM = %.3f" (%.3f")' %
(avg_fwhm0, avg_fwhm_Z), [0.50, 0.025],
xycoords='axes fraction',
ha='center',
va='bottom',
fontsize=8)
ax0[row].axhline(y=avg_fwhm0,
linewidth=2,
color='r',
linestyle='--',
zorder=1)
# Median FWHM | Later + on 10/03/2017
med_fwhm0 = np.median(fwhm0[good])
med_fwhm0_Z = med_fwhm0 / airmass**0.6 # + on 18/04/2018
ax0[row].axhline(y=med_fwhm0,
linewidth=2,
color='g',
linestyle='--',
zorder=1)
# Axes labeling
ax0[row].set_ylabel('FWHM [arcsec]')
if row == 1 or nn == n_files - 1:
ax0[row].set_xlabel('Y [pixel]')
else:
ax0[row].set_xticklabels([])
if nn == n_files - 1:
if row == 0:
ax0[row + 1].axis('off')
# Annotation
txt0 = files[nn] + '\nTarget: ' + hdr0[
'OBJECT'] #.split(' ')[0]
ax0[row].annotate(txt0, [0.025, 0.95],
xycoords='axes fraction',
ha='left',
va='top')
# + on 19/04/2018
ax0[row].annotate('AM = %.3f' % airmass, [0.025, 0.025],
xycoords='axes fraction',
ha='left',
va='bottom')
# Later + on 10/03/2017
if check_quality:
req = hdr0['REQIQ'].replace('-percentile', '%')
raw = hdr0['RAWIQ'].replace('-percentile', '%')
i_raw = [
ii for ii in range(len(FWHM_IQ_C))
if raw in FWHM_IQ_C[ii]
][0]
i_req = [
ii for ii in range(len(FWHM_IQ_C))
if raw in FWHM_IQ_C[ii]
][0]
txt0 = 'Req. IQ: %s [%.2f"]\n' % (req, FWHM_IQ_J[i_req])
txt0 += 'Raw IQ: %s [%.2f"]\n' % (raw, FWHM_IQ_J[i_raw])
# Mod on 18/04/2018
if med_fwhm0_Z <= FWHM_IQ_J[i_raw]:
txt0 += 'PASS'
FWHM_med_QA[nn] = 'PASS' # + on 19/05/2018
else:
if med_fwhm0_Z <= FWHM_IQ_J[i_raw] * 1.25:
txt0 += 'USABLE'
FWHM_med_QA[nn] = 'USABLE' # + on 19/05/2018
if med_fwhm0_Z > FWHM_IQ_J[i_raw] * 1.25:
txt0 += 'FAIL'
FWHM_med_QA[nn] = 'FAIL' # + on 19/05/2018
# + on 19/05/2018
if fit_good:
if avg_fwhm_Z <= FWHM_IQ_J[i_raw]:
FWHM_avg_QA[nn] = 'PASS'
else:
if avg_fwhm_Z <= FWHM_IQ_J[i_raw] * 1.25:
FWHM_avg_QA[nn] = 'USABLE'
if avg_fwhm_Z > FWHM_IQ_J[i_raw] * 1.25:
FWHM_avg_QA[nn] = 'FAIL'
ax0[row].annotate(txt0, [0.975, 0.05],
ha='right',
xycoords='axes fraction',
va='bottom')
# Aesthetics
ax0[row].set_xlim([0, 1050])
if max(fwhm0[good]) > 3:
ax0[row].set_ylim([0, 3.0])
else:
ax0[row].set_ylim([min(fwhm0) - 0.025, max(fwhm0) + 0.075])
ax0[row].minorticks_on()
# + on 18/04/2018
ax2 = ax0[row].twinx()
ax2.set_ylabel(r"FWHM @ Zenith [arcsec]")
ax2.set_ylim(np.array(ax0[row].get_ylim()) / airmass**0.6)
ax2.minorticks_on()
if row != 1:
ax2.set_xticklabels([])
if row == 1 or nn == n_files - 1:
subplots_adjust(left=0.11,
bottom=0.10,
top=0.975,
right=0.875,
wspace=0.03,
hspace=0.05)
fig.savefig(pp, format='pdf')
# + on 14/05/2018
if fit_good:
FWHM_avg_arr[nn] = avg_fwhm0
FWHM_avg_arr_Z[nn] = avg_fwhm_Z
FWHM_med_arr[nn] = med_fwhm0
FWHM_med_arr_Z[nn] = med_fwhm0_Z
#endfor
if silent == False: mylogger.info('Writing : ' + out_pdf)
pp.close()
# + on 14/05/2018
fwhm_out_file = out_pdf.replace('.pdf', '.tbl')
arr0 = [
files, FWHM_avg_arr, FWHM_avg_arr_Z, FWHM_med_arr,
FWHM_med_arr_Z, FWHM_avg_QA, FWHM_med_QA
] # Mod on 19/05/2018
names0 = ('files', 'FWHM_avg', 'FWHM_avg_Z', 'FWHM_med',
'FWHM_med_Z', 'FWHM_avg_QA', 'FWHM_med_QA'
) # Mod on 19/05/2018
tab0 = Table(arr0, names=names0)
if silent == False: mylogger.info('Writing : ' + fwhm_out_file)
asc.write(tab0,
fwhm_out_file,
format='fixed_width_two_line',
overwrite=True)
out_pdf = out_pdf_default
#endelse
#endfor
if silent == False: mylogger.info('### End main : ' + systime())
def get_offsets(path0, mylogger=None, silent=True, verbose=False):
'''
Function to get offsets from FITS header and write it to ASCII file
Parameters
----------
path0 : str
Path to FITS file. Must include '/' at the end
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
tab0 : astropy.table.Table
Astropy ASCII table written to [path0]+'sci_offsets.tbl'
Notes
-----
Created by Chun Ly, 30 May 2017
Modified by Chun Ly, 10 December 2017
- Implement glog logging, allow mylogger keyword input
Modified by Chun Ly, 17 December 2017
- Minor fix: log -> clog
- Call dir_check with mylogger input
'''
# + on 10/12/2017
if type(mylogger) == type(None):
mylog, clog = 0, log
else:
mylog, clog = 1, mylogger
if silent == False: clog.info('### Begin get_offsets : ' + systime())
# Mod on 17/12/2017
if not mylog:
dir_list, list_path = dir_check.main(path0,
silent=silent,
verbose=verbose)
else:
dir_list, list_path = dir_check.main(path0,
silent=silent,
verbose=verbose,
mylogger=clog)
for path in list_path:
outfile = path + 'sci_offsets.tbl'
if exists(outfile):
# Mod on 10/12/2017
clog.warning('File exists : ' + outfile)
clog.warning('Not over-writing!!! ')
else:
fits_files = np.loadtxt(path + 'obj.lis', dtype=type(str))
fits_files = [path + file0 for file0 in fits_files] # Bug fix
n_files = len(fits_files)
names0 = ('filename', 'xoffset', 'yoffset', 'poffset', 'qoffset')
dtype0 = ('S20', 'f8', 'f8', 'f8', 'f8')
tab0 = Table(names=names0, dtype=dtype0)
for nn in xrange(n_files):
basename = os.path.basename(fits_files[nn])
if verbose == True: log.info('## Reading : ' + basename)
h0 = fits.getheader(fits_files[nn])
vec0 = [
basename, h0['XOFFSET'], h0['YOFFSET'], h0['POFFSET'],
h0['QOFFSET']
]
tab0.add_row(vec0)
if silent == False: clog.info('Writing : ' + outfile)
asc.write(tab0, outfile, format='fixed_width_two_line')
#endelse
#endfor
if silent == False: clog.info('### End get_offsets : ' + systime())
def create_distort_grid(rawdir, silent=False, verbose=True):
'''
Create grid (and plot) of distortion for extraction
Parameters
----------
None
Returns
-------
None
Notes
-----
Created by Chun Ly, 29 June 2018
Modified by Chun Ly, 10 August 2018
- Plot best fit
Modified by Chun Ly, 13 August 2018
- Rewrite to work for specified rawdir path (instead of all)
- Fix typo with ax.axhline indexing
- Re-organize code (handle only one distort_trace.npz file)
- Minor bug fix: array name
- Plot aesthetics
'''
if rawdir[-1] != '/': rawdir += '/'
logfile = rawdir + 'distort_trace.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin ! ')
dir_list, list_path = dir_check.main(rawdir,
mylogger=mylogger,
silent=silent,
verbose=verbose)
for path in list_path:
mylogger.info('Working on : ' + path.split('/')[-2])
npz_files = glob(path + '/distort_trace.npz')
if len(npz_files) == 0:
log.warn('No files found!!!')
else:
fig, ax = plt.subplots(nrows=3)
npz = np.load(npz_files[0])
xcen_arr = npz['xcen_arr']
fit_arr = npz['fit_arr']
best_fit = npz['best_fit']
n_peaks = xcen_arr.shape[0]
for pp in range(n_peaks):
ax[0].scatter(xcen_arr[pp],
fit_arr[pp, :, 2],
marker='o',
edgecolor='k',
facecolor='none')
ax[0].axhline(y=best_fit[2], color='b')
ax[1].scatter(xcen_arr[pp],
fit_arr[pp, :, 1],
marker='o',
edgecolor='k',
facecolor='none')
ax[1].axhline(y=best_fit[1], color='b')
ax[2].scatter(xcen_arr[pp],
fit_arr[pp, :, 0],
marker='o',
edgecolor='k',
facecolor='none')
ax[2].axhline(y=best_fit[0], color='b')
ax[0].annotate(r'x = A y$^2$ + B y + C',
xy=(0.02, 0.95),
xycoords='axes fraction',
ha='left',
va='top')
ax[0].set_ylabel('C')
ax[1].set_ylabel('B')
ax[2].set_ylabel('A')
ax[0].set_ylim([-10, 0])
ax[1].set_ylim([-0.025, 0.025])
ax[2].set_ylim([-0.001, 0.001])
ax[0].set_xticklabels([])
ax[1].set_xticklabels([])
ax[2].set_xlabel('X [pix]')
plt.subplots_adjust(left=0.15, right=0.99, top=0.99, bottom=0.1)
out_pdf = path + 'distort_grid.pdf'
log.info('Writing : ' + out_pdf)
fig.savefig(out_pdf)
def main(path0, silent=False, verbose=True, overwrite=False):
'''
main() function to obtain information from FITS header and write to
ASCII file
Parameters
----------
path0 : str
Path to FITS file. Must include '/' at the end
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
tab0 : astropy.table.Table
Astropy ASCII table written to [path0]+'hdr_info.tbl'
Notes
-----
Created by Chun Ly, 4 March 2017
- Later re-organized to check for file first
Modified by Chun Ly, 5 March 2017
- File exists warning always printed out
- Include AIRMASS
Modified by Chun Ly, 23 March 2017
- Call dir_check.main() to handle multiple date directories
Modified by Chun Ly, 11 May 2017
- Handle longer filter1 and filter2 FITS values
Modified by Chun Ly, 8 December 2017
- Import glog and call for stdout and ASCII logging
- Pass mylogger to dir_check.main()
'''
# Moved up on 10/12/2017
logfile = path0 + 'hdr_info.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin main : ' + systime())
dir_list, list_path = dir_check.main(path0,
mylogger=mylogger,
silent=silent,
verbose=verbose)
# Mod on 23/03/2017
for path in list_path:
outfile = path + 'hdr_info.tbl'
# Mod later on 04/03/2017 to not overwrite file
# Mod on 05/03/2017 to always print out this warning
if overwrite == False and exists(outfile):
# Mod on 08/12/2017
mylogger.warning('File exists : ' + outfile)
mylogger.warning('Not over-writing!!! ')
else:
fits_files = glob.glob(path + 'N*fits')
n_files = len(fits_files)
# Mod on 05/03/2017 to include airmass
names0 = ('filename', 'datelabel', 'UT_date', 'obstype', 'object',
'exptime', 'airmass', 'grating', 'gratwave', 'filter1',
'filter2', 'slit')
dtype0 = ('S20', 'S30', 'S25', 'S8', 'S100', 'f8', 'f8', 'S15',
'f8', 'S20', 'S20', 'S20')
tab0 = Table(names=names0, dtype=dtype0)
for nn in xrange(n_files):
basename = os.path.basename(fits_files[nn])
if silent == False: mylogger.info('Reading : ' + basename)
h0 = fits.getheader(fits_files[nn])
# Mod on 05/03/2017 to include airmass
vec0 = [
basename, h0['DATALAB'], h0['DATE-OBS'] + 'T' + h0['UT'],
h0['OBSTYPE'], h0['OBJECT'], h0['EXPTIME'], h0['AIRMASS'],
h0['GRATING'], h0['GRATWAVE'], h0['FILTER1'],
h0['FILTER2'], h0['SLIT']
]
tab0.add_row(vec0)
if silent == False:
mylogger.info('Writing : ' + outfile) # Mod on 08/12/2017
asc.write(tab0, outfile, format='fixed_width_two_line')
#endelse
#endfor
if silent == False: mylogger.info('### End main : ' + systime())
def QA_combine(path0, targets0, out_pdf='', silent=False, verbose=True):
'''
Display sky-subtracted and shifted combined images for telluric and
science data
Parameters
----------
path0 : str
Full path to where output PDF and FITS file are located. Must end
with a '/'
targets0: list or numpy array
A list or array of source names available through path0
out_pdf : str
Filename for output PDF. Do NOT include full path.
Default: 'QA_combine.pdf'
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
multi-page PDF plot, 'QA_combine.pdf'
Notes
-----
Created by Chun Ly, 31 May 2017
Modified by Chun Ly, 1 June 2017
- Switch over to pyplot.imshow() since aplpy cannot allow for more customization
'''
if silent == False: log.info('### Begin QA_combine : ' + systime())
out_pdf = path0 + 'QA_combine.pdf' if out_pdf == '' else path0 + out_pdf
pp = PdfPages(out_pdf)
for target in targets0:
t_path = path0 + target + '/'
dir_list, list_path = dir_check.main(t_path,
silent=silent,
verbose=verbose)
for dpath in list_path:
tel_file = glob.glob(dpath + 'tell_comb.fits')
obj_file = glob.glob(dpath + 'obj_comb.fits')
if len(tel_file) == 0 and len(obj_file) == 0:
log.warn('## No tell_comb.fits and obj_comb.fits found in: ')
log.warn('## ' + dpath)
if len(tel_file) == 1 and len(obj_file) == 1:
fig, (ax1, ax2) = plt.subplots(1, 2) # Mod on 01/06/2017
# Mod on 01/06/2017
if len(tel_file) != 0:
t_im, t_hdr = fits.getdata(tel_file[0], header=True)
lam_max = t_hdr['CRVAL2'] + t_hdr['CD2_2'] * t_hdr['NAXIS2']
extent = [0, t_hdr['NAXIS1'], t_hdr['CRVAL2'], lam_max]
z1, z2 = zscale.get_limits(t_im)
norm = ImageNormalize(vmin=z2, vmax=z1)
ax1.imshow(t_im,
cmap='Greys',
origin='lower',
norm=norm,
extent=extent)
yticks = np.array(ax1.get_yticks())
ax1.set_yticklabels([val / 1e4 for val in yticks])
ax1.get_yaxis().set_tick_params(which='major',
direction='in',
right=True,
length=5,
width=1)
ax1.get_yaxis().set_tick_params(which='minor',
direction='in',
right=True,
length=2.5)
ax1.get_xaxis().set_tick_params(which='major',
direction='in',
top=True,
length=5,
width=1)
ax1.get_xaxis().set_tick_params(which='minor',
direction='in',
top=True,
length=2.5)
ax1.minorticks_on()
ax1.set_xlabel('X [pixels]', fontsize=14)
ax1.set_ylabel(r'Wavelength ($\mu$m)', fontsize=14)
ax1.annotate(tel_file[0], [0.025, 0.975],
xycoords='axes fraction',
ha='left',
va='top',
bbox=bbox_props)
# Mod on 01/06/2017
if len(obj_file) != 0:
o_im, o_hdr = fits.getdata(obj_file[0], header=True)
lam_max = o_hdr['CRVAL2'] + o_hdr['CD2_2'] * o_hdr['NAXIS2']
extent = [0, o_hdr['NAXIS1'], o_hdr['CRVAL2'], lam_max]
z1, z2 = zscale.get_limits(o_im)
norm = ImageNormalize(vmin=z2, vmax=z1)
ax2.imshow(o_im,
cmap='Greys',
origin='lower',
norm=norm,
extent=extent)
yticks = np.array(ax2.get_yticks())
ax2.set_yticklabels([val / 1e4 for val in yticks])
ax2.get_yaxis().set_tick_params(which='major',
direction='in',
right=True,
length=5,
width=1)
ax2.get_yaxis().set_tick_params(which='minor',
direction='in',
right=True,
length=2.5)
ax2.get_xaxis().set_tick_params(which='major',
direction='in',
top=True,
length=5,
width=1)
ax2.get_xaxis().set_tick_params(which='minor',
direction='in',
top=True,
length=2.5)
ax2.minorticks_on()
ax2.set_xlabel('X [pixels]', fontsize=14)
ax2.set_ylabel('')
ax2.set_yticklabels([])
ax2.annotate(obj_file[0], [0.025, 0.975],
xycoords='axes fraction',
ha='left',
va='top',
bbox=bbox_props)
if len(tel_file) == 1 and len(obj_file) == 1: # Mod on 01/06/2017
subplots_adjust(left=0.06,
bottom=0.06,
top=0.995,
right=0.99,
hspace=0.00,
wspace=0.00)
fig.set_size_inches(11, 7.3)
# fig.tight_layout()
fig.savefig(pp, format='pdf') #, bbox_inches='tight')
if silent == False: log.info('## Writing : ' + out_pdf)
pp.close()
if silent == False: log.info('### End QA_combine : ' + systime())
def main(file_list,
path0='',
out_pdf='',
silent=False,
verbose=True,
overwrite=False):
'''
main() function to read in each FITS image and display it on a zscale
using aplpy
Parameters
----------
file_list : str
Filename of ASCII file containing images to be plotted. Use 'all.lis'
from create_list() since this will include all that will be processed
Do NOT include full path
path0 : str
Full path to where output PDF and FITS file are located. Must end
with a '/'
out_pdf : str
Filename for output PDF. Do NOT include full path
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
overwrite : boolean
Overwrite files if they exists. Default: False
Returns
-------
multi-page PDF plot
Notes
-----
Created by Chun Ly, 6 March 2017
Modified by Chun Ly, 23 March 2017
- Call dir_check.main() to handle multiple date directories
Modified by Chun Ly, 1 June 2017
- Added overwrite keyword option to overwrite file. Default is not to
overwrite .pdf files
Modified by Chun Ly, 8 December 2017
- Import glog and call for stdout and ASCII logging
Modified by Chun Ly, 17 December 2017
- Change glog logging to common path
- Add begin and end QA_clean logging to glog logfile
- Pass mylogger to dir_check.main()
'''
# Moved up on 17/12/2017
logfile = path0 + 'QA_plot.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin main : ' + systime())
# + on 23/03/2017
dir_list, list_path = dir_check.main(path0,
mylogger=mylogger,
silent=silent,
verbose=verbose)
out_pdf_default = out_pdf
# Mod on 23/03/2017
for path in list_path:
if silent == False:
mylogger.info('Reading : ' + path + file_list) # Mod on 08/12/2017
files = np.loadtxt(path + file_list, dtype=type(str)).tolist()
n_files = len(files)
if out_pdf == '':
out_pdf = path + 'QA_plot.pdf'
else:
out_pdf = path + out_pdf
hdr_info_file = path + 'hdr_info.tbl'
if exists(hdr_info_file):
if silent == False:
mylogger.info('Reading : ' +
hdr_info_file) # Mod on 08/12/2017
tab0 = asc.read(hdr_info_file, format='fixed_width_two_line')
idx1, idx2 = match_nosort_str(files, tab0['filename'])
tab0 = tab0[idx2]
else:
if silent == False:
mylogger.warn('File not found : ' +
hdr_info_file) # Mod on 08/12/2017
if overwrite == False and exists(out_pdf):
mylogger.warn('File exists!! Will not overwrite ' +
out_pdf) # Mod on 08/12/2017
else:
pp = PdfPages(out_pdf)
for nn in xrange(n_files):
if silent == False:
mylogger.info('Reading : ' +
files[nn]) # Mod on 08/12/2017
# h0 = fits.getheader(path+files[nn], 0)
# im0 = fits.getdata(path+files[nn], 1)
hdu0 = fits.open(path + files[nn])
im0 = hdu0[1].data
hdu0[1].header = hdu0[0].header # Copy WCS header over
gc = aplpy.FITSFigure(hdu0, hdu=1, figsize=(8, 8))
z1, z2 = zscale.get_limits(im0)
gc.show_grayscale(invert=True, vmin=z1, vmax=z2)
gc.set_tick_color('black')
gc.set_tick_yspacing('auto')
gc.ticks.set_yspacing(1 / 60.0) # Every 1 arcmin in Dec
gc.set_tick_labels_format(xformat='hh:mm:ss', yformat='dd:mm')
txt0 = files[nn]
if exists(hdr_info_file):
txt0 += '\n'
tmp = tab0[nn]
txt0 += 'Date Label : ' + tmp['datelabel'] + '\n'
txt0 += 'UT Date : ' + tmp['UT_date'] + '\n'
txt0 += tmp['object'] + '\n'
txt0 += 'EXPTIME=%.1f ' % tmp['exptime']
txt0 += 'AIRMASS=%.3f \n' % tmp['airmass']
txt0 += '%s %.3f %s' % (tmp['grating'], tmp['gratwave'],
tmp['filter2'])
gc.add_label(0.975,
0.115,
txt0,
color='red',
relative=True,
ha='right',
va='bottom',
weight='medium',
size='medium',
bbox=bbox_props)
gc.savefig(pp, format='pdf')
if silent == False:
mylogger.info('Writing : ' + out_pdf) # Mod on 08/12/2017
pp.close()
#endelse
out_pdf = out_pdf_default
#endfor
if silent == False: mylogger.info('### End main : ' + systime())
def clean_QA(path0='',
out_pdf='',
silent=False,
verbose=True,
overwrite=False):
'''
Visually compare raw and cleanir-fixed FITS images
Parameters
----------
path0 : str
Full path to where output PDF and inputs FITS file are located. Must end
with a '/'
out_pdf : str
Filename for output PDF. Do NOT include full path
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
overwrite : boolean
Overwrite files if they exists. Default: False
Returns
-------
multi-page PDF plot
Notes
-----
Created by Chun Ly, 8 March 2017
Modified by Chun Ly, 10 March 2017
- Call quadrant_bias_values()
Modified by Chun Ly, 23 March 2017
- Call dir_check.main() to handle multiple date directories
Modified by Chun Ly, 25 April 2017
- Check cN*fits files if symlink.
Modified by Chun Ly, 26 April 2017
- Fix bug with call to dir_check()
- Change out_pdf default name (include clean suffix)
- Fix handling of symlink files; Check for False, not True
Modified by Chun Ly, 2 June 2017
- Added overwrite keyword option to overwrite file. Default is not to
overwrite .pdf files
Modified by Chun Ly, 8 December 2017
- Import glog and call for stdout and ASCII logging
Modified by Chun Ly, 17 December 2017
- Change glog logging to common path
- Add begin and end QA_clean logging to glog logfile
Modified by Chun Ly, 11 January 2018
- Pass mylogger to dir_check.main()
'''
# Moved up on 17/12/2017
logfile = path0 + 'QA_plot.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin clean_QA : ' + systime())
# + on 23/03/2017 | Mod on 26/04/2017
dir_list, list_path = dir_check.main(path0,
mylogger=mylogger,
silent=silent,
verbose=verbose)
out_pdf_default = out_pdf # + on 15/05/2017
# Mod on 23/03/2017
for path in list_path:
if out_pdf == '':
out_pdf = path + 'QA_plot.clean.pdf'
else:
out_pdf = path + out_pdf
files = glob.glob(path + 'cN*fits')
# Limit to non-symlinked files | + on 25/04/2017
files = [file for file in files if os.path.islink(file) == False]
n_files = len(files)
if overwrite == False and exists(out_pdf):
mylogger.warn('File exists!! Will not overwrite ' + out_pdf)
else:
pp = PdfPages(out_pdf)
for nn in xrange(n_files):
if silent == False: mylogger.info('Reading : ' + files[nn])
orig_file = files[nn].replace('cN', 'N')
im1 = fits.getdata(orig_file)
im2 = fits.getdata(files[nn])
#hdu1 = fits.open(orig_file)
#im1 = hdu1[1].data
#hdu1[1].header = hdu1[0].header # Copy WCS header over
#hdu2 = fits.open(files[nn])
#im2 = hdu2[1].data
#hdu2[1].header = hdu2[0].header # Copy WCS header over
fig = plt.figure(figsize=(16, 8))
gc1 = aplpy.FITSFigure(orig_file,
figure=fig,
subplot=[0.05, 0.055, 0.47, 0.94])
z1, z2 = zscale.get_limits(im1)
gc1.show_grayscale(invert=True, vmin=z1, vmax=z2)
gc1.set_tick_color('black')
gc1.set_tick_yspacing('auto')
#gc1.ticks.set_yspacing(1/60.0) # Every 1 arcmin in Dec
#gc1.set_tick_labels_format(xformat='hh:mm:ss', yformat='dd:mm')
gc1.add_label(0.025,
0.975,
orig_file,
color='red',
relative=True,
ha='left',
va='top',
weight='medium',
size='medium',
bbox=bbox_props)
quadrant_bias_values(fits.open(orig_file),
gc1) # + on 10/03/2017
gc2 = aplpy.FITSFigure(files[nn],
figure=fig,
subplot=[0.525, 0.055, 0.47, 0.94])
z1, z2 = zscale.get_limits(im2)
gc2.show_grayscale(invert=True, vmin=z1, vmax=z2)
gc2.set_tick_color('black')
gc2.set_tick_yspacing('auto')
gc2.hide_ytick_labels()
gc2.hide_yaxis_label()
gc2.add_label(0.025,
0.975,
files[nn],
color='red',
relative=True,
ha='left',
va='top',
weight='medium',
size='medium',
bbox=bbox_props)
quadrant_bias_values(fits.open(files[nn]),
gc2) # + on 10/03/2017
#gc2.ticks.set_yspacing(1/60.0) # Every 1 arcmin in Dec
#gc2.set_tick_labels_format(xformat='hh:mm:ss', yformat='dd:mm')
fig.savefig(pp, format='pdf')
if silent == False: mylogger.info('Writing : ' + out_pdf)
pp.close()
#endelse
out_pdf = out_pdf_default
#endfor
if silent == False: mylogger.info('### End clean_QA : ' + systime())
def main(path0, silent=False, verbose=True, overwrite=False):
'''
main() function to sort through data and create individual lists
Parameters
----------
path0 : str
Path to FITS file. Must include '/' at the end
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
ASCII files containing the filenames for each set
obj.lis - List of science data frames
sky.lis - List of science data frames to be used for sky-subtraction
arc.lis - List of Arclamp data frames
flat.lis - List of Flat data frames
telluric.lis - List of Telluric data frames
all.lis - List of all obj, sky, arc, flat, and telluric data frames
tab0 : astropy.table.Table
Astropy ASCII table with QA flag written to [path0]+'hdr_info.QA.tbl'
Notes
-----
Created by Chun Ly, 5 March 2017
- Later modified to output ASCII table (hdr_info.QA.tbl) containing a
column called 'QA' with information of how each FITS file was
classified. This is to enable quick check that each dataset if
properly classified
- Later modified to include all.lis output
- Later modified to define r0 (for slight speed improvement
Modified by Chun Ly, 23 March 2017
- Call dir_check.main() to handle multiple date directories
Modified by Chun Ly, 10 April 2017
- Handle cases when arc, flat, telluric, and sci data are not available
Modified by Chun Ly, 11 April 2017
- Handle case when no data for all.lis are available
(i.e., the observing sequence was canceled)
Modified by Chun Ly, 16 May 2017
- Change obj.lis and sky.lis to include both A-B and B-A sets
- Avoid QA flag of 'sky'
- Fix minor bug
Modified by Chun Ly, 17 May 2017
- Minor fix for i_sky to handle skipping of frames for i_obj
Modified by Chun Ly, 31 May 2017
- Minor fix for when no science data is available
Modified by Chun Ly, 1 June 2017
- Added overwrite keyword option to overwrite file
Modified by Chun Ly, 8 December 2017
- Import glog and call for stdout and ASCII logging
Modified by Chun Ly, 10 December 2017
- Adjust glog logging to a default file in path0, move up mylogger
definition
- Minor mylogger text changes
- Pass mylogger to dir_check.main()
Modified by Chun Ly, 12 January 2018
- Handle multiple telluric datasets
Modified by Chun Ly, 15 January 2018
- Handle multiple telluric datasets (cont'd) - Get indices for each dataset
Modified by Chun Ly, 16 January 2018
- Handle multiple telluric datasets (cont'd) - Update prefix and index lists
Modified by Chun Ly, 17 January 2018
- Create telluric.lis even for multi-telluric case
Modified by Chun Ly, 23 April 2018
- Handle no telluric data case
'''
logfile = path0 + 'create_list.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin main : ' + systime())
# + on 23/03/2017
dir_list, list_path = dir_check.main(path0,
mylogger=mylogger,
silent=silent,
verbose=verbose)
# Mod on 23/03/2017
for path in list_path:
infile = path + 'hdr_info.tbl'
if not exists(infile):
mylogger.warning('File does not exist : ' + infile)
mylogger.warning('Exiting!!! ' + systime())
return
if silent == False: mylogger.info('Reading: ' + infile)
tab0 = asc.read(infile, format='fixed_width_two_line')
len0 = len(tab0)
r0 = xrange(len0)
obstype = tab0['obstype']
object0 = tab0['object']
filter2 = tab0['filter2']
i_arc = [ii for ii in r0 if obstype[ii] == 'ARC']
i_flat = [ii for ii in r0 if obstype[ii] == 'FLAT']
i_tell = [
ii for ii in r0 if (obstype[ii] == 'OBJECT' and (
'HIP' in object0[ii] or 'HD' in object0[ii]) and (
'H2_' not in filter2[ii] and 'H_' not in filter2[ii]))
]
# Identify when multiple sets of telluric data is available
# Broken based on filename | + on 12/01/2018, Mod on 15/01/2018
# tel_name = list(set(np.array(object0[i_tell])))
multi_tell = 0
# Mod on 23/04/2018
if len(i_tell) != 0:
seq_tell = np.array([
np.int(str0.replace('.fits', '')[-4:])
for str0 in tab0['filename'][i_tell]
])
seq_diff = seq_tell[1:] - seq_tell[0:-1]
# +1 to factor first skip for seq_diff
seq_break = [
ii + 1 for ii in range(len(seq_diff)) if seq_diff[ii] != 1
]
if len(seq_break) > 0:
multi_tell = 1
i_tell0 = []
mylogger.warn('Multiple telluric star detected.')
mylogger.warn('Will split into separate files.')
ss_start = [0] + seq_break # relative to i_tell
ss_end = seq_break + [len(i_tell)]
for ss in range(len(ss_start)):
ss_idx = i_tell[ss_start[ss]:ss_end[ss]]
i_tell0.append(ss_idx)
else:
mylogger.info('Only one telluric star detected.')
i_sci = [
ii for ii in r0 if (obstype[ii] == 'OBJECT' and (
'HIP' not in object0[ii] and 'HD' not in object0[ii]) and (
'H2_' not in filter2[ii] and 'H_' not in filter2[ii]))
]
i_sci = np.array(i_sci)
# Note: Assumes that dithering pattern is ABA'B'
# Mod on 16/05/2017
# Mod on 17/05/2017 - Minor bug if things skip for sci frames
# Mod on 31/05/2017 - Minor bug if i_sci is empty
if len(i_sci) > 0:
i_obj = i_sci
i_off = [1, -1] * (len(i_sci) / 2)
if len(i_sci) % 2 == 1: i_off.append(-1) # Odd number correction
i_sky = i_sci[np.arange(len(i_sci)) + np.array(i_off)]
# i_sky = [a+b for a,b in zip(i_sci,i_off)]
# Mod on 10/04/2017
prefix, index = [], []
if len(i_arc) == 0:
mylogger.warn('No ARC data found!')
else:
prefix.append('arc')
index.append(i_arc)
if len(i_flat) == 0:
mylogger.warn('No FLAT data found!')
else:
prefix.append('flat')
index.append(i_flat)
if len(i_tell) == 0:
mylogger.warn('No Telluric data found!')
else:
# Mod on 16/01/2018, 17/01/2018
# Always create telluric.lis for all tellurics
prefix.append('telluric')
index.append(i_tell)
if multi_tell:
for ii in range(len(i_tell0)):
prefix.append('telluric' + str(ii + 1))
index.append(i_tell0[ii])
if len(i_sci) == 0:
mylogger.warn('No science data found!')
else:
prefix.append('obj')
prefix.append('sky')
index.append(i_obj)
index.append(i_sky)
zip0 = zip(prefix, index)
QA = ['N/A'] * len0 # Later + on 05/03/2017
for a, b in zip0:
if a != 'sky':
for idx in b:
QA[idx] = a # Mod on 16/05/2017
outfile = path + a + '.lis'
if overwrite == False and exists(outfile):
mylogger.warn('File exists! Will not write ' + outfile + '!!')
else:
if silent == False: mylogger.info('Writing : ' + outfile)
np.savetxt(outfile, tab0['filename'][b], fmt='%s')
#asc.write will not work. Will not produce single column
#asc.write(tab0[b], outfile, overwrite=True,
# format='no_header')
# Later + on 05/03/2017 | Mod on 11/04/2017
i_all = [ii for ii in r0 if QA[ii] != 'N/A']
if len(i_all) > 0:
outfile0 = path + 'all.lis'
if overwrite == False and exists(outfile0):
mylogger.warn('File exists! Will not write all.lis!!')
else:
if silent == False: mylogger.info('Writing : ' + outfile0)
np.savetxt(outfile0, tab0['filename'][i_all], fmt='%s')
else:
mylogger.warn('Will not write all.lis!!')
# Later + on 05/03/2017
col0 = Column(QA, name='QA')
tab0.add_column(col0)
# Later + on 05/03/2017
outfile2 = infile.replace('.tbl', '.QA.tbl')
if silent == False:
if overwrite == False and exists(outfile2):
mylogger.warn('File exists! Will not write ' + outfile2 + '!!')
else:
if not exists(outfile2):
mylogger.info('Writing : ' + outfile2)
else:
mylogger.info('Overwriting : ' + outfile2)
asc.write(tab0,
outfile2,
format='fixed_width_two_line',
overwrite=True)
if silent == False: mylogger.info('### End main : ' + systime())