def run(path0, silent=False, verbose=True):
'''
Main function to run for a given path containing the raw GNIRS data
Parameters
----------
path0 : str
Path to raw FITS file. Must include '/' at the end
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 22 March 2017
Modified by Chun Ly, 23 March 2017
- Call dir_check.main() to handle multiple date directories
Modified by Chun Ly, 8 January 2018
- Import glog and call for stdout and ASCII logging
- Pass mylogger to get_files()
Modified by Chun Ly, 7 May 2018
- Bug fix: Call check_path() to add '/' in path0
'''
rawdir = check_path(path0) # + on 07/05/2018
# + on 08/01/2018
logfile = path0+'symlink.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin run : '+systime())
# + on 23/03/2017
dir_list, list_path = dir_check.main(path0, mylogger=mylogger,
silent=silent, verbose=verbose)
# Mod on 23/03/2017
for path in list_path:
files, n_files = get_files(path, mylogger=mylogger, silent=silent,
verbose=verbose)
for nn in xrange(n_files):
c_file = path+'c'+files[nn]
if exists(c_file):
mylogger.warn('File exists : '+c_file)
else:
cmd0 = 'ln -fs '+files[nn]+' '+c_file
if silent == False:
mylogger.info(cmd0)
os.system(cmd0)
#endfor
#endfor
if silent == False: mylogger.info('### End run : '+systime())
def delete(path0, silent=False, verbose=True):
'''
Remove all symbolic links in given path containing the raw GNIRS data
Parameters
----------
path0 : str
Path to raw FITS file. Must include '/' at the end
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 22 March 2017
Modified by Chun Ly, 8 January 2018
- Import glog and call for stdout and ASCII logging
- Pass mylogger to get_files()
Modified by Chun Ly, 22 January 2018
- Pass rm command strings to mylogger
'''
# + on 08/01/2018
logfile = path0+'symlink.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin delete : '+systime())
files, n_files = get_files(path0, mylogger=mylogger, silent=silent,
verbose=verbose)
for nn in xrange(n_files):
c_file = path0+'c'+files[nn]
if exists(c_file):
if os.path.islink(c_file) == True:
cmd0 = 'rm '+c_file
mylogger.info(cmd0)
os.system(cmd0)
else:
mylogger.info('File is from cleanir: '+c_file)
if silent == False: mylogger.info('### End delete : '+systime())
def residual_wave_cal(path, dataset='', cal='', silent=False, verbose=True):
'''
Plot residuals of arc/OH lines against OH/arc dataset
Parameters
----------
path : str
Full path to where output PDF and FITS file are located. Must end
with a '/'
dataset : str
Either: 'arc' or 'OH'
cal : str
Either 'arc' or 'OH
To use OH calib on OH lines: dataset='OH', cal='OH'
To use OH calib on arc lines: dataset='arc', cal='OH'
To use arc calib on arc lines: dataset='arc', cal='arc'
To use arc calib on OH lines: dataset='OH', cal='arc'
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 4 June 2018
Modified by Chun Ly, 5 June 2018
- Define and read in OH/arc line database
- Import curve_fit
- Import gauss1d from IQ_plot
- Call curve_fit to fit Gaussian profiles to line
Modified by Chun Ly, 7 June 2018
- Plot residuals of wavelength solution
- Set minimum line peak brightness to fit
- Limit plotting to non-zero values for central wavelength
- Plot rms for each line
- Write average array to FITS file
- Plot aesthetics
- Move mylogger.info() call into else statement
- Handle dataset == cal cases
- Save average and rms for each arc/OH line
- Handle dataset == cal cases for output PDF file
- Plot aesthetics: Draw vertical lines for location of lines
- Handle dataset == cal cases for output FITS file
- If statement for lines with fits
- Write ASCII with averages and RMS file
Modified by Chun Ly, 19 June 2018
- Determine and plot average, median, and rms for each column
- Multi-page PDF: Separate plot illustrating difference from model
along longslit
- Plot sigma on second page, Exclude no value cases
- Add ax.annotation in upper left of second page
- Call get_database_model, include fitting model
- Overwrite ASCII tables in asc.write
Modified by Chun Ly, 20 June 2018
- Use FITS file if it exists, otherwise generate
- Use sigma_clipped_stats to remove outliers in statistics
- Remove outlier based on offset amount, not using sigma clipping
- Tab issue: Move fits.writeto out of for loop
- Compute stats for histogram distributions
- try/except for OH skyline failure (RunTimeError)
Modified by Chun Ly, 21 June 2018
- Use convolved Rousselot2000 table instead of original
- Read in OH npz file for line grouping
- Define use_lines array for arc case
- Handle multi-line fitting (works with dataset='arc')
- Handle multi-line fitting for dataset='OH'
- Include lower and upper bounds, Move multi-line fitting later in code
- Normalize spectrum to peak, Do not set bounds for curve_fit()
- Fix typo with np.absolute use
- Implement weighted average calculation for centers to expedite calculation
- Mark arc/OH lines that are not in database (potentially problematic)
- Ignore flag arc/OH lines in stats along X
Modified by Chun Ly, 21 June 2018
- Use default Rousselot table if convolved one not available
Note: This is OK with previous version of code that use the default table
for wavelength calibration (i.e., line_flag won't be all 1's)
- Handle old reduction when npz file is unavailable
'''
logfile = path+'QA_wave_cal.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin residual_wave_cal : '+systime())
if dataset != cal:
infile = path+'tf'+dataset+'_stack_'+cal+'.fits'
else:
infile = path+'tf'+dataset+'_stack.fits'
if not exists(infile):
mylogger.warn('File not found : '+infile)
mylogger.warn('Exiting!!!')
return
else:
if silent == False: mylogger.info('Reading : '+infile)
cal_2D, cal_hdr = fits.getdata(infile, extname='SCI', header=True)
NX = cal_2D.shape[1] # spatial direction
NY = cal_2D.shape[0] # dispersion direction
lam0 = cal_hdr['CRVAL2']
dlam = cal_hdr['CD2_2']
wave0 = lam0 + dlam * np.arange(NY)
bins_mid = np.arange(13,NX,10)
n_bins = len(bins_mid)
if dataset != cal:
out_fits = path+'wave_cal_resid_'+dataset+'_'+cal+'.fits'
else:
out_fits = path+'wave_cal_resid_'+dataset+'.fits'
if not exists(out_fits):
avg_arr = np.zeros( (NY, n_bins) )
for ii in range(n_bins):
start = np.max([0,bins_mid[ii]-1 - 5])
stop = np.min([NX-1,bins_mid[ii]-1 + 5])
avg_arr[:,ii] = np.average(cal_2D[:,start:stop], axis=1)
if silent == False: mylogger.info('Writing : '+out_fits)
fits.writeto(out_fits, avg_arr, overwrite=True)
else:
if silent == False: mylogger.info('Reading : '+out_fits)
avg_arr = fits.getdata(out_fits)
if dataset == 'OH':
cal_ref_file = path+'rousselot2000_convl.dat'
if not exists(cal_ref_file):
cal_ref_file = co_dirname+'/rousselot2000.dat'
mylogger.warn('Using default Rousselot2000 table, not convolved one!')
if dataset == 'arc':
cal_ref_file = co_dirname+'/argon.dat'
if silent == False: mylogger.info('Reading : '+cal_ref_file)
cal_line_data = asc.read(cal_ref_file, format='no_header')
cal_lines = cal_line_data['col1'].data
skip = np.zeros(len(cal_lines))
if dataset == 'OH':
npz_file = path+'rousselot2000_convl.npz'
if exists(npz_file):
mylogger.info('Reading : '+npz_file)
npz_tab = np.load(npz_file)
use_lines = npz_tab['use_lines']
for gg in range(len(use_lines)):
matches = [xx for xx in range(len(cal_lines)) if
cal_lines[xx] in use_lines[gg]]
if len(matches) > 1:
skip[np.array(matches[1:])] = 1
in_spec = np.where((cal_lines >= wave0[0]) &
(cal_lines <= wave0[-1]))[0]
cal_line_data = cal_line_data[in_spec]
cal_lines = cal_line_data['col1'].data
skip = skip[in_spec]
if dataset == 'OH':
cal_lines_int = cal_line_data['col2'].data
# Remove lines from use_lines if outside coverage | + on 21/06/2018
if exists(npz_file):
n_in_spec = np.zeros(len(use_lines))
for gg in range(len(use_lines)):
in_spec = np.where((use_lines[gg] >= wave0[0]) &
(use_lines[gg] <= wave0[-1]))[0]
n_in_spec[gg] = len(in_spec)
if len(in_spec) != len(use_lines[gg]) and len(in_spec) > 0:
use_lines[gg] = np.array(use_lines[gg])[in_spec].tolist()
ignore = np.where(n_in_spec == 0)[0]
if len(ignore) > 0:
use_lines = np.delete(use_lines, ignore)
n_lines = len(cal_lines)
# Use database for lines to avoid/use in analysis | + on 21/06/2018
func0, order0, l_val0 = get_database_model(path, dataset, get_lines=True)
line_flag = np.repeat(1,n_lines)
for ii in range(n_lines):
ii_fit = np.where(np.absolute(l_val0 - cal_lines[ii]) <= 0.1)[0]
if len(ii_fit) > 0: line_flag[ii] = 0
line_nouse = np.where(line_flag == 1)[0]
if dataset == 'arc':
use_lines = []
for ll in range(n_lines): use_lines.append([cal_lines[ll]])
use_lines = np.array(use_lines)
cen_arr = np.zeros( (n_lines,n_bins) )
if dataset != cal:
pdf_file = path+'wave_cal_resid_'+dataset+'_'+cal+'.pdf'
else:
pdf_file = path+'wave_cal_resid_'+dataset+'.pdf'
fig, ax = plt.subplots()
diff_avg = np.zeros(n_lines)
diff_rms = np.zeros(n_lines)
diff_num = np.zeros(n_lines, dtype=np.int)
for ll in range(n_lines):
#mylogger.info('ll=%i, u_l_ii=%i' % (ll, u_l_ii))
ax.axvline(cal_lines[ll], color='red', linestyle='dashed',
linewidth=0.25, zorder=1)
z_idx = np.where(np.absolute(wave0-cal_lines[ll]) <= 10)[0]
x0 = wave0[z_idx]
for ii in range(n_bins):
y0 = avg_arr[z_idx,ii]
y0 /= max(y0)
x_idx = np.where(np.absolute(x0 - cal_lines[ll]) <= 3)[0]
wht_val = np.sum(x0[x_idx]*y0[x_idx])/np.sum(y0[x_idx])
if np.absolute(wht_val-cal_lines[ll]) <= 1:
cen_arr[ll,ii] = wht_val
#endfor
good = np.where(cen_arr[ll] != 0)[0]
if len(good) > 0:
x_test = cen_arr[ll][good]
diff = x_test - cal_lines[ll]
ax.scatter(x_test, diff, marker='o', s=5, edgecolor='none',
facecolor='black', alpha=0.5, zorder=2)
diff_rms[ll], diff_avg[ll] = np.std(diff), np.average(diff)
diff_num[ll] = len(good)
#endif
#endfor
ax.scatter(cal_lines, diff_avg, marker='o', s=40, facecolor='blue',
edgecolor='none', alpha=0.5, zorder=3)
ax.errorbar(cal_lines, diff_avg, yerr=diff_rms, ecolor='blue',
capsize=2.0, elinewidth=2, fmt=None, alpha=0.5, zorder=4)
# Mark with 'X' lines that are not used in wavecal fitting
ax.scatter(cal_lines[line_nouse], diff_avg[line_nouse], marker='X', s=100,
facecolor='red', zorder=3)
ax.minorticks_on()
ax.set_xlabel(r'Wavelengths [$\AA$]')
ax.set_ylabel(r'Difference from reference values [$\AA$]')
func0, order0 = get_database_model(path, cal)
an_txt = 'dataset: %s calib: %s\n' % (dataset, cal)
an_txt += 'func: %s yorder: %i xorder: %i' % (func0, order0, xorder)
ax.annotate(an_txt, [0.05,0.95], xycoords='axes fraction',
ha='left', va='top')
plt.subplots_adjust(left=0.13, right=0.99, bottom=0.1, top=0.99)
fig.savefig(pdf_file)
asc_file = pdf_file.replace('.pdf', '.tbl')
tab0 = Table([cal_lines, diff_num, diff_avg, diff_rms],
names=('Line','N','Avg','RMS'))
asc.write(tab0, asc_file, format='fixed_width_two_line', overwrite=True)
# Determine average, median, and rms for each column | + on 19/06/2018
avg0 = np.zeros(n_bins)
N0 = np.zeros(n_bins)
med0 = np.zeros(n_bins)
rms0 = np.zeros(n_bins)
for nn in range(n_bins):
t_diff = cen_arr[:,nn] - cal_lines
good = np.where((cen_arr[:,nn] != 0) & (line_flag != 1) &
(np.absolute(t_diff) <= 1.5))[0]
if len(good) > 0:
N0[nn] = len(good)
avg0[nn] = np.average(t_diff[good])
med0[nn] = np.median(t_diff[good])
rms0[nn] = np.std(t_diff[good])
pdf_file2 = pdf_file.replace('.pdf', '.stat.pdf')
pp = PdfPages(pdf_file2)
fig, ax = plt.subplots(ncols=3)
good = np.where(rms0 != 0)
N1, b1, p1 = ax[0].hist(avg0[good], bins=10, align='mid', color='b',
alpha=0.5, edgecolor='none',
histtype='stepfilled')
N2, b2, p2 = ax[1].hist(med0[good], bins=10, align='mid', color='m',
alpha=0.5, edgecolor='none',
histtype='stepfilled')
N3, b3, p3 = ax[2].hist(rms0[good], bins=10, align='mid', color='k',
alpha=0.5, edgecolor='none',
histtype='stepfilled')
ax[0].set_ylabel('N')
ax[0].set_xlabel(r'Average [$\AA$]')
ax[1].set_xlabel(r'Median [$\AA$]')
ax[2].set_xlabel(r'$\sigma$ [$\AA$]')
stat_avg = [np.average(avg0[good]), np.average(med0[good]),
np.average(rms0[good])]
stat_med = [np.median(avg0[good]), np.median(med0[good]),
np.median(rms0[good])]
stat_rms = [np.std(avg0[good]), np.std(med0[good]), np.std(rms0[good])]
y_max = np.max([np.max(N1),np.max(N2),np.max(N3)])*1.1
for aa in range(3):
ax[aa].set_ylim([0,y_max])
ax[aa].axvline(stat_avg[aa], linestyle='solid')
ax[aa].axvline(stat_med[aa], linestyle='dashed')
txt0 = ''
str0 = [r'$<x>$', r'$\tilde{x}$', r'$\sigma$']
val = [stat_avg[aa], stat_med[aa], stat_rms[aa]]
for ss,vv in zip(str0,val): txt0 += ss+(' = %.3f' % vv) +'\n'
ax[aa].annotate(txt0, [0.95,0.95], xycoords='axes fraction',
ha='right', va='top')
ax[1].set_yticklabels([])
ax[2].set_yticklabels([])
ax[1].tick_params(axis='y', direction='in')
ax[2].tick_params(axis='y', direction='in')
plt.subplots_adjust(left=0.08, right=0.99, bottom=0.12, top=0.96,
wspace=0.03)
fig.set_size_inches(8,4)
fig.savefig(pp, format='pdf')
asc_file2 = pdf_file2.replace('.pdf', '.tbl')
tab2 = Table([bins_mid, N0, avg0, med0, rms0],
names=('Column','N','Avg','Med','RMS'))
tab2.write(asc_file2, format='ascii.fixed_width_two_line',
overwrite=True)
fig, ax = plt.subplots()
ax.scatter(bins_mid[good], avg0[good], marker='o', color='b',
alpha=0.5, edgecolor='none', label='Average')
ax.scatter(bins_mid[good], med0[good], marker='o', color='m',
alpha=0.5, edgecolor='none', label='Median')
ax.scatter(bins_mid[good], rms0[good], marker='o', color='k',
alpha=0.5, edgecolor='none', label=r'$\sigma$')
ax.annotate(an_txt, [0.05,0.95], xycoords='axes fraction',
ha='left', va='top')
ax.legend(loc='lower right', fancybox=True) #frameon=False)
ax.set_ylabel(r'Average / Median [$\AA$]')
ax.set_xlabel(r'X (Along longslit) [pixels]')
ax.minorticks_on()
fig.subplots_adjust(left=0.11, right=0.99, bottom=0.12, top=0.96)
fig.set_size_inches(8,4)
fig.savefig(pp, format='pdf')
pp.close()
if silent == False: mylogger.info('### End residual_wave_cal : '+systime())
def get_database_model(path, source, get_lines=False, silent=False, verbose=True):
'''
Determine fitting function and order to pass to nsfitcoords
Parameters
----------
path : str
Full path to where output PDF and FITS file are located. Must end
with a '/'
source : str
Either 'arc' or 'OH'
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Notes
-----
Created by Chun Ly, 19 June 2018
Modified by Chun Ly, 21 May 2017
- Add get_lines keyword to get arc/OH reference lines
Modified by Chun Ly, 22 May 2017
- Handle INDEF case
- Bug fix: get lines, not index
'''
logfile = path+'QA_wave_cal.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin get_database_model : '+systime())
ref_file = source+'_stack.fits'
if source == 'arc': dbase = 'database'
if source == 'OH': dbase = 'database_OH'
d_file = '%s%s/idw%s' % (path, dbase, ref_file.replace('.fits','_SCI_1_'))
mylogger.info("Reading : "+d_file)
f0 = open(d_file, 'r')
str0 = f0.readlines()
ii_beg = [ii for ii in xrange(len(str0)) if 'begin' in str0[ii]]
ii_feat = [ii for ii in xrange(len(str0)) if 'features' in str0[ii]]
ii_func = [ii for ii in xrange(len(str0)) if 'function' in str0[ii]]
ii_ord = [ii for ii in xrange(len(str0)) if 'order' in str0[ii]]
func0 = str0[ii_func[0]].split(' ')[-1].replace('\n','')
order0 = np.int(str0[ii_ord[0]].split(' ')[-1].replace('\n',''))
if get_lines: # + on 21/06/2018
n_features = np.int(str0[ii_feat[0]].split('\t')[2].replace('\n',''))
l_val0 = np.zeros(n_features)
for ff in range(n_features):
temp1 = str0[ii_feat[0]+1+ff].replace(' ','')
temp1 = temp1.split(' ')
temp1 = [val for val in temp1 if val != '']
if temp1[3] != 'INDEF': # Mod on 22/06/2018
l_val0[ff] = np.float(temp1[3])
l_val0 = l_val0[l_val0.nonzero()] # + on 22/06/2018
mylogger.info('Function for '+source+' line fitting : '+func0)
mylogger.info('Order of '+source+' line fitting : %i' % order0)
f0.close()
if silent == False: mylogger.info('### End get_database_model : '+systime())
# Mod on 21/06/2018
if not get_lines:
return func0, order0
else:
return func0, order0, l_val0
def arc_check2(path, arcs=[''], out_pdf='', stack=False, cross_check=False,
silent=False, verbose=True):
'''
Generate plot illustrating expected location of arc lines to
check wavelength calibration
Parameters
----------
path : str
Full path to where output PDF and FITS file are located. Must end
with a '/'
arcs : str or list (Optional)
List of raw filenames for the arc data (e.g., 'N20170101S0111.fits')
stack : boolean
Indicate whether to include stacked arc data. Default: False
cross_check : boolean
Check OH-based wavelength calibration against arc lines. Default: False
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 16 June 2017
- Started as a copy of OH_check
Modified by Chun Ly, 17 July 2017
- Fix bug with arcs handling
Modified by Chun Ly, 12 November 2017
- Added stack keyword option to include stacked arc products
Modified by Chun Ly, 16 November 2017
- Change prefix: tfrnc to tfrbnc
Modified by Chun Ly, 9 January 2018
- Import glog and call for stdout and ASCII logging
Modified by Chun Ly, 31 May 2018
- Add cross_check keyword; Update input/outputs for cross_check == True
'''
# + on 09/01/2018
logfile = path+'QA_wave_cal.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin arc_check2 : '+systime())
arc_file = co_dirname+'/argon.dat'
if exists(arc_file):
if silent == False: mylogger.info('Reading : '+arc_file)
arc_data = asc.read(arc_file, format='commented_header')
arc_lines = arc_data['Line'].data
arc_source = arc_data['Source'].data
if arcs[0] == '':
arcs_list = path + 'arc.lis'
if silent == False: mylogger.info('Reading : '+arcs_list)
arcs = np.loadtxt(arcs_list, dtype=type(str))
tfrnc_files = [path+'tfrbnc'+file0 for file0 in arcs] # Mod on 16/11/2017
# + on 12/11/2017
if stack == True:
arcs0 = [path + 'arc_stack.fits']
tfrnc_files0 = [path + 'tfarc_stack.fits']
arcs = arcs0 + arcs
tfrnc_files = tfrnc_files0 + tfrnc_files
# + on 31/05/2018
if cross_check == True:
arcs = [path + 'arc_stack_OH.fits']
tfrnc_files = [path + 'tfarc_stack_OH.fits']
chk = [file0 for file0 in tfrnc_files if exists(file0) == True]
if len(chk) == 0:
mylogger.warn('Files not found!!!')
mylogger.warn(', '.join(file0))
mylogger.warn('Exiting!!!')
return
n_arcs = len(arcs) # Moved lower on 12/11/2017
# Mod on 31/05/2018
if cross_check == False:
out_pdf = path+'arc_check2.pdf' if out_pdf == '' else path+out_pdf
else:
out_pdf = path+'arc_check_OH.pdf' if out_pdf == '' else path+out_pdf
pp = PdfPages(out_pdf)
for nn in xrange(n_arcs):
if exists(tfrnc_files[nn]): # Mod on 20/05/2017
hdu0 = fits.open(tfrnc_files[nn])
im0 = hdu0['SCI'].data
gc0 = aplpy.FITSFigure(hdu0, hdu='SCI', figsize=(7.65,10.5))
z1, z2 = zscale.get_limits(im0)
gc0.show_grayscale(invert=True, vmin=z1, vmax=z2)
gc0.set_tick_color('black')
# + on 20/05/2017
hdr0 = hdu0['SCI'].header
crval2 = hdr0['CRVAL2']
cdelt2 = hdr0['CDELT2']
npix = hdr0['NAXIS2']
lamb_max = crval2 + cdelt2*npix
arc_mark = np.where((arc_lines >= crval2) & (arc_lines <= lamb_max))[0]
line_list = []
for ll in arc_mark:
line_list.append(np.array([[0,700], [arc_lines[ll],arc_lines[ll]]]))
gc0.show_lines(line_list, color='red', alpha=0.5, linewidth=1.0,
linestyle='dashed')
str0 = tfrnc_files[nn].replace(path,'')
gc0.add_label(0.025, 0.975, str0, color='red', relative=True,
ha='left', va='top', weight='medium', fontsize=14,
bbox=bbox_props)
gc0.set_axis_labels(xlabel='X [pixels]', ylabel=r'Wavelength ($\AA$)')
gc0.savefig(pp, format='pdf')
#endif
#endfor
if silent == False: mylogger.info('Writing : '+out_pdf)
pp.close()
if silent == False: mylogger.info('### End arc_check2 : '+systime())
def main(rawdir, silent=False, verbose=True):
'''
Main function for get_OH_centers
Parameters
----------
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 12 June 2018
Modified by Chun Ly, 13 June 2018
- Fix rev_lines and rev_int (wrong indexing)
- Switch to micron units; Do subplots_adjust for white space
- Plot aesthetics: label plots, change page size, legend
- Include parameters/assumptions in title
- Plot aesthetics: Vertical lines for all possible OH skylines
- Plot aesthetics: Label OH skylines
- Plot aesthetics: Limit vertical lines for OH skylines
- Plot aesthetics: group and label OH skylines to avoid overlap
- Group lines (<5 Ang) before annotation
- Opaque white background behind lines
- Plot aesthetics: Draw OH lines to top of subplots
- Plot aesthetics: Remove legend; adjust white space
- WARN if more than six lines
- Attempt to constraint fit using bounds but that did not work
- Fix case if curve_fit solutions are outside of spectral range
Modified by Chun Ly, 21 June 2018
- Write npz file containing use lines that are grouped together
- mylogger call for writing files
- Switching back to uncompressed npz - Got interpret file pickle error (IOError)
Modified by Chun Ly, 25 June 2018
- Bug fix: Crash on call to group_OH_lines. Require in_zoom to not be empty array
Modified by Chun Ly, 25 June 2018
- Bug fix: Handle odd fit results (check wavelength within 5 Ang of guess)
- Bug fix: Add try/except for curve_fit RuntimeError
- Bug fix: tab fix
- Bug fix: Add try/except for curve_fit RuntimeError for gauss_multi
- Bug fix: list to np.array
'''
# + on 09/01/2018
logfile = rawdir + 'get_OH_centers.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin main : ' + systime())
if exists(OH_file):
if silent == False: mylogger.info('Reading : ' + OH_file)
OH_data = np.loadtxt(OH_file)
OH_lines = OH_data[:, 0]
OH_int = OH_data[:, 1]
infile = rawdir + 'hdr_info.QA.tbl'
tab0 = asc.read(infile, format='fixed_width_two_line')
i_obj = [xx for xx in range(len(tab0)) if tab0['QA'][xx] == 'obj'][0]
o_tab0 = tab0[i_obj]
gratwave = o_tab0['gratwave']
slitwidth = np.float(o_tab0['slit'].split('arcsec')[0])
if '111/mm' in o_tab0['grating']:
if o_tab0['filter2'] == 'X_G0518':
R_spec = 6600.0 / (slitwidth / 0.3)
x_diff = 0.06
if o_tab0['filter2'] == 'J_G0517':
R_spec = 7200.0 / (slitwidth / 0.3)
x_diff = 0.07
x_min = (gratwave - x_diff) * 1.E4
x_max = (gratwave + x_diff) * 1.E4
npix = np.round((x_max - x_min) / 0.15)
x0 = x_min + 0.15 * np.arange(npix)
OH_spec_mod = np.zeros(len(x0))
in_rge = np.where((OH_lines >= x0[0]) & (OH_lines <= x0[-1]))[0]
for ii in range(len(in_rge)):
idx = in_rge[ii]
temp = OH_int[idx] * gaussian_R(x0, OH_lines[idx], R_spec)
OH_spec_mod += temp
y_max = max(OH_spec_mod) * 1.25
i_lines = np.where(OH_spec_mod >= np.max(OH_spec_mod) * 0.01)[0]
lines_set = list(group(i_lines))
OH_spec_mod_resid = OH_spec_mod.copy()
rev_lines = [] #np.zeros(len(lines_set))
rev_int = [] #np.zeros(len(lines_set))
nrows = 3
fig, ax = plt.subplots(nrows=nrows)
use_lines = [] # + on 21/06/2018
xlim_arr = []
dx = (x0[-1] - x0[0]) / 3.0
for aa in range(nrows):
ax[aa].plot(x0 / 1e4,
OH_spec_mod,
color='black',
zorder=3,
label="Rousselot (2000)")
xlim = np.array([x_min + dx * aa, x_min + dx * (aa + 1)])
xlim_arr.append(xlim)
ax[aa].set_xlim(xlim / 1e4)
ax[aa].set_ylim([-10, y_max])
# Draw vertical lines for all possible OH skylines
for val in OH_lines[in_rge]:
ax[aa].axvline(x=val / 1e4,
color='black',
linewidth=0.25,
linestyle=':',
zorder=2)
for ii in range(len(lines_set)):
x_avg = np.int(np.average(lines_set[ii]))
tl_min, tl_max = x0[lines_set[ii][0]], x0[lines_set[ii][1]]
in_zoom = np.where((OH_lines >= tl_min) & (OH_lines <= tl_max))[0]
if len(in_zoom) > 0:
group_lines, group_int = group_OH_lines(OH_lines[in_zoom],
OH_int[in_zoom])
# print ii, group_lines
sig = group_lines / R_spec / (2 * np.sqrt(2 * np.log(2)))
peak0 = OH_spec_mod[np.int_(
(group_lines - x_min) / (x0[1] - x0[0]))]
if len(group_lines) == 1:
p0 = [0.0, peak0, group_lines[0], sig[0]]
#plt.axvline(group_lines[0]/1e4, color='blue')
else:
if len(group_lines) > n_multi:
log.warn('More than 8 lines found, N=' +
str(len(group_lines)) + '!!!')
t_peak0 = peak0.tolist()
t_lines = group_lines.tolist()
t_sig = sig.tolist()
t_peak0 += np.zeros(n_multi - len(group_lines)).tolist()
t_lines += np.zeros(n_multi - len(group_lines)).tolist()
t_sig += np.zeros(n_multi - len(group_lines)).tolist()
p0 = t_peak0
p0 += t_lines
p0 += t_sig
p0 = np.array(p0)
zoom = np.arange(lines_set[ii][0], lines_set[ii][1])
# print p0
#bounds = ((-0.001, 0.0, lam_cen-0.5, 0.1),
# (0.001, 1.25*p0[1], lam_cen+0.5, 1.5*p0[3]))
if len(group_lines) == 1:
try:
popt, pcov = curve_fit(gauss1d,
x0[zoom],
OH_spec_mod[zoom],
p0=p0)
except RuntimeError:
mylogger.warn('Did not converge!')
mylogger.warn('Using initial guess : %.3f' %
group_lines[0])
popt = list(p0)
if np.absolute(popt[2] - p0[2]) >= 5:
mylogger.warn('Reliable fit not determined!')
mylogger.warn('Using initial guess : %.3f' %
group_lines[0])
popt = list(p0)
t_mod = gauss1d(x0, *popt)
use_lines.append([popt[2]]) # + on 21/06/2018
rev_lines.append(popt[2])
rev_int.append(popt[1])
i_ax = [
xx for xx in range(nrows) if
(popt[2] >= xlim_arr[xx][0] and popt[2] <= xlim_arr[xx][1])
][0]
ax[i_ax].annotate('%.2f' % popt[2],
[popt[2] / 1e4, y_max * 0.99],
ha='center',
va='top',
rotation=90,
fontsize=4,
bbox=bbox_props)
else:
#low_bound = tuple([0] * n_multi) + tuple(p0[n_multi:2*n_multi]-0.5) + \
# tuple([0] * n_multi)
#up_bound = tuple(p0[0:n_multi]*1.25+0.1) + tuple(p0[n_multi:2*n_multi]+0.5) + \
# tuple(p0[2*n_multi:]+1)
try:
popt, pcov = curve_fit(gauss_multi,
x0[zoom],
OH_spec_mod[zoom],
p0=p0)
except RuntimeError:
mylogger.warn('Did not converge!')
mylogger.warn('Using initial guesses')
popt = np.array(p0)
t_mod = gauss_multi(x0, *popt)
t_loc = popt[n_multi:2 * n_multi] # Line wavelengths (Ang)
t_str = popt[0:n_multi] # Line peak strength
nonzero = np.where(t_loc != 0)[0]
use_lines.append(t_loc[nonzero].tolist()) # + on 21/06/2018
wave0 = t_loc[nonzero]
wave0.sort()
# Check that lines are within range
in_rge = np.where((wave0 >= x_min) & (wave0 <= x_max))[0]
wave0 = wave0[in_rge]
t_str = t_str[nonzero[in_rge]]
rev_lines += wave0.tolist()
rev_int += t_str.tolist()
wave0 = np.array(wave0)
# Label lines
skip = np.zeros(len(wave0))
for ww in range(len(wave0)):
if skip[ww] == 0:
w_diff = wave0[ww:] - wave0[ww]
t_close = np.where(w_diff <= 5)[0]
close = np.arange(ww, len(wave0))[t_close]
str_comb = "\n".join(
['%.2f' % val for val in wave0[close]])
w_cen = np.average(wave0[close])
#0.5*(wave0[0]+wave0[-1]) #np.average(wave0)
i_ax = [
xx for xx in range(nrows)
if (w_cen >= xlim_arr[xx][0]
and w_cen <= xlim_arr[xx][1])
][0]
ax[i_ax].annotate(str_comb,
[w_cen / 1e4, y_max * 0.99],
ha='center',
va='top',
rotation=90,
fontsize=4,
bbox=bbox_props)
skip[close] = 1
#endif
#endfor
# print '## t_mod : ', np.min(t_mod), np.max(t_mod)
OH_spec_mod_resid -= t_mod
for aa in range(nrows):
ax[aa].plot(x0 / 1e4,
OH_spec_mod_resid,
linestyle='dashed',
color='blue',
zorder=3,
label='Residual')
for t_line in rev_lines:
ax[aa].axvline(x=t_line / 1e4,
color='red',
linestyle='--',
linewidth=1.0,
zorder=1)
l_tab = Table([rev_lines, rev_int])
out_file = rawdir + 'rousselot2000_convl.dat'
mylogger.info('Writing : ' + out_file)
asc.write(l_tab, out_file, format='no_header')
ann_txt = r'%.2f $\mu$m; %s; ' % (gratwave, o_tab0['filter2'])
ann_txt += '%s; %.3f" slit; R = %i' % (o_tab0['grating'], slitwidth,
R_spec)
ax[0].set_title(ann_txt)
#leg = ax[0].legend(loc='upper right', fancybox=True) #, frameon=False)
#leg.get_frame().set_alpha(0.75)
ax[2].set_xlabel(r'Wavelength [$\mu$m]')
plt.subplots_adjust(left=0.08,
right=0.95,
bottom=0.06,
top=0.96,
hspace=0.12)
fig.set_size_inches(6, 8)
out_pdf = out_file.replace('.dat', '.pdf')
mylogger.info('Writing : ' + out_pdf)
fig.savefig(out_pdf)
# Write npz file containing final grouping | + on 21/06/2018
savez_file = out_file.replace('.dat', '.npz')
mylogger.info('Writing : ' + savez_file)
np.savez(savez_file, use_lines=use_lines)
if silent == False: mylogger.info('### End main : ' + systime())
def run(path0, clean_file='', out_script='', silent=False, verbose=True,
overwrite=False):
'''
Create a .sh script to run cleanir.py for a set of files
Parameters
----------
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
overwrite : boolean
Overwrite files if they exists. Default: False
Returns
-------
Notes
-----
Created by Chun Ly, 7 March 2017
Modified by Chun Ly, 15 May 2017
- Call dir_check.main() to handle multiple date directories
Modified by Chun Ly, 30 May 2017
- Added overwrite option. Default is to not overwrite .sh files
- Fix bug when only one file is found
Modified by Chun Ly, 18 June 2017
- Fix to work with no date directory
Modified by Chun Ly, 22 January 2018
- Import glog and call for stdout and ASCII logging
- Pass mylogger to dir_check.main()
Modified by Chun Ly, 20 April 2018
- Change afq to rfq - row filtering generally produces better results
'''
logfile = path0+'cleanir_script.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin run : '+systime())
if clean_file == '': clean_file = 'clean.lis'
# + on 15/05/2017
dir_list, list_path = dir_check.main(path0, mylogger=mylogger, silent=silent,
verbose=verbose)
# Mod on 15/05/2017
for date,path in zip(dir_list,list_path):
clean_file0 = path+clean_file
if not exists(clean_file0):
mylogger.warn('File does not exist!!!')
mylogger.warn(clean_file0)
else:
if silent == False: mylogger.info('Reading : '+clean_file0)
files = np.loadtxt(clean_file0, dtype=type(str)).tolist()
if type(files) == str: files = [files] # Bug fix. Mod on 30/05/2017
out_script0 = path+'run_cleanir.sh' if out_script == '' \
else out_script
if date != '': # Mod on 18/06/2017
out_script0 = out_script0.replace('.sh', '.'+date+'.sh')
# Mod on 30/05/2017
if overwrite == False and exists(out_script0):
log.warn('## File found!!! : '+out_script0)
log.warn('## Will not overwrite!!!')
else:
if silent == False:
stat0 = 'Overwriting' if exists(out_script0) else 'Writing'
mylogger.info(stat0+' : '+out_script0)
f = open(out_script0, 'w')
for ii in xrange(len(files)):
cmd1 = cmd0+' -rfqo '+path+'c'+files[ii]+' '+path+files[ii]
f.write(cmd1+'\n')
f.close()
#endelse
#endelse
#endfor
if silent == False: mylogger.info('End run : '+systime())
def main(rawdir, out_pdf='', gz=False, silent=False, verbose=True):
'''
Main function to compute and plot statistics
Parameters
----------
rawdir : str
Path to raw files.
gz : boolean
Indicate using gz-compressed files (mostly for debugging). Default: False
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 8 May 2018
- Change output PDF filename
- Plot averages of statistic measurements
- Plot expected Poissonian level
- Plot aesthetics (legend, limits, x/ylabel)
- Compute and plot rms of stacked image
Modified by Chun Ly, 21 May 2018
- Bug fix: Mistake in computing RMS for combined 2-D data
- Write RMS to mylogger; Label 'Stack' RMS on right side of plot
- Add gz keyword option to use with compressed files
Modified by Chun Ly, 28 May 2018
- Add '/' after rawdir if not provided
'''
if rawdir[-1] != '/': rawdir = rawdir + '/'
logfile = rawdir + 'image_stats.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin run ! ')
dir_list, list_path = dir_check.main(rawdir,
mylogger=mylogger,
silent=silent,
verbose=verbose)
out_pdf_default = out_pdf
for path in list_path:
file_lis = path + 'obj.lis'
if not exists(file_lis):
if silent == False: mylogger.info('File not found : ' + file_lis)
else:
if silent == False: mylogger.info('Reading : ' + file_lis)
t_files = np.loadtxt(file_lis, dtype=type(str)).tolist()
files0 = [path + 'tfrbnc' + t_file for t_file in t_files]
if gz == True: # + on 21/05/2018
files0 = [t_file + '.gz' for t_file in files0]
n_files0 = len(files0)
avg_arr = np.zeros(n_files0)
med_arr = np.zeros(n_files0)
sig_arr = np.zeros(n_files0)
for nn in range(n_files0):
im0 = fits.getdata(files0[nn], extname='SCI')
c_mean, c_med, c_sig = sigma_clipped_stats(im0,
sigma=2.0,
iters=10)
avg_arr[nn] = c_mean
med_arr[nn] = c_med
sig_arr[nn] = c_sig
#endfor
#print avg_arr, med_arr, sig_arr
if out_pdf == '':
out_pdf = path + 'image_stats.pdf'
else:
out_pdf = path + out_pdf
fig, ax_arr = plt.subplots(nrows=2)
num0 = 1 + np.arange(n_files0)
ax_arr[0].scatter(num0,
avg_arr,
marker='o',
s=50,
facecolor='none',
edgecolor='k',
label='Mean')
avg_avg = np.average(avg_arr)
ax_arr[0].axhline(y=avg_avg, c='k', linestyle='dashed')
ax_arr[0].scatter(num0,
med_arr,
marker='x',
s=25,
color='b',
label='Median')
avg_med = np.average(med_arr)
ax_arr[0].axhline(y=avg_med, c='b', linestyle='dotted')
ax_arr[0].legend(loc='lower right', fontsize=8)
ax_arr[1].scatter(num0,
sig_arr,
marker='o',
s=50,
facecolor='none',
edgecolor='k')
avg_sig = np.average(sig_arr)
ax_arr[1].axhline(y=avg_sig, c='k', linestyle='dashed')
# Expected Poissionan level
sig_poisson = avg_sig / np.sqrt(n_files0)
mylogger.info('Expected (Poisson) RMS : %.4f' %
sig_poisson) # + on 21/05/2018
ax_arr[1].axhline(y=sig_poisson, c='b', linestyle='dashed')
ax_arr[1].text(0,
sig_poisson,
'Poisson',
color='b',
ha='left',
va='bottom')
ax_arr[0].minorticks_on()
ax_arr[1].minorticks_on()
ax_arr[0].set_xticklabels([])
ax_arr[0].set_xlim([-0.25, n_files0 + 1])
ax_arr[1].set_xlim([-0.25, n_files0 + 1])
ax_arr[1].set_ylabel(r'$\sigma$')
ax_arr[1].set_xlabel('Frame No.')
# Plot rms of stacked image
comb_file = glob.glob(path + 'obj_comb.fits')
if len(comb_file) != 0:
mylogger.info('Reading : ' + comb_file[0])
comb_data = fits.getdata(comb_file[0],
extname='SCI') # Mod on 21/05/2018
# Mod on 21/05/2018
c_mean, c_med, c_sig = sigma_clipped_stats(comb_data,
sigma=2.0,
iters=10)
mylogger.info('Measured RMS in stack : %.4f' %
c_sig) # + on 21/05/2018
ax_arr[1].axhline(y=c_sig, c='g', linestyle='dashed')
ax_arr[1].text(n_files0 + 1,
c_sig,
'Stack',
color='g',
ha='right',
va='bottom') # Mod on 21/05/2018
#fig, ax_arr = plt.subplots(ncols=2)
#ax_arr[0].hist(avg_arr, bins=5, align='mid', color='b',
# linestyle='solid', alpha=0.5, edgecolor='b',
# histtype='step', label='Average')
#ax_arr[0].hist(med_arr, bins=5, align='mid', # color='g',
# linestyle='dashed', edgecolor='k',
# histtype='step', label='Median')
#ax_arr[1].hist(sig_arr, bins=5, align='mid', color='k',
# linestyle='solid', alpha=0.5, edgecolor='k',
# histtype='step')
mylogger.info('Writing : ' + out_pdf)
fig.set_size_inches(8, 4)
fig.savefig(out_pdf, bbox_inches='tight')
plt.close()
out_pdf = out_pdf_default
#endfor
if silent == False: mylogger.info('### End run ! ')
def run(rawdir, silent=False, verbose=False):
'''
Main function to combine science data to produce a 2-D FITS image
containing OH night skylines for wavelength calibration. Median
filtering is used to remove detected objects
Parameters
----------
rawdir : str
Path to raw files. Must end in a '/'
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
2-D image containing OH skyline called 'OH_stack.fits'
Notes
-----
Created by Chun Ly, 25 June 2017
Modified by Chun Ly, 20 September 2017
- Call check_path()
Modified by Chun Ly, 16 November 2017
- Change prefix: rnc to rbnc
Modified by Chun Ly, 9 January 2018
- Import glog and call for stdout and ASCII logging
'''
# + on 09/01/2018
logfile = rawdir+'OH_stack.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin run : '+systime())
rawdir = check_path(rawdir) # + on 20/09/2017
obj_list = rawdir + 'obj.lis'
if silent == False: mylogger.info('Reading : '+obj_list)
objs = np.loadtxt(obj_list, dtype=type(str))
rbnc_files = [rawdir+'rbnc'+file0.replace('.fits','.OH.fits') for
file0 in objs] # Mod on 16/11/2017
# Mod on 16/11/2017
hdu0 = fits.open(rbnc_files[0])
hdr = fits.getheader(rbnc_files[0], extname='SCI')
naxis1 = hdr['NAXIS1']
naxis2 = hdr['NAXIS2']
arr0 = np.zeros((len(rbnc_files), naxis2, naxis1)) # Mod on 16/11/2017
for ii in range(len(rbnc_files)):
if verbose == True: mylogger.info('Reading : '+obj_list)
t_data = fits.getdata(rbnc_files[ii], extname='SCI') # Mod on 16/11/2017
t_med0 = np.median(t_data, axis=0) # Median along columns
# Remove median along columns
med_off = np.repeat(t_med0, naxis2).reshape(naxis1,naxis2).transpose()
arr0[ii] = t_data - med_off
# Compute median
med_arr0 = np.median(arr0, axis=0)
hdu0['SCI'].data = med_arr0
hdu0['VAR'].data = np.zeros((naxis2,naxis1))
outfile = rawdir+'OH_stack.fits'
stat0 = 'Overwriting' if exists(outfile) else 'Writing'
if silent == False:
mylogger.info(stat0+' : '+outfile)
hdu0.writeto(outfile, output_verify='ignore', overwrite=True)
if silent == False: mylogger.info('### End run : '+systime())
def plot_spec(rawdir, out_pdf='', silent=False, verbose=False):
'''
Plot median spectrum of OH skylines
Parameters
----------
rawdir : str
Path to raw files. Must end in a '/'
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Produces 'OH_spec.pdf'
Notes
-----
Created by Chun Ly, 4 July 2017
- Overlay Rousselot (2000) model spectrum
Modified by Chun Ly, 20 September 2017
- Call check_path()
Modified by Chun Ly, 9 January 2018
- Implement glog stdout and ASCII logging
'''
# + on 09/01/2018
logfile = rawdir+'OH_stack.log'
mylogger = glog.log0(logfile)._get_logger()
rawdir = check_path(rawdir) # + on 20/09/2017
out_pdf = rawdir+'OH_spec.pdf' if out_pdf == '' else rawdir+out_pdf
im0, hdr = fits.getdata(rawdir+'tfOH_stack.fits', extname='SCI',
header=True)
OH_med0 = np.median(im0, axis=1)
crval2 = hdr['CRVAL2']
cd2_2 = hdr['CD2_2']
crpix = hdr['CRPIX2']
x0 = crval2 + cd2_2*np.arange(len(OH_med0))
x0 /= 1e4 # In microns
y0 = OH_med0/max(OH_med0)
fig, ax = plt.subplots()
if exists(OH_file):
if silent == False: mylogger.info('Reading : '+OH_file)
OH_data = np.loadtxt(OH_file)
OH_lines = OH_data[:,0] / 1E4
OH_int = OH_data[:,1]
i_max = np.where(OH_med0 == max(OH_med0))[0]
l_max = x0[i_max[0]]
p0 = [0.0, max(y0), l_max, 2.0/1e4]
popt, pcov = curve_fit(gauss1d, x0, y0, p0=p0)
fwhm = popt[3]*2*np.sqrt(2*np.log(2))
R_spec = l_max / fwhm
OH_spec_mod = np.zeros(len(x0))
in_rge = np.where((OH_lines >= x0[0]) & (OH_lines <= x0[-1]))[0]
for ii in range(len(in_rge)):
idx = in_rge[ii]
#ax.plot(np.repeat(OH_lines[in_rge[ii]],2), [0,1.1], 'r--')
temp = OH_int[idx] * gaussian_R(x0, OH_lines[idx], R_spec)
OH_spec_mod += temp
ax.plot(x0, y0, 'k-', label='OH_stack')
OH_spec_mod /= np.max(OH_spec_mod)
ax.plot(x0, OH_spec_mod, 'r--', alpha=0.75,
label='OH model (Rousselot 2000)')
ax.set_xlabel(r'Wavelength ($\mu$m)', fontsize=16)
ax.set_ylabel('Normalized Flux', fontsize=16)
ax.set_ylim([0.0, 1.10])
ax.minorticks_on()
ax.tick_params(labelsize=14)
subplots_adjust(left=0.07, right=0.99, bottom=0.07, top=0.99)
ax.legend(loc='upper right', fontsize=14, frameon=False)
fig.set_size_inches(11,8)
fig.savefig(out_pdf)
def transform(rawdir, silent=False, verbose=False):
'''
Transform OH_stack 2-D FITS image for wavelength calibration checks
Parameters
----------
rawdir : str
Path to raw files. Must end in a '/'
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
2-D image with transformation called 'fOH_stack.fits' and 'tfOH_stack.fits'
Notes
-----
Created by Chun Ly, 4 July 2017
Modified by Chun Ly, 20 September 2017
- Call check_path()
Modified by Chun Ly, 22 November 2017
- Add file checking and log.warn calls
Modified by Chun Ly, 29 November 2017
- Bug fix: Missing else statement
Modified by Chun Ly, 9 January 2018
- Import glog and call for stdout and ASCII logging
Modified by Chun Ly, 19 June 2018
- Call QA_wave_cal.get_database_model
- Pass function and order to nsfitcoords for OH stack
Modified by Chun Ly, 20 June 2018
- Set nsfitcoords xorder fitting
Modified by Chun Ly, 22 June 2018
- Call residual_wave_cal for wavelength solution check
'''
# + on 09/01/2018
logfile = rawdir+'OH_stack.log'
mylogger = glog.log0(logfile)._get_logger()
cdir = os.getcwd()+'/' # + on 06/05/2017
rawdir = check_path(rawdir) # + on 20/09/2017
iraf.chdir(rawdir)
mylogger.info("Running nsfitcoords on OH_stack") # Mod on 09/01/2018
outfile1 = rawdir + 'fOH_stack.fits'
if not exists(outfile1):
func0, order0 = QA_wave_cal.get_database_model(rawdir, 'OH')
iraf.gnirs.nsfitcoords('wOH_stack.fits', outprefix='',
outspectra='fOH_stack.fits',
lamp='wOH_stack.fits', database='database_OH/',
function=func0, lyorder=order0,
lxorder=QA_wave_cal.xorder)
else:
# Mod on 09/01/2018
mylogger.warn('File exists!!! : '+outfile1)
mylogger.warn('Will not run nsfitcoords on OH stacked data')
outfile2 = rawdir + 'tfOH_stack.fits'
if not exists(outfile2):
iraf.gnirs.nstransform('fOH_stack.fits', outprefix='',
outspectra='tfOH_stack.fits',
database='database_OH/')
else:
# Mod on 09/01/2018
mylogger.warn('File exists!!! : '+outfile2)
mylogger.warn('Will not run nstransform on OH stacked data')
iraf.chdir(cdir)
# + on 22/06/2018
QA_wave_cal.residual_wave_cal(rawdir, dataset='OH', cal='OH')
def wave_cal(rawdir, cdir, silent=False, verbose=False):
'''
Run gnirs.nswavelength on OH_stack 2-D FITS image for wavelength
calibration
Parameters
----------
rawdir : str
Path to raw files. Must end in a '/'
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
2-D image with transformation called 'fOH_stack.fits' and 'tfOH_stack.fits'
Notes
-----
Created by Chun Ly, 13 July 2017
Modified by Chun Ly, 20 September 2017
- Call check_path()
Modified by Chun Ly, 16 November 2017
- Call wave_cal_script to get PyRAF code
Modified by Chun Ly, 16 November 2017
- Bug fix: indentation typo with else statement
Modified by Chun Ly, 20 November 2017
- Bug fix: Pass in cdir
Modified by Chun Ly, 9 January 2018
- Import glog and call for stdout and ASCII logging
Modified by Chun Ly, 14 June 2018
- Import and call get_OH_centers
Modified by Chun Ly, 19 June 2018
- Pass gnirs logfile to mylogger
'''
# + on 09/01/2018
logfile = rawdir+'OH_stack.log'
mylogger = glog.log0(logfile)._get_logger()
rawdir = check_path(rawdir) # + on 20/09/2017
iraf.chdir(rawdir)
timestamp = systime().replace(':','.')
logfile = rawdir+'gnirs_'+timestamp+'.log'
iraf.gemini.gnirs.logfile = logfile
mylogger.info("GNIRS logfile : "+logfile)
get_OH_centers.main(rawdir)
# + on 16/11/2017
script_file = 'wave_cal_OH.py'
if not exists(script_file):
wave_cal_script.main(rawdir, line_source='OH')
else:
# Mod on 09/01/2018
mylogger.info('File exists!!! : '+script_file)
mylogger.info('Will not override!!!')
# + on 16/11/2017
do_run = 0
if not exists('wOH_stack.fits'): do_run = 1
if do_run:
# Mod on 09/01/2018
mylogger.info("In order to perform interactive calibration, open up")
mylogger.info("a PyRAF terminal in an anaconda IRAF environment")
mylogger.info("'cd' into "+rawdir)
mylogger.info("Execute the following command :")
mylogger.info("execfile('"+script_file+"')")
t_out = raw_input("## Hit RETURN when OH wavelength calibration is completed")
else:
# Mod on 09/01/2018
mylogger.warn('Files exist!!!')
mylogger.warn('Will not run nswavelength on OH stacked data')
iraf.chdir(cdir)
def main(path0, silent=False, verbose=True, overwrite=False):
'''
main() function to obtain information from FITS header and write to
ASCII file
Parameters
----------
path0 : str
Path to FITS file. Must include '/' at the end
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
tab0 : astropy.table.Table
Astropy ASCII table written to [path0]+'hdr_info.tbl'
Notes
-----
Created by Chun Ly, 4 March 2017
- Later re-organized to check for file first
Modified by Chun Ly, 5 March 2017
- File exists warning always printed out
- Include AIRMASS
Modified by Chun Ly, 23 March 2017
- Call dir_check.main() to handle multiple date directories
Modified by Chun Ly, 11 May 2017
- Handle longer filter1 and filter2 FITS values
Modified by Chun Ly, 8 December 2017
- Import glog and call for stdout and ASCII logging
- Pass mylogger to dir_check.main()
'''
# Moved up on 10/12/2017
logfile = path0 + 'hdr_info.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin main : ' + systime())
dir_list, list_path = dir_check.main(path0,
mylogger=mylogger,
silent=silent,
verbose=verbose)
# Mod on 23/03/2017
for path in list_path:
outfile = path + 'hdr_info.tbl'
# Mod later on 04/03/2017 to not overwrite file
# Mod on 05/03/2017 to always print out this warning
if overwrite == False and exists(outfile):
# Mod on 08/12/2017
mylogger.warning('File exists : ' + outfile)
mylogger.warning('Not over-writing!!! ')
else:
fits_files = glob.glob(path + 'N*fits')
n_files = len(fits_files)
# Mod on 05/03/2017 to include airmass
names0 = ('filename', 'datelabel', 'UT_date', 'obstype', 'object',
'exptime', 'airmass', 'grating', 'gratwave', 'filter1',
'filter2', 'slit')
dtype0 = ('S20', 'S30', 'S25', 'S8', 'S100', 'f8', 'f8', 'S15',
'f8', 'S20', 'S20', 'S20')
tab0 = Table(names=names0, dtype=dtype0)
for nn in xrange(n_files):
basename = os.path.basename(fits_files[nn])
if silent == False: mylogger.info('Reading : ' + basename)
h0 = fits.getheader(fits_files[nn])
# Mod on 05/03/2017 to include airmass
vec0 = [
basename, h0['DATALAB'], h0['DATE-OBS'] + 'T' + h0['UT'],
h0['OBSTYPE'], h0['OBJECT'], h0['EXPTIME'], h0['AIRMASS'],
h0['GRATING'], h0['GRATWAVE'], h0['FILTER1'],
h0['FILTER2'], h0['SLIT']
]
tab0.add_row(vec0)
if silent == False:
mylogger.info('Writing : ' + outfile) # Mod on 08/12/2017
asc.write(tab0, outfile, format='fixed_width_two_line')
#endelse
#endfor
if silent == False: mylogger.info('### End main : ' + systime())
def main(file_list,
path0='',
out_pdf='',
silent=False,
verbose=True,
overwrite=False):
'''
main() function to read in each FITS image and display it on a zscale
using aplpy
Parameters
----------
file_list : str
Filename of ASCII file containing images to be plotted. Use 'all.lis'
from create_list() since this will include all that will be processed
Do NOT include full path
path0 : str
Full path to where output PDF and FITS file are located. Must end
with a '/'
out_pdf : str
Filename for output PDF. Do NOT include full path
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
overwrite : boolean
Overwrite files if they exists. Default: False
Returns
-------
multi-page PDF plot
Notes
-----
Created by Chun Ly, 6 March 2017
Modified by Chun Ly, 23 March 2017
- Call dir_check.main() to handle multiple date directories
Modified by Chun Ly, 1 June 2017
- Added overwrite keyword option to overwrite file. Default is not to
overwrite .pdf files
Modified by Chun Ly, 8 December 2017
- Import glog and call for stdout and ASCII logging
Modified by Chun Ly, 17 December 2017
- Change glog logging to common path
- Add begin and end QA_clean logging to glog logfile
- Pass mylogger to dir_check.main()
'''
# Moved up on 17/12/2017
logfile = path0 + 'QA_plot.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin main : ' + systime())
# + on 23/03/2017
dir_list, list_path = dir_check.main(path0,
mylogger=mylogger,
silent=silent,
verbose=verbose)
out_pdf_default = out_pdf
# Mod on 23/03/2017
for path in list_path:
if silent == False:
mylogger.info('Reading : ' + path + file_list) # Mod on 08/12/2017
files = np.loadtxt(path + file_list, dtype=type(str)).tolist()
n_files = len(files)
if out_pdf == '':
out_pdf = path + 'QA_plot.pdf'
else:
out_pdf = path + out_pdf
hdr_info_file = path + 'hdr_info.tbl'
if exists(hdr_info_file):
if silent == False:
mylogger.info('Reading : ' +
hdr_info_file) # Mod on 08/12/2017
tab0 = asc.read(hdr_info_file, format='fixed_width_two_line')
idx1, idx2 = match_nosort_str(files, tab0['filename'])
tab0 = tab0[idx2]
else:
if silent == False:
mylogger.warn('File not found : ' +
hdr_info_file) # Mod on 08/12/2017
if overwrite == False and exists(out_pdf):
mylogger.warn('File exists!! Will not overwrite ' +
out_pdf) # Mod on 08/12/2017
else:
pp = PdfPages(out_pdf)
for nn in xrange(n_files):
if silent == False:
mylogger.info('Reading : ' +
files[nn]) # Mod on 08/12/2017
# h0 = fits.getheader(path+files[nn], 0)
# im0 = fits.getdata(path+files[nn], 1)
hdu0 = fits.open(path + files[nn])
im0 = hdu0[1].data
hdu0[1].header = hdu0[0].header # Copy WCS header over
gc = aplpy.FITSFigure(hdu0, hdu=1, figsize=(8, 8))
z1, z2 = zscale.get_limits(im0)
gc.show_grayscale(invert=True, vmin=z1, vmax=z2)
gc.set_tick_color('black')
gc.set_tick_yspacing('auto')
gc.ticks.set_yspacing(1 / 60.0) # Every 1 arcmin in Dec
gc.set_tick_labels_format(xformat='hh:mm:ss', yformat='dd:mm')
txt0 = files[nn]
if exists(hdr_info_file):
txt0 += '\n'
tmp = tab0[nn]
txt0 += 'Date Label : ' + tmp['datelabel'] + '\n'
txt0 += 'UT Date : ' + tmp['UT_date'] + '\n'
txt0 += tmp['object'] + '\n'
txt0 += 'EXPTIME=%.1f ' % tmp['exptime']
txt0 += 'AIRMASS=%.3f \n' % tmp['airmass']
txt0 += '%s %.3f %s' % (tmp['grating'], tmp['gratwave'],
tmp['filter2'])
gc.add_label(0.975,
0.115,
txt0,
color='red',
relative=True,
ha='right',
va='bottom',
weight='medium',
size='medium',
bbox=bbox_props)
gc.savefig(pp, format='pdf')
if silent == False:
mylogger.info('Writing : ' + out_pdf) # Mod on 08/12/2017
pp.close()
#endelse
out_pdf = out_pdf_default
#endfor
if silent == False: mylogger.info('### End main : ' + systime())
def clean_QA(path0='',
out_pdf='',
silent=False,
verbose=True,
overwrite=False):
'''
Visually compare raw and cleanir-fixed FITS images
Parameters
----------
path0 : str
Full path to where output PDF and inputs FITS file are located. Must end
with a '/'
out_pdf : str
Filename for output PDF. Do NOT include full path
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
overwrite : boolean
Overwrite files if they exists. Default: False
Returns
-------
multi-page PDF plot
Notes
-----
Created by Chun Ly, 8 March 2017
Modified by Chun Ly, 10 March 2017
- Call quadrant_bias_values()
Modified by Chun Ly, 23 March 2017
- Call dir_check.main() to handle multiple date directories
Modified by Chun Ly, 25 April 2017
- Check cN*fits files if symlink.
Modified by Chun Ly, 26 April 2017
- Fix bug with call to dir_check()
- Change out_pdf default name (include clean suffix)
- Fix handling of symlink files; Check for False, not True
Modified by Chun Ly, 2 June 2017
- Added overwrite keyword option to overwrite file. Default is not to
overwrite .pdf files
Modified by Chun Ly, 8 December 2017
- Import glog and call for stdout and ASCII logging
Modified by Chun Ly, 17 December 2017
- Change glog logging to common path
- Add begin and end QA_clean logging to glog logfile
Modified by Chun Ly, 11 January 2018
- Pass mylogger to dir_check.main()
'''
# Moved up on 17/12/2017
logfile = path0 + 'QA_plot.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin clean_QA : ' + systime())
# + on 23/03/2017 | Mod on 26/04/2017
dir_list, list_path = dir_check.main(path0,
mylogger=mylogger,
silent=silent,
verbose=verbose)
out_pdf_default = out_pdf # + on 15/05/2017
# Mod on 23/03/2017
for path in list_path:
if out_pdf == '':
out_pdf = path + 'QA_plot.clean.pdf'
else:
out_pdf = path + out_pdf
files = glob.glob(path + 'cN*fits')
# Limit to non-symlinked files | + on 25/04/2017
files = [file for file in files if os.path.islink(file) == False]
n_files = len(files)
if overwrite == False and exists(out_pdf):
mylogger.warn('File exists!! Will not overwrite ' + out_pdf)
else:
pp = PdfPages(out_pdf)
for nn in xrange(n_files):
if silent == False: mylogger.info('Reading : ' + files[nn])
orig_file = files[nn].replace('cN', 'N')
im1 = fits.getdata(orig_file)
im2 = fits.getdata(files[nn])
#hdu1 = fits.open(orig_file)
#im1 = hdu1[1].data
#hdu1[1].header = hdu1[0].header # Copy WCS header over
#hdu2 = fits.open(files[nn])
#im2 = hdu2[1].data
#hdu2[1].header = hdu2[0].header # Copy WCS header over
fig = plt.figure(figsize=(16, 8))
gc1 = aplpy.FITSFigure(orig_file,
figure=fig,
subplot=[0.05, 0.055, 0.47, 0.94])
z1, z2 = zscale.get_limits(im1)
gc1.show_grayscale(invert=True, vmin=z1, vmax=z2)
gc1.set_tick_color('black')
gc1.set_tick_yspacing('auto')
#gc1.ticks.set_yspacing(1/60.0) # Every 1 arcmin in Dec
#gc1.set_tick_labels_format(xformat='hh:mm:ss', yformat='dd:mm')
gc1.add_label(0.025,
0.975,
orig_file,
color='red',
relative=True,
ha='left',
va='top',
weight='medium',
size='medium',
bbox=bbox_props)
quadrant_bias_values(fits.open(orig_file),
gc1) # + on 10/03/2017
gc2 = aplpy.FITSFigure(files[nn],
figure=fig,
subplot=[0.525, 0.055, 0.47, 0.94])
z1, z2 = zscale.get_limits(im2)
gc2.show_grayscale(invert=True, vmin=z1, vmax=z2)
gc2.set_tick_color('black')
gc2.set_tick_yspacing('auto')
gc2.hide_ytick_labels()
gc2.hide_yaxis_label()
gc2.add_label(0.025,
0.975,
files[nn],
color='red',
relative=True,
ha='left',
va='top',
weight='medium',
size='medium',
bbox=bbox_props)
quadrant_bias_values(fits.open(files[nn]),
gc2) # + on 10/03/2017
#gc2.ticks.set_yspacing(1/60.0) # Every 1 arcmin in Dec
#gc2.set_tick_labels_format(xformat='hh:mm:ss', yformat='dd:mm')
fig.savefig(pp, format='pdf')
if silent == False: mylogger.info('Writing : ' + out_pdf)
pp.close()
#endelse
out_pdf = out_pdf_default
#endfor
if silent == False: mylogger.info('### End clean_QA : ' + systime())
def cross_check(path, cdir, dbase):
'''
Check arc/OH calibration against OH/arc dataset
Parameters
----------
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
skysub : boolean
Display skysubtracted or un-skysubtracted images. Default: False
Returns
-------
Notes
-----
Created by Chun Ly, 31 May 2018
- Generate transformed image with iraf.gnirs.nstransform
- Call mylogger to warn if files exist
- Change gnirs logfile
- Move call to iraf.gemini.nsheaders to this function
- iraf.nstransform does not like suffixes. Using underscores
- Call OH_check
- Check for database file before running nsfitcoords and nstransform
Modified by Chun Ly, 3 June 2018
- Bug fix: Call OH_check with cross_check=True
- Call arc_check2 for database_OH case
Modified by Chun Ly, 8 June 2018
- Exit when db_file not available
Modified by Chun Ly, 19 June 2018
- Call get_database_model
- Pass function and order to nsfitcoords
Modified by Chun Ly, 20 June 2018
- Set nsfitcoords xorder fitting
'''
logfile = path+'QA_wave_cal.log'
mylogger = glog.log0(logfile)._get_logger()
timestamp = systime().replace(':','.')
logfile = path+'gnirs_'+timestamp+'.log'
iraf.gemini.gnirs.logfile = logfile
iraf.gemini.nsheaders("gnirs")
mylogger.info("Raw data is located in : %s" % path)
mylogger.info("GNIRS logfile : "+logfile)
iraf.chdir(path)
if dbase == 'database/':
infile = 'OH_stack.fits'
outfile = 'fOH_stack_arc.fits'
lamp = 'warc_stack.fits'
if dbase == 'database_OH/':
infile = 'arc_stack.fits'
outfile = 'farc_stack_OH.fits'
lamp = 'wOH_stack.fits'
db_file = dbase+'id'+lamp.replace('.fits','_SCI_1_')
if not exists(db_file):
mylogger.warn("Wavelength calibration file not found : "+db_file)
mylogger.warn("Exiting!!!")
return
else:
if not exists(outfile):
source0 = 'OH' if '_OH' in dbase else 'arc'
func0, order0 = get_database_model(path, source0)
iraf.gnirs.nsfitcoords(infile, outprefix='', outspectra=outfile,
lamp=lamp, database=dbase,
function=func0, lyorder=order0, lxorder=xorder)
else:
mylogger.warn('File exists! : '+outfile)
t_outfile = 't'+outfile
if not exists(t_outfile):
iraf.gnirs.nstransform(outfile, outprefix='', outspectra=t_outfile,
database=dbase)
else:
mylogger.warn('File exists! : '+t_outfile)
iraf.chdir(cdir)
if dbase == 'database/':
OH_check(path, cross_check=True)
# + on 03/06/2018
if dbase == 'database_OH/':
arc_check2(path, cross_check=True)
def main(path0='',
out_pdf='',
check_quality=True,
skysub=False,
silent=False,
verbose=True,
overwrite=False):
'''
main() function to compute natural seeing (image quality) from bright
alignment star
Parameters
----------
path0 : str
Full path to where output PDF and FITS file are located. Must end
with a '/'
out_pdf : str
Filename for output PDF. Do NOT include full path
check_quality : boolean
Check whether data meets IQ requirements. Default: True
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
overwrite : boolean
Overwrite files if they exists. Default: False
Returns
-------
multi-page PDF plot
Notes
-----
Created by Chun Ly, 10 March 2017
- Later modified to include check_quality keyword option
- Later modified to include inset that shows the stacked line profile
Modified by Chun Ly, 11 April 2017
- Call dir_check.main() to handle multiple date directories
Modified by Chun Ly, 13 April 2017
- Minor bug: Check if file exists first
Modified by Chun Ly, 10 May 2017
- Minor bug: When .lis file contains only one entry, problem for
appending to list
Modified by Chun Ly, 1 June 2017
- Added overwrite keyword option to overwrite file. Default is not to
overwrite .pdf files
- Bug found: No longer need sky.lis since obj.lis includes all
Modified by Chun Ly, 6 June 2017
- Add skysub keyword option to operate on sky-subtracted images
Modified by Chun Ly, 14 July 2017
- Fix tick mark locations
- Fix y limit range for extreme outliers
Modified by Chun Ly, 16 November 2017
- Change prefix: rnc to rbnc
Modified by Chun Ly, 18 December 2017
- Import glog and call for stdout and ASCII logging
- Pass mylogger to compute_fwhm()
Modified by Chun Ly, 11 January 2018
- Pass mylogger to dir_check.main()
Modified by Chun Ly, 18 April 2018
- Compute and report seeing at zenith
- Show FWHM @ Zenith on right y-axis
Modified by Chun Ly, 19 April 2018
- Include airmass info in plots
Modified by Chun Ly, 14 May 2018
- Write ASCII table containing FWHM determination
Modified by Chun Ly, 18 May 2018
- Adjust subplots_adjust to avoid labels being cut off
- Handle case when extra plot window (odd numbers) is available
- Handle case with extra plot window (cont'd)
Modified by Chun Ly, 19 May 2018
- Include QA (PASS/USABLE/FAIL) info in table
Modified by Chun Ly, 28 June 2018
- Bug troubleshooting with ValueError
- Handle ValueError for avg FWHM
'''
# + on 18/12/2017
logfile = path0 + 'IQ_plot.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin main : ' + systime())
# + on 11/04/2017
dir_list, list_path = dir_check.main(path0,
mylogger=mylogger,
silent=silent,
verbose=verbose)
out_pdf_default = out_pdf
# Mod on 11/04/2017
for path in list_path:
files = []
file_lis = ['obj.lis', 'telluric.lis'] # Minor bug fix on 01/06/2017
for file0 in file_lis:
# Mod on 13/04/2017
if exists(path + file0):
if silent == False: mylogger.info('Reading : ' + path + file0)
t_files = np.loadtxt(path + file0, dtype=type(str)).tolist()
# Bug fix - 10/05/2017
if type(t_files) == str: t_files = [t_files]
files += t_files
else:
if silent == False:
mylogger.info('File not found : ' + path + file0)
files.sort()
n_files = len(files)
# + on 14/05/2018
FWHM_avg_arr = np.zeros(n_files) # FWHM from averaging stack0_shift
FWHM_avg_arr_Z = np.zeros(
n_files) # FWHM from averaging stack0_shift at Zenith
FWHM_avg_QA = [
''
] * n_files # QA check from averaging stack0_shift at Zenith
FWHM_med_arr = np.zeros(n_files) # FWHM from median along Y
FWHM_med_arr_Z = np.zeros(
n_files) # FWHM from median along Y at Zenith
FWHM_med_QA = [''] * n_files # QA check from median along Y at Zenith
# Mod on 06/06/2017
if skysub == True:
files = ['brnc' + file0 for file0 in files] # Mod on 16/11/2017
out_pdf = path+'IQ_plot.skysub.pdf' if out_pdf == '' else \
path+out_pdf
else:
out_pdf = path+'IQ_plot.raw.pdf' if out_pdf == '' else \
path+out_pdf
if overwrite == False and exists(out_pdf):
mylogger.warn('File exists!! Will not overwrite ' + out_pdf)
else:
pp = PdfPages(out_pdf)
for nn in xrange(n_files):
if silent == False: mylogger.info('Reading : ' + files[nn])
hdr0 = fits.getheader(path + files[nn])
# Mod on 06/06/2017
if skysub == False:
im0 = fits.getdata(path + files[nn])
else:
im0 = fits.getdata(path + files[nn], 'sci')
airmass = hdr0['AIRMASS'] # + on 18/04/2018
# Mod on 18/12/2017
bins, fwhm0, stack0_shift = compute_fwhm(im0,
mylogger=mylogger)
row = nn % 2
if row == 0: fig, ax0 = plt.subplots(2, 1)
good = np.where(fwhm0 > 0)[0]
ax0[row].plot(bins[good],
fwhm0[good],
marker='o',
alpha=0.5,
mec='none',
mfc='b',
linestyle='none',
zorder=2)
ax0[row].get_yaxis().set_tick_params(which='both',
right=True,
width=1,
direction='in')
ax0[row].get_xaxis().set_tick_params(which='both',
top=True,
width=1,
direction='in')
# Compute average line profile from stack0_shift
# Later + on 10/03/2017
avg_stack = np.average(stack0_shift, axis=0)
x0_avg = np.arange(-25, 25)
if row == 0: axi = fig.add_axes([0.60, 0.81, 0.25, 0.15])
if row == 1: axi = fig.add_axes([0.60, 0.36, 0.25, 0.15])
axi.plot(x0_avg * pscale, avg_stack, 'k-')
axi.set_ylim([-0.4, 1.1])
axi.set_xlim([-2.0, 2.0])
axi.set_xticks(range(-2, 2, 1))
axi.set_xlabel('X [arcsec]', fontsize=8)
axi.tick_params(labelsize=8)
axi.minorticks_on()
p0 = [0.0, 1.0, 0.0, 2.0]
try:
popt, pcov = curve_fit(gauss1d, x0_avg, avg_stack, p0=p0)
fit_good = 1
except ValueError:
print len(np.where(np.isnan(x0_avg))[0])
print len(np.where(np.isnan(avg_stack))[0])
fit_good = 0
if fit_good:
avg_fwhm0 = popt[3] * 2 * np.sqrt(2 * np.log(2)) * pscale
avg_fwhm_Z = avg_fwhm0 / airmass**0.6 # + on 18/04/2018
axi.plot(x0_avg * pscale, gauss1d(x0_avg, *popt), 'r--')
axi.annotate('FWHM = %.3f" (%.3f")' %
(avg_fwhm0, avg_fwhm_Z), [0.50, 0.025],
xycoords='axes fraction',
ha='center',
va='bottom',
fontsize=8)
ax0[row].axhline(y=avg_fwhm0,
linewidth=2,
color='r',
linestyle='--',
zorder=1)
# Median FWHM | Later + on 10/03/2017
med_fwhm0 = np.median(fwhm0[good])
med_fwhm0_Z = med_fwhm0 / airmass**0.6 # + on 18/04/2018
ax0[row].axhline(y=med_fwhm0,
linewidth=2,
color='g',
linestyle='--',
zorder=1)
# Axes labeling
ax0[row].set_ylabel('FWHM [arcsec]')
if row == 1 or nn == n_files - 1:
ax0[row].set_xlabel('Y [pixel]')
else:
ax0[row].set_xticklabels([])
if nn == n_files - 1:
if row == 0:
ax0[row + 1].axis('off')
# Annotation
txt0 = files[nn] + '\nTarget: ' + hdr0[
'OBJECT'] #.split(' ')[0]
ax0[row].annotate(txt0, [0.025, 0.95],
xycoords='axes fraction',
ha='left',
va='top')
# + on 19/04/2018
ax0[row].annotate('AM = %.3f' % airmass, [0.025, 0.025],
xycoords='axes fraction',
ha='left',
va='bottom')
# Later + on 10/03/2017
if check_quality:
req = hdr0['REQIQ'].replace('-percentile', '%')
raw = hdr0['RAWIQ'].replace('-percentile', '%')
i_raw = [
ii for ii in range(len(FWHM_IQ_C))
if raw in FWHM_IQ_C[ii]
][0]
i_req = [
ii for ii in range(len(FWHM_IQ_C))
if raw in FWHM_IQ_C[ii]
][0]
txt0 = 'Req. IQ: %s [%.2f"]\n' % (req, FWHM_IQ_J[i_req])
txt0 += 'Raw IQ: %s [%.2f"]\n' % (raw, FWHM_IQ_J[i_raw])
# Mod on 18/04/2018
if med_fwhm0_Z <= FWHM_IQ_J[i_raw]:
txt0 += 'PASS'
FWHM_med_QA[nn] = 'PASS' # + on 19/05/2018
else:
if med_fwhm0_Z <= FWHM_IQ_J[i_raw] * 1.25:
txt0 += 'USABLE'
FWHM_med_QA[nn] = 'USABLE' # + on 19/05/2018
if med_fwhm0_Z > FWHM_IQ_J[i_raw] * 1.25:
txt0 += 'FAIL'
FWHM_med_QA[nn] = 'FAIL' # + on 19/05/2018
# + on 19/05/2018
if fit_good:
if avg_fwhm_Z <= FWHM_IQ_J[i_raw]:
FWHM_avg_QA[nn] = 'PASS'
else:
if avg_fwhm_Z <= FWHM_IQ_J[i_raw] * 1.25:
FWHM_avg_QA[nn] = 'USABLE'
if avg_fwhm_Z > FWHM_IQ_J[i_raw] * 1.25:
FWHM_avg_QA[nn] = 'FAIL'
ax0[row].annotate(txt0, [0.975, 0.05],
ha='right',
xycoords='axes fraction',
va='bottom')
# Aesthetics
ax0[row].set_xlim([0, 1050])
if max(fwhm0[good]) > 3:
ax0[row].set_ylim([0, 3.0])
else:
ax0[row].set_ylim([min(fwhm0) - 0.025, max(fwhm0) + 0.075])
ax0[row].minorticks_on()
# + on 18/04/2018
ax2 = ax0[row].twinx()
ax2.set_ylabel(r"FWHM @ Zenith [arcsec]")
ax2.set_ylim(np.array(ax0[row].get_ylim()) / airmass**0.6)
ax2.minorticks_on()
if row != 1:
ax2.set_xticklabels([])
if row == 1 or nn == n_files - 1:
subplots_adjust(left=0.11,
bottom=0.10,
top=0.975,
right=0.875,
wspace=0.03,
hspace=0.05)
fig.savefig(pp, format='pdf')
# + on 14/05/2018
if fit_good:
FWHM_avg_arr[nn] = avg_fwhm0
FWHM_avg_arr_Z[nn] = avg_fwhm_Z
FWHM_med_arr[nn] = med_fwhm0
FWHM_med_arr_Z[nn] = med_fwhm0_Z
#endfor
if silent == False: mylogger.info('Writing : ' + out_pdf)
pp.close()
# + on 14/05/2018
fwhm_out_file = out_pdf.replace('.pdf', '.tbl')
arr0 = [
files, FWHM_avg_arr, FWHM_avg_arr_Z, FWHM_med_arr,
FWHM_med_arr_Z, FWHM_avg_QA, FWHM_med_QA
] # Mod on 19/05/2018
names0 = ('files', 'FWHM_avg', 'FWHM_avg_Z', 'FWHM_med',
'FWHM_med_Z', 'FWHM_avg_QA', 'FWHM_med_QA'
) # Mod on 19/05/2018
tab0 = Table(arr0, names=names0)
if silent == False: mylogger.info('Writing : ' + fwhm_out_file)
asc.write(tab0,
fwhm_out_file,
format='fixed_width_two_line',
overwrite=True)
out_pdf = out_pdf_default
#endelse
#endfor
if silent == False: mylogger.info('### End main : ' + systime())
def create_distort_grid(rawdir, silent=False, verbose=True):
'''
Create grid (and plot) of distortion for extraction
Parameters
----------
None
Returns
-------
None
Notes
-----
Created by Chun Ly, 29 June 2018
Modified by Chun Ly, 10 August 2018
- Plot best fit
Modified by Chun Ly, 13 August 2018
- Rewrite to work for specified rawdir path (instead of all)
- Fix typo with ax.axhline indexing
- Re-organize code (handle only one distort_trace.npz file)
- Minor bug fix: array name
- Plot aesthetics
'''
if rawdir[-1] != '/': rawdir += '/'
logfile = rawdir + 'distort_trace.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin ! ')
dir_list, list_path = dir_check.main(rawdir,
mylogger=mylogger,
silent=silent,
verbose=verbose)
for path in list_path:
mylogger.info('Working on : ' + path.split('/')[-2])
npz_files = glob(path + '/distort_trace.npz')
if len(npz_files) == 0:
log.warn('No files found!!!')
else:
fig, ax = plt.subplots(nrows=3)
npz = np.load(npz_files[0])
xcen_arr = npz['xcen_arr']
fit_arr = npz['fit_arr']
best_fit = npz['best_fit']
n_peaks = xcen_arr.shape[0]
for pp in range(n_peaks):
ax[0].scatter(xcen_arr[pp],
fit_arr[pp, :, 2],
marker='o',
edgecolor='k',
facecolor='none')
ax[0].axhline(y=best_fit[2], color='b')
ax[1].scatter(xcen_arr[pp],
fit_arr[pp, :, 1],
marker='o',
edgecolor='k',
facecolor='none')
ax[1].axhline(y=best_fit[1], color='b')
ax[2].scatter(xcen_arr[pp],
fit_arr[pp, :, 0],
marker='o',
edgecolor='k',
facecolor='none')
ax[2].axhline(y=best_fit[0], color='b')
ax[0].annotate(r'x = A y$^2$ + B y + C',
xy=(0.02, 0.95),
xycoords='axes fraction',
ha='left',
va='top')
ax[0].set_ylabel('C')
ax[1].set_ylabel('B')
ax[2].set_ylabel('A')
ax[0].set_ylim([-10, 0])
ax[1].set_ylim([-0.025, 0.025])
ax[2].set_ylim([-0.001, 0.001])
ax[0].set_xticklabels([])
ax[1].set_xticklabels([])
ax[2].set_xlabel('X [pix]')
plt.subplots_adjust(left=0.15, right=0.99, top=0.99, bottom=0.1)
out_pdf = path + 'distort_grid.pdf'
log.info('Writing : ' + out_pdf)
fig.savefig(out_pdf)
def main(rawdir, line_source='', mylogger=None, silent=False, verbose=True):
'''
Main function for wave_cal_script
Parameters
----------
rawdir : str
Path to raw files. Must end in a '/'
line_source : str
Type of lines to use for wavelength calibration. Option is
either 'arc' or 'OH
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 7-8 October 2017
Modified by Chun Ly, 8 November 2017
- Specify lampspec and outspec
Modified by Chun Ly, 16 November 2017
- Handle line_source == 'OH'
- Generalized arrays: arc -> frame
- Specify GNIRS log file
Modified by Chun Ly, 20 November 2017
- Bug fix: prefix needed for [frames] for line_source == OH
Modified by Chun Ly, 9 January 2018
- Import glog and call for stdout and ASCII logging
Modified by Chun Ly, 9 January 2018
- Allow mylogger keyword
Modified by Chun Ly, 30 May 2018
- Change order for nswavelength fitting
Modified by Chun Ly, 16 June 2018
- Change coordlist for OH skylines
Modified by Chun Ly, 19 June 2018
- Force function to legendre
- Set fwidth in nswavelength call to depend on slitwidth
Modified by Chun Ly, 21 June 2018
- Include ending print statement
Modified by Chun Ly, 10 July 2018
- Modify threshold for OH lines from 50 to 25
'''
# Mod on 10/01/2018
if type(mylogger) == type(None):
logfile = rawdir + 'QA_wave_cal.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin main : ' + systime())
timestamp = systime().replace(':', '.')
logfile = rawdir + 'gnirs_' + timestamp + '.log'
rawdir = check_path(rawdir)
if line_source == '':
mylogger.warn("Must specify line_source keyword: ")
mylogger.warn("line_source='arc' or line_source='OH'")
mylogger.warn("Exiting!!!")
return
else:
out_script = rawdir + 'wave_cal_' + line_source + '.py'
if silent == False: mylogger.info('Writing : ' + out_script)
f0 = open(out_script, 'w')
line0 = [
'iraf.gemini(_doprint=0)', 'iraf.gemini.gnirs(_doprint=0)',
'iraf.gemini.unlearn()', 'iraf.gemini.gemtools.unlearn()',
'iraf.gemini.gnirs.unlearn()', 'iraf.set(stdimage="imt4096")',
'iraf.gemini.nsheaders("gnirs")'
]
#'iraf.gemini.gnirs.logfile = "%s"' % logfile] # + on 16/11/2017
f0.writelines("\n".join(line0) + "\n")
if line_source == 'arc': frame_list = rawdir + 'arc.lis'
if line_source == 'OH': frame_list = rawdir + 'obj.OH.lis'
frames = np.loadtxt(frame_list, dtype=type(str))
if line_source == 'OH':
frames = ['rbnc' + file0 for file0 in frames]
frame_hdr = fits.getheader(rawdir + frames[0])
crpix = n_sp_pix / 2.0
crval = frame_hdr['gratwave'] * 1e4 # in Angstroms
if frame_hdr['FILTER2'] == 'X_G0518':
cdelt = -0.094 * 1e4 / n_sp_pix
if frame_hdr['FILTER2'] == 'J_G0517':
cdelt = -0.113 * 1e4 / n_sp_pix
mylogger.info('## CRVAL : %.1f ' % crval)
mylogger.info('## CDELT : %.1f CRPIX : %.1f' % (cdelt, crpix))
line1 = ['crval = %f' % crval, 'crpix = %f' % crpix, 'cdelt = %f' % cdelt]
f0.writelines("\n".join(line1) + "\n")
# Get slit length | + on 19/06/2018q
slitwidth = np.float(frame_hdr['SLIT'].split('arcsec')[0])
pscale = np.sqrt(frame_hdr['CD1_1']**2 +
frame_hdr['CD2_1']**2) * 3600.0 * u.arcsec
fwidth = 1.5 * slitwidth / pscale.to(u.arcsec).value
if line_source == 'arc':
coordlist = 'gnirs$data/argon.dat'
database = 'database/'
threshold = 50
if line_source == 'OH':
coordlist = rawdir + 'rousselot2000_convl.dat'
if not exists(coordlist):
log.warn('File does not exists!!! : ' + coordlist)
database = 'database_OH/'
threshold = 25
# Mod on 08/11/2017
line2 = [
"coordlist = '%s'" % coordlist,
"database = '%s'" % database,
"lampspec = '%s_stack.fits'" % line_source,
"outspec = 'w%s_stack.fits'" % line_source,
"threshold = %f" % threshold,
"logfile = '%s'" % logfile
] # + on 16/11/2017
f0.writelines("\n".join(line2) + "\n")
# Mod on 08/11/2017, 16/11/2017
cmd = "iraf.gnirs.nswavelength(lampspec, outprefix='',"+\
"outspectra=outspec, crval=crval, cdelt=cdelt, crpix=crpix, "+\
"coordlist=coordlist, database=database, fl_inter='yes', "+\
"function='legendre', cradius=20, threshold=threshold, "+\
"fwidth=%.1f, " % fwidth +"order=3, logfile=logfile)"
f0.write(cmd + '\n')
f0.write('print("Completed! Return to other terminal and hit RETURN")\n')
f0.close()
if silent == False: mylogger.info('### End main : ' + systime())
def main(rawdir, silent=False, verbose=True):
'''
Main function for distort_trace
Parameters
----------
rawdir : str
Path to FITS file. Must include '/' at the end
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 27 June 2018
- Write multi-page PDF file
- Remove center value for middle of spectra
- Plot offsets
- Bug fix for curve_fit (use bb_med0); plot aesthetics (legend)
- Plot aesthetics (axes labeling), margin adjustments
- Call group() to get matplotlib markers and colors
- Write npz file using np.savez and np.load when available
- Do linear regression fit and save to npz file
Modified by Chun Ly, 28 June 2018
- Change linear regression to 2nd order, plot fit
Modified by Chun Ly, 29 June 2018
- Write PDF file in each datedir
- Minor changes to mylogger calls
Modified by Chun Ly, 31 July 2018
- Use sigma_clipped_stats; Search for multiple peaks
Modified by Chun Ly, 1 August 2018
- Smooth median signal (boxcar); mask for peaks
- Call group_index(); Require at leat 5 pixels for peak identification
- Handle multiple peaks
- Compute number of peaks only once
Modified by Chun Ly, 2 August 2018
- Use combine stack for peak identification (more sensitive)
- Get quick peak centers from combine stack
- Use peak in med0 if no combine stack or telluric spec
- Handle peak finding for both telluric and science data
- Handle multiple peaks in plotting
- Restrict fitting to within 10 pixels
Modified by Chun Ly, 3 August 2018
- Simplify code (x0_bb -> x0)
- Fix ValueError: Invalid rgba arg ""
- Fix bugs with n_files and use of 'x' datapoints
- Use median for x_cen_middle to deal with outliers
- Compute median using sigma_clipped_stats, exclude outliers from polyfit
- Switch flag to flag0 to avoid conflict
- Use peak when combine stack has single line
- Switch from curve_fit to weighted centering
- Define p_idx for single peak
Modified by Chun Ly, 7 August 2018
- Bug fix: Incorrect x0_diff
- Handle case when not all skysubtracted frames are used
(check for obj_rev.lis file)
- Force integer for index
- Avoid right edge issue (specific hack for one target)
- Use DQ array to masked bad pixels and edges
- Limit weighted computation within expected location
- Shift x0_max1 for sub-indexing
- Force x-limit range (zoom-in)
- Plot aesthetics: ax annotation
Modified by Chun Ly, 8 August 2018
- Plot fitting results
- Bug fix with savefig location, change labeling
Modified by Chun Ly, 9 August 2018
- Fix typo with wrong ax subplots use (row determination)
- Annotate plot with center
- Plot aesthetics: toplabel, white space, ylim
- Changes for right center for telluric and single peak science data cases
- Adjust ax.plot xrange
Modified by Chun Ly, 10 August 2018
- Compute median of trace_arr in each bin, best fit polynomial fit
- Plot best fit to median of trace_arr
- Update npz savefile with best_fit results
- Annotate plot with best fit
- Fix IndexError issue (limit v1 >= 0)
- Change fig suptitle (just the filename)
'''
if rawdir[-1] != '/': rawdir += '/'
logfile = rawdir + 'distort_trace.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin ! ')
bin_size = 25
dir_list, list_path = dir_check.main(rawdir,
mylogger=mylogger,
silent=silent,
verbose=verbose)
for path in list_path:
mylogger.info('Working on : ' + path.split('/')[-2])
out_pdf = path + 'distort_trace.pdf'
out_pdf1 = path + 'distort_trace_fit.pdf'
npz_file = path + 'distort_trace.npz'
obj_file = path + 'obj_rev.lis'
if not exists(npz_file):
if not exists(obj_file):
files = glob(path + 'tfrbncN????????S????.fits')
else:
mylogger.info('File found : ' + obj_file)
npfiles = np.loadtxt(obj_file, dtype=type(str))
files = [path + 'tfrbnc' + t_file for t_file in npfiles]
npfilesT = np.loadtxt(path + 'telluric.lis', dtype=type(str))
filesT = [path + 'tfrbnc' + t_file for t_file in npfilesT]
files += filesT
files.sort()
n_files = len(files)
mylogger.info('Number of files found : %i ' % n_files)
if n_files > 0:
pdf_pp = PdfPages(out_pdf1)
hdr0 = fits.getheader(files[0], extname='SCI')
n_bins = np.int(np.ceil(np.float(hdr0['NAXIS2'])) / bin_size)
x0 = np.arange(hdr0['NAXIS1'])
y0 = bin_size / 2.0 + bin_size * np.arange(n_bins)
no_c_file = 0
c_file = glob(path + 'obj_comb.fits')
if len(c_file) == 0:
mylogger.warn('Combine frame not found! Using single peak')
no_c_file = 1
n_peaks = 1
else:
t_c_im = fits.getdata(c_file[0], extname='SCI')
c_med0 = np.median(t_c_im, axis=0)
# Find peaks | + on 31/07/2018
x0_max = np.argmax(c_med0)
x0_min = np.argmin(c_med0)
idx_mask = np.where((np.absolute(x0 - x0_max) >= 10)
& (np.absolute(x0 - x0_min) >= 10))[0]
sm_c_med0 = convolve(c_med0, box_kernel)
t_mean, t_med, t_std = sigma_clipped_stats(
sm_c_med0[idx_mask], sigma=2, iters=20)
idx_det = np.where((sm_c_med0 - t_med) / t_std >= 5)[0]
list_peak = np.array(list(group_index(idx_det)))
peak_idx = [
xx for xx in range(len(list_peak))
if (list_peak[xx][1] -
list_peak[xx][0] >= 5 and list_peak[xx][0] < 625)
] # Avoid right edge issues
list_peak = list_peak[peak_idx]
n_peaks = len(peak_idx)
mylogger.info('Number of peaks found : ' + str(n_peaks))
peak_ctr = np.zeros(n_peaks)
for pp in range(n_peaks):
i1, i2 = list_peak[pp][0], list_peak[pp][1]
peak_ctr[pp] = i1 + np.argmax(c_med0[i1:i2])
trace_arr = np.zeros((n_peaks, n_files, n_bins))
xcen_arr = np.zeros((n_peaks, n_files))
fit_arr = np.zeros((n_peaks, n_files, 3))
for ff in range(n_files):
fig, ax = plt.subplots(nrows=nrows, ncols=ncols)
t_im0, t_hdr = fits.getdata(files[ff],
extname='SCI',
header=True)
t_dq = fits.getdata(files[ff], extname='DQ')
t_im = np.ma.array(t_im0, mask=t_dq)
med0 = np.ma.median(t_im, axis=0)
x0_max0 = np.ma.argmax(med0)
h_obj = t_hdr['OBJECT']
if no_c_file or ('HIP' in h_obj or 'HD' in h_obj):
n_peak, use_peak = 1, 1
else:
n_peak = n_peaks
use_peak = 1 if n_peaks == 1 else 0
x0_diff = peak_ctr[0] - x0_max0
for bb in range(n_bins):
row, col = bb / ncols, bb % ncols
ty1, ty2 = (0 + bb) * bin_size, (1 + bb) * bin_size
bb_med0 = np.ma.median(t_im[ty1:ty2], axis=0)
for pp in range(n_peak):
if use_peak == 1:
if no_c_file or ('HIP' in h_obj
or 'HD' in h_obj):
v1, v2 = x0_max0 - 15, x0_max0 + 15
else:
v1 = np.max([
np.int(list_peak[0][0] - x0_diff - 15),
0
])
v2 = np.int(list_peak[0][1] - x0_diff + 15)
# print ff, bb, pp, v1, v2, x0_max0
x0_max1 = v1 + np.ma.argmax(bb_med0[v1:v2])
p_idx = np.arange(x0_max1 - 10, x0_max1 + 10)
# print ff, bb, pp, x0_max0, x0_max1, p_idx[0], p_idx[-1]
else:
p_idx = np.arange(
np.int(list_peak[pp][0] - x0_diff),
np.int(list_peak[pp][1] - x0_diff))
p_med0 = bb_med0[p_idx]
x_cen = np.sum(p_med0 * p_idx) / np.sum(p_med0)
#p0 = [0.0, y0_max, x0_max, 2.0]
#try:
# popt, pcov = curve_fit(gauss1d, x0, bb_med0, p0=p0)
# x_cen = popt[2]
#except RuntimeError:
# print 'Runtime error'
# x_cen = p0[2]
trace_arr[pp, ff, bb] = x_cen
x_off = p_idx[len(p_idx) / 2]
ax[row, col].plot(p_idx - x_off,
p_med0 / max(p_med0))
ax[row, col].axvline(x=x_cen - x_off)
ax[row, col].annotate('%.1f' % x_cen,
xy=[0.95, 0.95],
ha='right',
va='top',
fontsize=9,
xycoords='axes fraction')
if row != nrows - 1:
ax[row, col].set_xticklabels([])
if col != 0:
ax[row, col].set_yticklabels([])
ax[row, col].set_ylim([-0.05, 1.05])
#endfor
#endfor
plt.subplots_adjust(left=0.08,
right=0.99,
bottom=0.05,
top=0.95,
hspace=0.02,
wspace=0.02)
fig.suptitle(os.path.basename(files[ff]), fontsize=12)
fig.savefig(pdf_pp, format='pdf')
#endfor
mylogger.info('Writing : ' + out_pdf1)
pdf_pp.close()
flag0 = np.ones((n_peaks, n_files, n_bins))
for ff in range(n_files):
for pp in range(n_peaks):
t_me, t_md, t_s = sigma_clipped_stats(trace_arr[pp,
ff],
sigma=3,
iters=10)
x_cen_middle = t_md
xcen_arr[pp, ff] = x_cen_middle
trace_arr[pp, ff] -= x_cen_middle
diff = trace_arr[pp, ff] - ((y0 - 512) * 0.019)
use = np.where(np.absolute(diff) <= 5)[0]
if len(use) > 0: flag0[pp, ff, use] = 0
fit = np.polyfit(y0[use], trace_arr[pp, ff][use], 2)
fit_arr[pp, ff] = fit
#endfor
#endfor
mylogger.info('Writing : ' + npz_file)
np.savez(npz_file,
trace_arr=trace_arr,
xcen_arr=xcen_arr,
fit_arr=fit_arr,
y0=y0,
flag0=flag0)
else:
mylogger.warn('Files not found !')
else:
mylogger.info('Reading : ' + npz_file)
npz = np.load(npz_file)
trace_arr = npz['trace_arr']
xcen_arr = npz['xcen_arr']
y0 = npz['y0']
fit_arr = npz['fit_arr']
flag0 = npz['flag0']
n_files = xcen_arr.shape[1]
n_peaks = xcen_arr.shape[0]
if n_files > 0:
fig, ax = plt.subplots()
xlim = [-10, 10]
ctype, mtype, labels = group(xcen_arr)
x_fit0 = np.zeros(len(y0))
for bb in range(len(y0)):
use = np.where(flag0[:, :, bb] == 0)
x_fit0[bb] = np.median(trace_arr[use[0], use[1], bb])
best_fit = np.polyfit(y0, x_fit0, 2)
mylogger.info('Best fit : [%f, %f, %f]', best_fit[0], best_fit[1],
best_fit[2])
for pp in range(n_peaks):
for ff in range(n_files):
if labels[pp, ff] != '':
fc = ctype[pp, ff] if mtype[pp, ff] == 'x' else 'none'
ax.scatter(trace_arr[pp, ff, :],
y0,
marker=mtype[pp, ff],
alpha=0.5,
edgecolor=ctype[pp, ff],
facecolor=fc,
label=labels[pp, ff])
ax.scatter(trace_arr[pp, ff, flag0[pp, ff] == 1],
y0[flag0[pp, ff] == 1],
marker='x',
color='r')
pd = np.poly1d(fit_arr[pp, ff])
ax.plot(pd(y0),
y0,
color=ctype[pp, ff],
linewidth=0.75,
alpha=0.5)
ax.scatter(x_fit0,
y0,
marker='o',
edgecolor='none',
facecolor='black',
linewidth=2.0,
alpha=0.9)
best_pd = np.poly1d(best_fit)
ax.plot(best_pd(y0), y0, color='black', linewidth=2.0, alpha=0.9)
b_txt = r'x = A y$^2$ + B y + C'+\
'\nA = %.3e\nB = %.3e\nC = %.3f' % (best_fit[0],best_fit[1],
best_fit[2])
ax.annotate(b_txt,
xy=(0.05, 0.75),
xycoords='axes fraction',
ha='left',
va='top')
out_plt = np.where((trace_arr > xlim[1]) | (trace_arr < xlim[0]))
n_out_plt = len(out_plt[0])
flagged = np.where(flag0 == 1)
n_flagged = len(flagged[0])
a_txt = 'N(exclude) = ' + str(n_out_plt) + '\nN(flagged) = ' + str(
n_flagged)
ax.annotate(a_txt,
xy=(0.05, 0.95),
xycoords='axes fraction',
ha='left',
va='top')
ax.legend(loc='lower right')
ax.set_ylabel('Y [pixels]', fontsize=14)
ax.set_xlabel('X relative to center [pixels]', fontsize=14)
ax.set_xlim(xlim) #[-100,100])
ax.set_ylim([-10, 1050])
fig.suptitle(path)
ax.minorticks_on()
fig.set_size_inches(8, 8)
plt.subplots_adjust(left=0.1, right=0.98, bottom=0.07, top=0.98)
mylogger.info('Writing : ' + out_pdf)
fig.savefig(out_pdf, format='pdf')
mylogger.info('Updating : ' + npz_file)
np.savez(npz_file,
trace_arr=trace_arr,
xcen_arr=xcen_arr,
fit_arr=fit_arr,
y0=y0,
flag0=flag0,
best_fit=best_fit)
#endfor
if silent == False: mylogger.info('### End ! ')
def main(path0, out_pdf='', silent=False, verbose=True, overwrite=False):
'''
Main function to generate PDF illustrating alignment on target
Parameters
----------
path0 : str
Path to FITS file. Must include '/' at the end
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
overwrite : boolean
Overwrite files if they exists. Default: False
Returns
-------
Notes
-----
Created by Chun Ly, 24 March 2017
Modified by Chun Ly, 01 April 2017
- Handle CRs and bad pixels using cosmicrays_lacosmic
Modified by Chun Ly, 04 April 2017
- Use find_gnirs_window_mean to find center
Modified by Chun Ly, 04-05 April 2017
- Adjust greyscale limits to handle slit image (make it black),
and faint sources
Use find_gnirs_window_mean to find center
Modified by Chun Ly, 05 April 2017
- Handle alignment sequences with more than just 4 frames
- Handle excess subplots for individual PDF pages (remove axes)
- Compute seeing FWHM for acquisition images
Modified by Chun Ly, 06 April 2017
- Get coordinates for slit in cutout
Modified by Chun Ly, 11 May 2017
- Use slit image to find center when telluric data is only available
Modified by Chun Ly, 3 July 2017
- Add overwrite option to prevent overwriting file
Modified by Chun Ly, 9 January 2018
- Import glog and call for stdout and ASCII logging
- Pass mylogger to find_gnirs_window_mean(), find_gnirs_window()
Modified by Chun Ly, 20 April 2018
- Pass mylogger to gauss2d_fit()
- Switch print statements to mylogger calls
Modified by Chun Ly, 22 April 2018
- Bug fix: mylogger calls mistakes
- Bug fix: mylogger calls mistakes (cont'd)
'''
# + on 09/01/2018
logfile = path0 + 'align_check.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin main : ' + systime())
dir_list, list_path = dir_check.main(path0,
mylogger=mylogger,
silent=silent,
verbose=verbose)
out_pdf_default = out_pdf
for path in list_path:
infile = path + 'hdr_info.QA.tbl'
if not exists(infile):
mylogger.warning('File does not exist : ' + infile)
mylogger.warning('Exiting!!! ' + systime())
return
out_pdf = path + 'align_check.pdf' if out_pdf == '' else path + out_pdf
# Mod on 03/07/2017
if overwrite == False and exists(out_pdf):
mylogger.warn('File exists!! Will not overwrite ' + out_pdf)
else:
pp = PdfPages(out_pdf)
if silent == False: mylogger.info('Reading: ' + infile)
tab0 = asc.read(infile, format='fixed_width_two_line')
align = [ii for ii in xrange(len(tab0)) if tab0['QA'][ii] == 'N/A']
if silent == False:
mylogger.info('Number of alignment images found : ' +
str(len(align)))
ID = tab0['object'][align]
ID0 = list(set(ID)) #Unique ID's
if silent == False:
mylogger.info('Sources found : ' + ', '.join(ID0))
# + on 04/04/2017
win_ref_idx = [
tt for tt in xrange(len(tab0))
if (tab0['QA'][tt] == 'N/A') and ('Acq' in tab0['slit'][tt])
and ('HIP' not in tab0['object'][tt]) and (
'HD' not in tab0['object'][tt])
]
# Mod on 11/05/2017
if len(win_ref_idx) > 0:
win_ref_file = path + tab0['filename'][win_ref_idx[0]]
mylogger.info('Reference image for finding GNIRS window : ' +
win_ref_file)
x_min, x_max, y_min, y_max, x_cen, \
y_cen = find_gnirs_window_mean(win_ref_file, mylogger=mylogger)
else:
mylogger.info('Using telluric image as reference')
win_ref_file = path + tab0['filename'][0]
slit_x0, slit_y0_lo, slit_y0_hi = get_slit_trace(win_ref_file)
x_min, x_max = min(slit_x0), max(slit_x0)
x_cen = (x_min + x_max) / 2.0
y_cen = (np.median(slit_y0_lo) + np.median(slit_y0_hi)) / 2.0
y_min, y_max = y_cen - size2d[0].value / 2.0, y_cen + size2d[
0].value / 2.0
pos_cen = (x_cen, y_cen)
new_size = u.Quantity((y_max - y_min, x_max - x_min), u.pixel)
# + on 20/04/2018, Mod on 22/04/2018
mylogger.info('pos_cen : (%f, %f) ' % (pos_cen[0], pos_cen[1]))
mylogger.info('new_size : [%f, %f] pix ' %
(new_size[0].value, new_size[1].value))
for ii in xrange(len(ID0)):
t_idx = [
tt for tt in xrange(len(tab0)) if
(tab0['object'][tt] == ID0[ii] and tab0['QA'][tt] == 'N/A')
]
t_files = [path + a for a in tab0['filename'][t_idx]]
ncols = 2.0
nrows = 2 # np.ceil(len(t_idx)/ncols)
ncols, nrows = np.int(ncols), np.int(nrows)
# Mod on 05/04/2017
if len(t_idx) <= nrows * ncols:
fig, ax_arr = plt.subplots(nrows=nrows, ncols=ncols)
#med0, x_min, x_max, y_min, \
# y_max, x_cen, y_cen = find_gnirs_window(t_files[1], mylogger=mylogger)
# Later + on 24/03/2017 | Mod on 04/04/2017
xcen, ycen = find_star(t_files[-1],
pos=pos_cen,
find_size2d=new_size)
# Fix to get relative coordinate for Cutout2D image
#xcen -= pos_cen[0]-new_size[1].value/2.0
#ycen -= pos_cen[1]-new_size[0].value/2.0
slit_x0, slit_y0_lo, slit_y0_hi = get_slit_trace(
t_files[0]) #, x_min, x_max)
# Adjust values for offset that is applied
# Bug: Mod on 04/04/2017 to get proper coordinate
slit_x0 -= np.int64(pos_cen[0] - size2d[1].value / 2.0)
slit_y0_lo -= pos_cen[1] - size2d[0].value / 2.0
slit_y0_hi -= pos_cen[1] - size2d[0].value / 2.0
for jj in xrange(len(t_idx)):
jj_idx = t_idx[jj]
# + on 05/04/2017
if len(t_idx) > (nrows * ncols):
if jj % (nrows * ncols) == 0:
fig, ax_arr = plt.subplots(nrows=nrows,
ncols=ncols)
im0 = fits.getdata(t_files[jj])
hdr0 = fits.getheader(t_files[jj], ext=0) # Get WCS header
# + 01/04/2017
im0_clean = cosmicray_lacosmic(im0, sigclip=10)[0]
cutout = Cutout2D(im0_clean,
pos_cen,
size2d,
mode='partial',
fill_value=np.nan)
t_col, t_row = jj % ncols, (jj / ncols) % nrows
# Mod on 04/04/2017 to handle bright and faint stars
max0 = np.max(cutout.data)
# Compute median within GNIRS window
# + on 04-05/04/2017
temp = im0_clean[-50:-1, :]
bgd0, sig0 = np.median(temp), np.std(temp)
idx_y, idx_x = np.where(im0_clean > (bgd0 + 5 * sig0))
med0 = np.median(im0_clean[idx_y, idx_x])
mylogger.info('## max0 : %f med0 : %f ' % (max0, med0))
if max0 > 50000:
z1, z2 = zscale.get_limits(cutout.data)
z2 = max0 # Change for better stretch for telluric star
else:
if ('Acq_' not in tab0['slit'][jj_idx]) and \
(tab0['exptime'][jj_idx] == 3):
# First frame that will show the longslit
z1, z2 = 0.0, 0.5 * max0
else:
# This should handle faint and bright stars
z1, z2 = 0.5 * med0, max0
norm = ImageNormalize(vmin=z1, vmax=z2)
t_ax = ax_arr[t_row, t_col]
t_ax.imshow(cutout.data,
cmap='Greys',
origin='lower',
norm=norm)
#aplpy.FITSFigure(cutout)
# Draw trace of slit
t_ax.plot(slit_x0, slit_y0_lo, 'r-')
t_ax.plot(slit_x0, slit_y0_hi, 'r-')
t_ax.xaxis.set_ticklabels([])
t_ax.yaxis.set_ticklabels([])
fig.suptitle(path, fontsize=14)
txt0 = tab0['filename'][jj_idx] + '\n'
txt0 += tab0['datelabel'][jj_idx] + '\n'
txt0 += tab0['UT_date'][jj_idx] + '\n'
txt0 += tab0['object'][jj_idx]
t_ax.annotate(txt0, [0.025, 0.95],
xycoords='axes fraction',
ha='left',
va='top')
# Plot inset | Later + on 24/03/2017
axins = zoomed_inset_axes(t_ax, 6, loc=4)
norm2 = ImageNormalize(vmin=z1, vmax=z2)
axins.imshow(cutout.data,
cmap='Greys',
origin='lower',
norm=norm2)
# Draw trace of slit
axins.plot(slit_x0, slit_y0_lo, 'r-')
axins.plot(slit_x0, slit_y0_hi, 'r-')
# Mod on 04/04/2017 to get Cutout2d coordinates
c_xcen = xcen - (pos_cen[0] - size2d[1].value / 2.0)
c_ycen = ycen - (pos_cen[1] - size2d[0].value / 2.0)
x1, x2, y1, y2 = c_xcen - 20, c_xcen + 20, c_ycen - 20, c_ycen + 20
axins.set_xlim([x1, x2])
axins.set_ylim([y1, y2])
axins.xaxis.set_ticklabels([])
axins.yaxis.set_ticklabels([])
mark_inset(t_ax,
axins,
loc1=1,
loc2=3,
fc="none",
ec="b",
ls='dotted',
lw=0.5)
# Compute FWHM of alignment star | + on 05/04/2017
if ('Acq_' not in tab0['slit'][jj_idx]) and \
(tab0['exptime'][jj_idx] == 3):
mylogger.info('No source in slit : ' +
tab0['filename'][jj_idx])
else:
# + on 06/04/2017
c_size2d = u.Quantity((40, 40), u.pixel)
c_slit_x0 = slit_x0 - (c_xcen -
c_size2d[1].value / 2.0)
c_slit_y0_lo = slit_y0_lo - (c_ycen -
c_size2d[0].value / 2.0)
c_slit_y0_hi = slit_y0_hi - (c_ycen -
c_size2d[0].value / 2.0)
im0_crop = Cutout2D(cutout.data, (c_xcen, c_ycen),
c_size2d,
mode='partial',
fill_value=np.nan)
gauss2d_fit(im0_crop.data,
hdr0,
t_ax,
c_slit_x0,
c_slit_y0_lo,
c_slit_y0_hi,
mylogger=mylogger) # Mod on 06/04/2017
# Write each page separately | + on 05/04/2017
if len(t_idx) > (nrows * ncols):
# Mod later on 05/04/2017 to handle excess subplots
if jj == len(t_idx) - 1:
rem0 = len(t_idx) % (nrows * ncols) # remainder
if rem0 != 0:
for rr in range(rem0, nrows * ncols, 1):
t_col, t_row = rr % ncols, (rr /
ncols) % nrows
ax_arr[t_row, t_col].axis('off')
if (jj % (nrows * ncols) == nrows*ncols-1) or \
(jj == len(t_idx)-1):
subplots_adjust(left=0.02,
bottom=0.02,
top=0.95,
right=0.98,
wspace=0.02,
hspace=0.02)
fig.set_size_inches(11, 8)
fig.savefig(pp, format='pdf')
#endfor
# Mod on 05/04/2017
if len(t_idx) <= nrows * ncols:
# Mod later on 05/04/2017 to handle excess subplots
for rr in range(len(t_idx), nrows * ncols):
t_col, t_row = rr % ncols, (rr / ncols) % nrows
ax_arr[t_row, t_col].axis('off')
subplots_adjust(left=0.02,
bottom=0.02,
top=0.95,
right=0.98,
wspace=0.02,
hspace=0.02)
fig.set_size_inches(11, 8)
fig.savefig(pp, format='pdf')
#endfor
pp.close()
#endelse
out_pdf = out_pdf_default
if silent == False: mylogger.info('### End main : ' + systime())
def arc_check(path, arcs=[''], out_pdf='', stack=False, silent=False, verbose=True):
'''
Generate plot illustrating wavelength calibration from IRAF database
Parameters
----------
path : str
Full path to where output PDF and FITS file are located. Must end
with a '/'
arcs : str or list (Optional)
List of raw filenames for the arc data (e.g., 'N20170101S0111.fits')
out_pdf : str
Filename for output PDF. Do NOT include full path
stack : boolean
Indicate whether to use stacked arc data. Default: False
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
Notes
-----
Created by Chun Ly, 18 May 2017
Modified by Chun Ly, 19 May 2017
- Fix bug with scatter plot (wrong indexing)
- Fix bug with l_val0 and l_val (those were flipped)
- Draw dashed lines connecting the points
- Adjust margins, add labels
- Adjust image scale (Using zscale)
- Annotate each page with the dataframe information
- Label lines
- Aesthetic changes for plotting
- Aesthetic changes for plotting
- subplots_adjust
- Better labeling of lines
- xlim, ylim set
- Adjust PDF paper size
Modified by Chun Ly, 11 November 2017
- Added stack keyword option to use stacked arc products
Modified by Chun Ly, 15 November 2017
- Bug fix: Incorrect replace
Modified by Chun Ly, 16 November 2017
- Change prefix: rnc to rbnc
Modified by Chun Ly, 9 January 2018
- Import glog and call for stdout and ASCII logging
'''
# + on 09/01/2018
logfile = path+'QA_wave_cal.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin arc_check : '+systime())
# Mod on 11/11/2017
if stack == True:
arcs = ['arc_stack.fits'] # + on 15/11/2017
rnc_files = [path + 'arc_stack.fits']
d_files = [path+'database/idwarc_stack_SCI_1_']
else:
if arcs[0] == '':
arc_list = path + 'arc.lis'
if silent == False: mylogger.info('Reading : '+arc_list)
arcs = np.loadtxt(arc_list, dtype=type(str))
# Mod on 16/11/2017
rnc_files = [path+'rbnc'+file0 for file0 in arcs]
d_files = [path+'database/idwrbnc'+file0.replace('.fits','_SCI_1_') \
for file0 in arcs]
n_arcs = len(rnc_files) # Mod on 11/11/2017
out_pdf = path+'arc_check.pdf' if out_pdf == '' else path+out_pdf
pp = PdfPages(out_pdf)
for nn in xrange(n_arcs):
hdu0 = fits.open(rnc_files[nn])
im0 = hdu0['SCI'].data
fig, ax = plt.subplots()
z1, z2 = zscale.get_limits(im0)
norm = ImageNormalize(vmin=z1, vmax=z2)
ax.imshow(im0, cmap='Greys', origin='lower', norm=norm)
f0 = open(d_files[nn], 'r')
str0 = f0.readlines()
beg_mark = [ii for ii in xrange(len(str0)) if 'begin' in str0[ii]]
feat_mark = [ii for ii in xrange(len(str0)) if 'features' in str0[ii]]
func_mark = [ii for ii in xrange(len(str0)) if 'function' in str0[ii]]
n_cols = len(beg_mark)
n_features = [np.int(val.split('\t')[2].replace('\n','')) for val in
np.array(str0)[feat_mark]]
n_max = max(n_features)
x_cols, y_val = np.zeros(n_cols), np.zeros((n_cols,n_max))
l_val0, l_val = np.zeros((n_cols,n_max)), np.zeros((n_cols,n_max))
for cc in xrange(n_cols):
temp = str0[beg_mark[cc]].split(' ')[1]
# Mod on 16/11/2017
temp = temp.replace('wrbnc'+arcs[nn].replace('.fits','[SCI,1]['),'')
temp = temp.replace('w'+arcs[nn].replace('.fits','[SCI,1]['),'') # + on 15/11/2017
x_cols[cc] = np.int(temp.split(',')[0])
for rr in xrange(n_features[cc]): #beg_mark[cc]+1, func_mark[cc]):
temp1 = str0[feat_mark[cc]+1+rr].replace(' ','')
temp1 = temp1.split(' ')
temp1 = [val for val in temp1 if val != '']
y_val[cc,rr] = np.float(temp1[1])
try:
l_val[cc,rr] = np.float(temp1[2])
except ValueError:
pass
try:
l_val0[cc,rr] = np.float(temp1[3])
except ValueError:
pass
#endfor
x_temp = np.repeat(x_cols[cc],n_features[cc])
y_temp = y_val[cc,0:n_features[cc]]
ax.scatter(x_temp, y_temp, facecolor='none', edgecolor='red',
marker='o', s=25, alpha=0.5)
#endfor
line_list = list(set(l_val0.reshape(n_cols*n_max).tolist()))
for ll in range(len(line_list)):
if line_list[ll] != 0:
x_idx, y_idx = np.where(l_val0 == line_list[ll])
sort0 = np.argsort(x_cols[x_idx])
x1 = x_cols[x_idx][sort0]
y1 = y_val[x_idx,y_idx][sort0]
ax.plot(x1, y1, 'r--', alpha=0.5)
ax.annotate(str(line_list[ll]), [690,np.average(y1)],
ha='right', va='center', fontsize=10,
color='red', bbox=bbox_props)
ax.set_xlabel('X [pixels]')
ax.set_ylabel('Y [pixels]')
ax.set_xlim([0,690])
ax.set_ylim([0,1025])
ax.minorticks_on()
subplots_adjust(left=0.095, bottom=0.025, top=0.995, right=0.99)
str0 = (rnc_files[nn]+'\n'+d_files[nn]).replace(path,'')
ax.annotate(str0, [0.025,0.975], xycoords='axes fraction', ha='left',
va='top', fontsize=14, bbox=bbox_props)
ax.set_title(path)
fig.set_size_inches(7.65,10.5) # Scale proportionally
fig.savefig(pp, format='pdf')
#endfor
pp.close()
if silent == False: mylogger.info('### End arc_check : '+systime())
def OH_check(path, objs='', out_pdf='', skysub=False, silent=False,
verbose=True, cross_check=False):
'''
Generate plot illustrating expected location of OH skyline to
check wavelength calibration
Parameters
----------
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
skysub : boolean
Display skysubtracted or un-skysubtracted images. Default: False
cross_check : boolean
Check arc-based wavelength calibration against OH skylines. Default: False
Returns
-------
Notes
-----
Created by Chun Ly, 19 May 2017
Modified by Chun Ly, 20 May 2017
- Draw OH night skyline emission on plots
Modified by Chun Ly, 2 June 2017
- Add skysub keyword option to use either sky-subtracted
images or those without skysubtraction (OH skylines more visible)
Modified by Chun Ly, 26 June 2017
- Plot more OH skylines (change threshold for OH_mark)
Modified by Chun Ly, 16 November 2017
- Change prefix: tfrnc to tfrbnc
Modified by Chun Ly, 9 January 2018
- Import glog and call for stdout and ASCII logging
Modified by Chun Ly, 31 May 2018
- Add cross_check keyword; Update input/outputs for cross_check == True
- Bug fix: n_obj for cross_check == True
Modified by Chun Ly, 10 July 2018
- Change OH skyline file to use the convolved one
'''
# + on 09/01/2018
logfile = path+'QA_wave_cal.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin OH_check : '+systime())
# + on 20/05/2017
OH_file = path+'rousselot2000_convl.dat'
if exists(OH_file):
if silent == False: mylogger.info('Reading : '+OH_file)
OH_data = np.loadtxt(OH_file)
OH_lines = OH_data[:,0]
OH_int = OH_data[:,1]
# Mod on 02/06/2017, 31/05/2018
if cross_check == False:
if objs == '':
obj_list = path + 'obj.lis' if skysub == True else \
path + 'obj.OH.lis'
if silent == False: mylogger.info('Reading : '+obj_list)
objs = np.loadtxt(obj_list, dtype=type(str))
n_obj = len(objs)
tfrnc_files = [path+'tfrbnc'+file0 for file0 in objs] # Mod on 16/11/2017
else:
tfrnc_files = [path+'tfOH_stack_arc.fits']
n_obj = len(tfrnc_files)
# + on 20/05/2017
chk = [file0 for file0 in tfrnc_files if exists(file0) == True]
if len(chk) == 0:
mylogger.warn('Files not found!!!')
mylogger.warn(', '.join(file0))
mylogger.warn('Exiting!!!')
return
# Mod on 02/06/2017, 31/05/2018
if cross_check == False:
if out_pdf == '':
out_pdf = path+'OH_check.pdf' if skysub == True else \
path+'OH_check.raw.pdf'
else:
out_pdf = path+out_pdf
else:
if out_pdf == '':
out_pdf = path+'OH_check_arc.pdf'
else:
out_pdf = path+out_pdf
pp = PdfPages(out_pdf)
for nn in xrange(n_obj):
if exists(tfrnc_files[nn]): # Mod on 20/05/2017
hdu0 = fits.open(tfrnc_files[nn])
im0 = hdu0['SCI'].data
gc0 = aplpy.FITSFigure(hdu0, hdu='SCI', figsize=(7.65,10.5))
z1, z2 = zscale.get_limits(im0)
gc0.show_grayscale(invert=True, vmin=z1, vmax=z2)
gc0.set_tick_color('black')
# + on 20/05/2017
hdr0 = hdu0['SCI'].header
crval2 = hdr0['CRVAL2']
cdelt2 = hdr0['CDELT2']
npix = hdr0['NAXIS2']
lamb_max = crval2 + cdelt2*npix
OH_mark = np.where((OH_lines >= crval2) & (OH_lines <= lamb_max))[0]
# Mod on 26/06/2017
OH_mark = np.where(OH_int >= 0.005*np.max(OH_int[OH_mark]))[0]
line_list = []
for ll in OH_mark:
line_list.append(np.array([[0,700], [OH_lines[ll],OH_lines[ll]]]))
gc0.show_lines(line_list, color='red', alpha=0.5, linewidth=1.0,
linestyle='dashed')
#subplots_adjust(left=0.095, bottom=0.025, top=0.995, right=0.99)
str0 = tfrnc_files[nn].replace(path,'')
gc0.add_label(0.025, 0.975, str0, color='red', relative=True,
ha='left', va='top', weight='medium', fontsize=14,
bbox=bbox_props)
gc0.set_axis_labels(xlabel='X [pixels]', ylabel=r'Wavelength ($\AA$)')
#gc0.set_axis_label_rotation(90)
gc0.savefig(pp, format='pdf')
#endif
#endfor
if silent == False: mylogger.info('Writing : '+out_pdf)
pp.close()
if silent == False: mylogger.info('### End OH_check : '+systime())
def main(path0, silent=False, verbose=True, overwrite=False):
'''
main() function to sort through data and create individual lists
Parameters
----------
path0 : str
Path to FITS file. Must include '/' at the end
silent : boolean
Turns off stdout messages. Default: False
verbose : boolean
Turns on additional stdout messages. Default: True
Returns
-------
ASCII files containing the filenames for each set
obj.lis - List of science data frames
sky.lis - List of science data frames to be used for sky-subtraction
arc.lis - List of Arclamp data frames
flat.lis - List of Flat data frames
telluric.lis - List of Telluric data frames
all.lis - List of all obj, sky, arc, flat, and telluric data frames
tab0 : astropy.table.Table
Astropy ASCII table with QA flag written to [path0]+'hdr_info.QA.tbl'
Notes
-----
Created by Chun Ly, 5 March 2017
- Later modified to output ASCII table (hdr_info.QA.tbl) containing a
column called 'QA' with information of how each FITS file was
classified. This is to enable quick check that each dataset if
properly classified
- Later modified to include all.lis output
- Later modified to define r0 (for slight speed improvement
Modified by Chun Ly, 23 March 2017
- Call dir_check.main() to handle multiple date directories
Modified by Chun Ly, 10 April 2017
- Handle cases when arc, flat, telluric, and sci data are not available
Modified by Chun Ly, 11 April 2017
- Handle case when no data for all.lis are available
(i.e., the observing sequence was canceled)
Modified by Chun Ly, 16 May 2017
- Change obj.lis and sky.lis to include both A-B and B-A sets
- Avoid QA flag of 'sky'
- Fix minor bug
Modified by Chun Ly, 17 May 2017
- Minor fix for i_sky to handle skipping of frames for i_obj
Modified by Chun Ly, 31 May 2017
- Minor fix for when no science data is available
Modified by Chun Ly, 1 June 2017
- Added overwrite keyword option to overwrite file
Modified by Chun Ly, 8 December 2017
- Import glog and call for stdout and ASCII logging
Modified by Chun Ly, 10 December 2017
- Adjust glog logging to a default file in path0, move up mylogger
definition
- Minor mylogger text changes
- Pass mylogger to dir_check.main()
Modified by Chun Ly, 12 January 2018
- Handle multiple telluric datasets
Modified by Chun Ly, 15 January 2018
- Handle multiple telluric datasets (cont'd) - Get indices for each dataset
Modified by Chun Ly, 16 January 2018
- Handle multiple telluric datasets (cont'd) - Update prefix and index lists
Modified by Chun Ly, 17 January 2018
- Create telluric.lis even for multi-telluric case
Modified by Chun Ly, 23 April 2018
- Handle no telluric data case
'''
logfile = path0 + 'create_list.log'
mylogger = glog.log0(logfile)._get_logger()
if silent == False: mylogger.info('### Begin main : ' + systime())
# + on 23/03/2017
dir_list, list_path = dir_check.main(path0,
mylogger=mylogger,
silent=silent,
verbose=verbose)
# Mod on 23/03/2017
for path in list_path:
infile = path + 'hdr_info.tbl'
if not exists(infile):
mylogger.warning('File does not exist : ' + infile)
mylogger.warning('Exiting!!! ' + systime())
return
if silent == False: mylogger.info('Reading: ' + infile)
tab0 = asc.read(infile, format='fixed_width_two_line')
len0 = len(tab0)
r0 = xrange(len0)
obstype = tab0['obstype']
object0 = tab0['object']
filter2 = tab0['filter2']
i_arc = [ii for ii in r0 if obstype[ii] == 'ARC']
i_flat = [ii for ii in r0 if obstype[ii] == 'FLAT']
i_tell = [
ii for ii in r0 if (obstype[ii] == 'OBJECT' and (
'HIP' in object0[ii] or 'HD' in object0[ii]) and (
'H2_' not in filter2[ii] and 'H_' not in filter2[ii]))
]
# Identify when multiple sets of telluric data is available
# Broken based on filename | + on 12/01/2018, Mod on 15/01/2018
# tel_name = list(set(np.array(object0[i_tell])))
multi_tell = 0
# Mod on 23/04/2018
if len(i_tell) != 0:
seq_tell = np.array([
np.int(str0.replace('.fits', '')[-4:])
for str0 in tab0['filename'][i_tell]
])
seq_diff = seq_tell[1:] - seq_tell[0:-1]
# +1 to factor first skip for seq_diff
seq_break = [
ii + 1 for ii in range(len(seq_diff)) if seq_diff[ii] != 1
]
if len(seq_break) > 0:
multi_tell = 1
i_tell0 = []
mylogger.warn('Multiple telluric star detected.')
mylogger.warn('Will split into separate files.')
ss_start = [0] + seq_break # relative to i_tell
ss_end = seq_break + [len(i_tell)]
for ss in range(len(ss_start)):
ss_idx = i_tell[ss_start[ss]:ss_end[ss]]
i_tell0.append(ss_idx)
else:
mylogger.info('Only one telluric star detected.')
i_sci = [
ii for ii in r0 if (obstype[ii] == 'OBJECT' and (
'HIP' not in object0[ii] and 'HD' not in object0[ii]) and (
'H2_' not in filter2[ii] and 'H_' not in filter2[ii]))
]
i_sci = np.array(i_sci)
# Note: Assumes that dithering pattern is ABA'B'
# Mod on 16/05/2017
# Mod on 17/05/2017 - Minor bug if things skip for sci frames
# Mod on 31/05/2017 - Minor bug if i_sci is empty
if len(i_sci) > 0:
i_obj = i_sci
i_off = [1, -1] * (len(i_sci) / 2)
if len(i_sci) % 2 == 1: i_off.append(-1) # Odd number correction
i_sky = i_sci[np.arange(len(i_sci)) + np.array(i_off)]
# i_sky = [a+b for a,b in zip(i_sci,i_off)]
# Mod on 10/04/2017
prefix, index = [], []
if len(i_arc) == 0:
mylogger.warn('No ARC data found!')
else:
prefix.append('arc')
index.append(i_arc)
if len(i_flat) == 0:
mylogger.warn('No FLAT data found!')
else:
prefix.append('flat')
index.append(i_flat)
if len(i_tell) == 0:
mylogger.warn('No Telluric data found!')
else:
# Mod on 16/01/2018, 17/01/2018
# Always create telluric.lis for all tellurics
prefix.append('telluric')
index.append(i_tell)
if multi_tell:
for ii in range(len(i_tell0)):
prefix.append('telluric' + str(ii + 1))
index.append(i_tell0[ii])
if len(i_sci) == 0:
mylogger.warn('No science data found!')
else:
prefix.append('obj')
prefix.append('sky')
index.append(i_obj)
index.append(i_sky)
zip0 = zip(prefix, index)
QA = ['N/A'] * len0 # Later + on 05/03/2017
for a, b in zip0:
if a != 'sky':
for idx in b:
QA[idx] = a # Mod on 16/05/2017
outfile = path + a + '.lis'
if overwrite == False and exists(outfile):
mylogger.warn('File exists! Will not write ' + outfile + '!!')
else:
if silent == False: mylogger.info('Writing : ' + outfile)
np.savetxt(outfile, tab0['filename'][b], fmt='%s')
#asc.write will not work. Will not produce single column
#asc.write(tab0[b], outfile, overwrite=True,
# format='no_header')
# Later + on 05/03/2017 | Mod on 11/04/2017
i_all = [ii for ii in r0 if QA[ii] != 'N/A']
if len(i_all) > 0:
outfile0 = path + 'all.lis'
if overwrite == False and exists(outfile0):
mylogger.warn('File exists! Will not write all.lis!!')
else:
if silent == False: mylogger.info('Writing : ' + outfile0)
np.savetxt(outfile0, tab0['filename'][i_all], fmt='%s')
else:
mylogger.warn('Will not write all.lis!!')
# Later + on 05/03/2017
col0 = Column(QA, name='QA')
tab0.add_column(col0)
# Later + on 05/03/2017
outfile2 = infile.replace('.tbl', '.QA.tbl')
if silent == False:
if overwrite == False and exists(outfile2):
mylogger.warn('File exists! Will not write ' + outfile2 + '!!')
else:
if not exists(outfile2):
mylogger.info('Writing : ' + outfile2)
else:
mylogger.info('Overwriting : ' + outfile2)
asc.write(tab0,
outfile2,
format='fixed_width_two_line',
overwrite=True)
if silent == False: mylogger.info('### End main : ' + systime())
